25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
28#if __cplusplus >= 201703L
30#include "simd_detail.h"
34#ifdef _GLIBCXX_DEBUG_UB
42#if _GLIBCXX_SIMD_X86INTRIN
44#elif _GLIBCXX_SIMD_HAVE_NEON
71_GLIBCXX_SIMD_BEGIN_NAMESPACE
73#if !_GLIBCXX_SIMD_X86INTRIN
74using __m128 [[__gnu__::__vector_size__(16)]] = float;
75using __m128d [[__gnu__::__vector_size__(16)]] = double;
76using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
77using __m256 [[__gnu__::__vector_size__(32)]] = float;
78using __m256d [[__gnu__::__vector_size__(32)]] = double;
79using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
80using __m512 [[__gnu__::__vector_size__(64)]] = float;
81using __m512d [[__gnu__::__vector_size__(64)]] = double;
82using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
104template <
int _UsedBytes>
107template <
int _UsedBytes>
108 struct _VecBltnBtmsk;
110template <
typename _Tp,
int _Np>
111 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
113template <
int _UsedBytes = 16>
114 using _Sse = _VecBuiltin<_UsedBytes>;
116template <
int _UsedBytes = 32>
117 using _Avx = _VecBuiltin<_UsedBytes>;
119template <
int _UsedBytes = 64>
120 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
122template <
int _UsedBytes = 16>
123 using _Neon = _VecBuiltin<_UsedBytes>;
128using __avx512 = _Avx512<>;
129using __neon = _Neon<>;
130using __neon128 = _Neon<16>;
131using __neon64 = _Neon<8>;
134template <
typename _Tp,
size_t _Np,
typename...>
138 using fixed_size = _Fixed<_Np>;
140using scalar = _Scalar;
145template <
typename _Tp>
148template <
typename _Tp>
151template <
typename _Tp,
typename _Abi>
154template <
typename _Tp,
typename _Abi>
157template <
typename _Tp,
typename _Abi>
162struct element_aligned_tag
164 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
165 static constexpr size_t _S_alignment =
alignof(_Up);
167 template <
typename _Tp,
typename _Up>
168 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
173struct vector_aligned_tag
175 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
176 static constexpr size_t _S_alignment
177 = std::__bit_ceil(
sizeof(_Up) *
_Tp::size());
179 template <
typename _Tp,
typename _Up>
180 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
183 return static_cast<_Up*
>(
184 __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
188template <
size_t _Np>
struct overaligned_tag
190 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
191 static constexpr size_t _S_alignment = _Np;
193 template <
typename _Tp,
typename _Up>
194 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
196 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
199inline constexpr element_aligned_tag element_aligned = {};
201inline constexpr vector_aligned_tag vector_aligned = {};
204 inline constexpr overaligned_tag<_Np> overaligned = {};
208 using _SizeConstant = integral_constant<size_t, _Xp>;
214 template <
typename _Tp>
215 _GLIBCXX_SIMD_INTRINSIC
constexpr
217 operator()(_Tp __a, _Tp __b)
const
220 return min(__a, __b);
226 template <
typename _Tp>
227 _GLIBCXX_SIMD_INTRINSIC
constexpr
229 operator()(_Tp __a, _Tp __b)
const
232 return max(__a, __b);
239template <
typename _Fp,
size_t... _I>
240 _GLIBCXX_SIMD_INTRINSIC
constexpr void
241 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
242 { ((void)__f(_SizeConstant<_I>()), ...); }
244template <
typename _Fp>
245 _GLIBCXX_SIMD_INTRINSIC
constexpr void
246 __execute_on_index_sequence(_Fp&&, index_sequence<>)
249template <
size_t _Np,
typename _Fp>
250 _GLIBCXX_SIMD_INTRINSIC
constexpr void
251 __execute_n_times(_Fp&& __f)
253 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
254 make_index_sequence<_Np>{});
259template <
typename _R,
typename _Fp,
size_t... _I>
260 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
261 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
262 {
return _R{__f(_SizeConstant<_I>())...}; }
264template <
size_t _Np,
typename _R,
typename _Fp>
265 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
266 __generate_from_n_evaluations(_Fp&& __f)
268 return __execute_on_index_sequence_with_return<_R>(
269 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
274template <
size_t... _I,
typename _F0,
typename _FArgs>
275 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
276 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
277 {
return __f0(__fargs(_SizeConstant<_I>())...); }
279template <
size_t _Np,
typename _F0,
typename _FArgs>
280 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
281 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
283 return __call_with_n_evaluations(make_index_sequence<_Np>{},
284 static_cast<_F0&&
>(__f0),
285 static_cast<_FArgs&&
>(__fargs));
290template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
291 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
292 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
293 {
return __fun(__x[_First + _It]...); }
295template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
296 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
297 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
299 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
300 make_index_sequence<_Np>(),
301 static_cast<_Fp&&
>(__fun));
308using _UChar =
unsigned char;
309using _SChar =
signed char;
310using _UShort =
unsigned short;
311using _UInt =
unsigned int;
312using _ULong =
unsigned long;
313using _ULLong =
unsigned long long;
314using _LLong =
long long;
318template <
typename _T0,
typename...>
319 struct __first_of_pack
320 {
using type = _T0; };
322template <
typename... _Ts>
323 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
327template <
typename _Tp>
328 typename _Tp::value_type
329 __value_type_or_identity_impl(
int);
331template <
typename _Tp>
333 __value_type_or_identity_impl(
float);
335template <
typename _Tp>
336 using __value_type_or_identity_t
337 =
decltype(__value_type_or_identity_impl<_Tp>(
int()));
341template <
typename _Tp>
342 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
345 struct __is_vectorizable<bool> :
public false_type {};
347template <
typename _Tp>
348 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
351template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
352 using _Vectorizable = _Tp;
356template <
typename _Ptr,
typename _ValueType>
357 struct __is_possible_loadstore_conversion
358 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
361 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
364template <
typename _Ptr,
typename _ValueType,
366 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
367 using _LoadStorePtr = _Ptr;
371template <
typename _Tp,
typename =
void_t<>>
374template <
typename _Tp>
375 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
378template <
typename _Tp>
379 struct __is_bitmask<_Tp,
380 void_t<decltype(
declval<unsigned&>() = declval<_Tp>() & 1u)>>
385#pragma GCC diagnostic push
386#pragma GCC diagnostic ignored "-Wpedantic"
387template <
size_t _Bytes>
391 if constexpr (_Bytes ==
sizeof(int))
394 else if constexpr (_Bytes ==
sizeof(char))
397 else if constexpr (_Bytes ==
sizeof(_SChar))
400 else if constexpr (_Bytes ==
sizeof(short))
403 else if constexpr (_Bytes ==
sizeof(long))
406 else if constexpr (_Bytes ==
sizeof(_LLong))
408 #ifdef __SIZEOF_INT128__
409 else if constexpr (_Bytes ==
sizeof(__int128))
412 else if constexpr (_Bytes %
sizeof(int) == 0)
414 constexpr size_t _Np = _Bytes /
sizeof(int);
419 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
422 return __generate_from_n_evaluations<_Np, _Ip>(
423 [&](
auto __i) {
return __rhs._M_data[__i] & _M_data[__i]; });
426 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
429 return __generate_from_n_evaluations<_Np, _Ip>(
430 [&](
auto __i) {
return __rhs._M_data[__i] | _M_data[__i]; });
433 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
436 return __generate_from_n_evaluations<_Np, _Ip>(
437 [&](
auto __i) {
return __rhs._M_data[__i] ^ _M_data[__i]; });
440 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
443 return __generate_from_n_evaluations<_Np, _Ip>(
444 [&](
auto __i) {
return ~_M_data[__i]; });
450 static_assert(_Bytes != _Bytes,
"this should be unreachable");
452#pragma GCC diagnostic pop
454template <
typename _Tp>
455 using __int_for_sizeof_t =
decltype(__int_for_sizeof<sizeof(_Tp)>());
458 using __int_with_sizeof_t =
decltype(__int_for_sizeof<_Np>());
462template <
typename _Tp>
466 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
468template <
typename _Tp>
469 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
473constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
474constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
475constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
476constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
477constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
478constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
479constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
480constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
481constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
482constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
483constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
484constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
485constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
486constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
487constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
488constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
489constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
490constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
491constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
492constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
493constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
494constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
495constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
496constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
498constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
499constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
500constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
501constexpr inline bool __support_neon_float =
502#if defined __GCC_IEC_559
504#elif defined __FAST_MATH__
511constexpr inline bool __have_power10vec =
true;
513constexpr inline bool __have_power10vec =
false;
515#ifdef __POWER9_VECTOR__
516constexpr inline bool __have_power9vec =
true;
518constexpr inline bool __have_power9vec =
false;
520#if defined __POWER8_VECTOR__
521constexpr inline bool __have_power8vec =
true;
523constexpr inline bool __have_power8vec = __have_power9vec;
526constexpr inline bool __have_power_vsx =
true;
528constexpr inline bool __have_power_vsx = __have_power8vec;
530#if defined __ALTIVEC__
531constexpr inline bool __have_power_vmx =
true;
533constexpr inline bool __have_power_vmx = __have_power_vsx;
538template <
typename _Abi>
541 {
return is_same_v<simd_abi::scalar, _Abi>; }
545template <
template <
int>
class _Abi,
int _Bytes>
547 __abi_bytes_impl(_Abi<_Bytes>*)
550template <
typename _Tp>
552 __abi_bytes_impl(_Tp*)
555template <
typename _Abi>
556 inline constexpr int __abi_bytes_v
557 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
561template <
typename _Abi>
563 __is_builtin_bitmask_abi()
564 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
568template <
typename _Abi>
572 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
573 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
578template <
typename _Abi>
582 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
583 return _Bytes > 16 && _Bytes <= 32
584 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
589template <
typename _Abi>
593 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
594 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
599template <
typename _Abi>
603 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
604 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
609template <
typename,
typename _Up>
610 struct __make_dependent
611 {
using type = _Up; };
613template <
typename _Tp,
typename _Up>
614 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
620template <
typename... _Args>
621 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
622 __invoke_ub([[maybe_unused]]
const char* __msg,
623 [[maybe_unused]]
const _Args&... __args)
625#ifdef _GLIBCXX_DEBUG_UB
626 __builtin_fprintf(stderr, __msg, __args...);
629 __builtin_unreachable();
635template <
typename _Tp>
636 struct __assert_unreachable
637 {
static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
641template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
643 __size_or_zero_dispatch(
int)
646template <
typename _Tp,
typename _Ap>
648 __size_or_zero_dispatch(
float)
651template <
typename _Tp,
typename _Ap>
652 inline constexpr size_t __size_or_zero_v
653 = __size_or_zero_dispatch<_Tp, _Ap>(0);
657inline constexpr size_t
658__div_roundup(
size_t __a,
size_t __b)
659{
return (__a + __b - 1) / __b; }
668 _GLIBCXX_SIMD_INTRINSIC
constexpr _ExactBool(
bool __b) : _M_data(__b) {}
670 _ExactBool(
int) =
delete;
672 _GLIBCXX_SIMD_INTRINSIC
constexpr operator bool()
const {
return _M_data; }
682template <
typename _Tp>
683 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
688struct _UnsupportedBase
690 _UnsupportedBase() =
delete;
691 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
692 _UnsupportedBase& operator=(
const _UnsupportedBase&) =
delete;
693 ~_UnsupportedBase() =
delete;
709 using _SimdBase = _UnsupportedBase;
710 using _MaskBase = _UnsupportedBase;
712 static constexpr size_t _S_full_size = 0;
713 static constexpr bool _S_is_partial =
false;
715 static constexpr size_t _S_simd_align = 1;
717 struct _SimdMember {};
718 struct _SimdCastType;
720 static constexpr size_t _S_mask_align = 1;
722 struct _MaskMember {};
723 struct _MaskCastType;
728template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
729 struct _SimdTraits : _InvalidTraits {};
737inline constexpr struct _PrivateInit {} __private_init = {};
739inline constexpr struct _BitsetInit {} __bitset_init = {};
743template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
744 bool = is_arithmetic_v<_To>>
745 struct __is_narrowing_conversion;
749template <
typename _From,
typename _To>
750 struct __is_narrowing_conversion<_From, _To, true, true>
751 :
public __bool_constant<(
752 __digits_v<_From> > __digits_v<_To>
753 || __finite_max_v<_From> > __finite_max_v<_To>
754 || __finite_min_v<_From> < __finite_min_v<_To>
755 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
757template <
typename _Tp>
758 struct __is_narrowing_conversion<_Tp, bool, true, true>
762 struct __is_narrowing_conversion<bool, bool, true, true>
765template <
typename _Tp>
766 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
769template <
typename _From,
typename _To>
770 struct __is_narrowing_conversion<_From, _To, false, true>
771 :
public negation<is_convertible<_From, _To>> {};
775template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
776 struct __converts_to_higher_
integer_rank : public true_type {};
779template <
typename _From,
typename _To>
780 struct __converts_to_higher_integer_rank<_From, _To, false>
781 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
785template <
typename _Tp,
typename _Ap>
786 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
787 __data(
const simd<_Tp, _Ap>& __x);
789template <
typename _Tp,
typename _Ap>
790 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
791 __data(simd<_Tp, _Ap>& __x);
793template <
typename _Tp,
typename _Ap>
794 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
795 __data(
const simd_mask<_Tp, _Ap>& __x);
797template <
typename _Tp,
typename _Ap>
798 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
799 __data(simd_mask<_Tp, _Ap>& __x);
803template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
805 struct _SimdConverter;
807template <
typename _Tp,
typename _Ap>
808 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
810 template <
typename _Up>
811 _GLIBCXX_SIMD_INTRINSIC
const _Up&
812 operator()(
const _Up& __x)
818template <
typename _V>
819 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
820 __to_value_type_or_member_type(
const _V& __x) ->
decltype(__data(__x))
821 {
return __data(__x); }
823template <
typename _V>
824 _GLIBCXX_SIMD_INTRINSIC
constexpr const typename _V::value_type&
825 __to_value_type_or_member_type(
const typename _V::value_type& __x)
830template <
size_t _Size>
831 struct __bool_storage_member_type;
833template <
size_t _Size>
834 using __bool_storage_member_type_t =
835 typename __bool_storage_member_type<_Size>::type;
845template <
typename _Tp,
typename... _Abis>
850template <
typename _Tp,
int _Np>
851 struct __fixed_size_storage;
853template <
typename _Tp,
int _Np>
854 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
858template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
861template <
typename _Tp>
862 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
863template <
typename _Tp>
864 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
865template <
typename _Tp>
866 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
867template <
typename _Tp>
868 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
872template <
typename _Tp>
875template <
typename _Tp,
size_t _Np>
876 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
878template <
typename _Tp>
879 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
886 template <
typename _Tp,
typename _Fp>
888 _S_bit_iteration(_Tp __mask, _Fp&& __f)
890 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
891 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
892 if constexpr (is_convertible_v<_Tp,
decltype(__k)>)
895 __k = __mask.to_ullong();
898 __f(std::__countr_zero(__k));
908template <
typename _Tp =
void>
910 {
constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
913 struct __increment<void>
915 template <
typename _Tp>
917 operator()(_Tp __a)
const
921template <
typename _Tp =
void>
923 {
constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
926 struct __decrement<void>
928 template <
typename _Tp>
930 operator()(_Tp __a)
const
936template <
typename _From,
typename _To,
938 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
939 using _ValuePreserving = _From;
941template <
typename _From,
typename _To,
942 typename _DecayedFrom = __remove_cvref_t<_From>,
944 is_convertible<_From, _To>,
946 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
947 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
948 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
949 using _ValuePreservingOrInt = _From;
953template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
954 struct __intrinsic_type;
956template <
typename _Tp,
size_t _Size>
957 using __intrinsic_type_t =
958 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
960template <
typename _Tp>
961 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
962template <
typename _Tp>
963 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
964template <
typename _Tp>
965 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
966template <
typename _Tp>
967 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
968template <
typename _Tp>
969 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
970template <
typename _Tp>
971 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
975template <
size_t _Np,
bool _Sanitized = false>
978template <
size_t _Np,
bool _Sanitized>
979 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
982 using _SanitizedBitMask = _BitMask<_Np, true>;
984template <
size_t _Np,
bool _Sanitized>
987 static_assert(_Np > 0);
989 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
993 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
995 static constexpr int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
997 _Tp _M_bits[_S_array_size];
999 static constexpr int _S_unused_bits
1000 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
1002 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1004 constexpr _BitMask() noexcept = default;
1006 constexpr _BitMask(
unsigned long long __x) noexcept
1007 : _M_bits{
static_cast<_Tp
>(__x)} {}
1009 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1011 constexpr _BitMask(
const _BitMask&)
noexcept =
default;
1013 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
1014 && _Sanitized ==
true>>
1015 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1016 : _BitMask(__rhs._M_sanitized()) {}
1018 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1020 static_assert(_S_array_size == 1);
1026 _M_to_bits() const noexcept
1028 static_assert(_S_array_size == 1);
1033 constexpr unsigned long long
1034 to_ullong() const noexcept
1036 static_assert(_S_array_size == 1);
1041 constexpr unsigned long
1042 to_ulong() const noexcept
1044 static_assert(_S_array_size == 1);
1048 constexpr bitset<_Np>
1049 _M_to_bitset() const noexcept
1051 static_assert(_S_array_size == 1);
1055 constexpr decltype(
auto)
1056 _M_sanitized()
const noexcept
1058 if constexpr (_Sanitized)
1060 else if constexpr (_Np == 1)
1061 return _SanitizedBitMask<_Np>(_M_bits[0]);
1064 _SanitizedBitMask<_Np> __r = {};
1065 for (
int __i = 0; __i < _S_array_size; ++__i)
1066 __r._M_bits[__i] = _M_bits[__i];
1067 if constexpr (_S_unused_bits > 0)
1068 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1073 template <
size_t _Mp,
bool _LSanitized>
1074 constexpr _BitMask<_Np + _Mp, _Sanitized>
1075 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1077 constexpr size_t _RN = _Np + _Mp;
1078 using _Rp = _BitMask<_RN, _Sanitized>;
1079 if constexpr (_Rp::_S_array_size == 1)
1081 _Rp __r{{_M_bits[0]}};
1082 __r._M_bits[0] <<= _Mp;
1083 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1087 __assert_unreachable<_Rp>();
1093 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1095 _M_extract() const noexcept
1097 static_assert(_Np > _DropLsb);
1098 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1099 "not implemented for bitmasks larger than one ullong");
1100 if constexpr (_NewSize == 1)
1102 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1104 return _BitMask<_NewSize,
1105 ((_NewSize + _DropLsb ==
sizeof(_Tp) * __CHAR_BIT__
1106 && _NewSize + _DropLsb <= _Np)
1107 || ((_Sanitized || _Np ==
sizeof(_Tp) * __CHAR_BIT__)
1108 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1114 all() const noexcept
1116 if constexpr (_Np == 1)
1118 else if constexpr (!_Sanitized)
1119 return _M_sanitized().all();
1122 constexpr _Tp __allbits = ~_Tp();
1123 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1124 if (_M_bits[__i] != __allbits)
1126 return _M_bits[_S_array_size - 1] == _S_bitmask;
1133 any() const noexcept
1135 if constexpr (_Np == 1)
1137 else if constexpr (!_Sanitized)
1138 return _M_sanitized().any();
1141 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1142 if (_M_bits[__i] != 0)
1144 return _M_bits[_S_array_size - 1] != 0;
1150 none() const noexcept
1152 if constexpr (_Np == 1)
1154 else if constexpr (!_Sanitized)
1155 return _M_sanitized().none();
1158 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1159 if (_M_bits[__i] != 0)
1161 return _M_bits[_S_array_size - 1] == 0;
1168 count() const noexcept
1170 if constexpr (_Np == 1)
1172 else if constexpr (!_Sanitized)
1173 return _M_sanitized().none();
1176 int __result = __builtin_popcountll(_M_bits[0]);
1177 for (
int __i = 1; __i < _S_array_size; ++__i)
1178 __result += __builtin_popcountll(_M_bits[__i]);
1185 operator[](
size_t __i)
const noexcept
1187 if constexpr (_Np == 1)
1189 else if constexpr (_S_array_size == 1)
1190 return (_M_bits[0] >> __i) & 1;
1193 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1194 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1195 return (_M_bits[__j] >> __shift) & 1;
1199 template <
size_t __i>
1201 operator[](_SizeConstant<__i>)
const noexcept
1203 static_assert(__i < _Np);
1204 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1205 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1206 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1211 set(
size_t __i,
bool __x)
noexcept
1213 if constexpr (_Np == 1)
1215 else if constexpr (_S_array_size == 1)
1217 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1218 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1222 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1223 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1224 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1225 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1229 template <
size_t __i>
1231 set(_SizeConstant<__i>,
bool __x)
noexcept
1233 static_assert(__i < _Np);
1234 if constexpr (_Np == 1)
1238 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1239 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1240 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1241 _M_bits[__j] &= __mask;
1242 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1248 operator~() const noexcept
1250 if constexpr (_Np == 1)
1254 _BitMask __result{};
1255 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1256 __result._M_bits[__i] = ~_M_bits[__i];
1257 if constexpr (_Sanitized)
1258 __result._M_bits[_S_array_size - 1]
1259 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1261 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1267 operator^=(
const _BitMask& __b) &
noexcept
1269 __execute_n_times<_S_array_size>(
1270 [&](
auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1275 operator|=(
const _BitMask& __b) &
noexcept
1277 __execute_n_times<_S_array_size>(
1278 [&](
auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1283 operator&=(
const _BitMask& __b) &
noexcept
1285 __execute_n_times<_S_array_size>(
1286 [&](
auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1290 friend constexpr _BitMask
1291 operator^(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1298 friend constexpr _BitMask
1299 operator|(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1306 friend constexpr _BitMask
1307 operator&(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1314 _GLIBCXX_SIMD_INTRINSIC
1316 _M_is_constprop()
const
1318 if constexpr (_S_array_size == 0)
1319 return __builtin_constant_p(_M_bits[0]);
1322 for (
int __i = 0; __i < _S_array_size; ++__i)
1323 if (!__builtin_constant_p(_M_bits[__i]))
1334template <
typename _Tp =
void>
1335 static inline constexpr int __min_vector_size = 2 *
sizeof(_Tp);
1337#if _GLIBCXX_SIMD_HAVE_NEON
1339 inline constexpr int __min_vector_size<void> = 8;
1342 inline constexpr int __min_vector_size<void> = 16;
1347template <
typename _Tp,
size_t _Np,
typename =
void>
1348 struct __vector_type_n {};
1351template <
typename _Tp>
1352 struct __vector_type_n<_Tp, 0, void> {};
1355template <
typename _Tp>
1356 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1357 {
using type = _Tp; };
1360template <
typename _Tp,
size_t _Np>
1361 struct __vector_type_n<_Tp, _Np,
1362 enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1364 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1366 static constexpr size_t _S_Bytes =
1372 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1375 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1378template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1379 struct __vector_type;
1381template <
typename _Tp,
size_t _Bytes>
1382 struct __vector_type<_Tp, _Bytes, 0>
1383 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1385template <
typename _Tp,
size_t _Size>
1386 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1388template <
typename _Tp>
1389 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1390template <
typename _Tp>
1391 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1392template <
typename _Tp>
1393 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1394template <
typename _Tp>
1395 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1396template <
typename _Tp>
1397 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1398template <
typename _Tp>
1399 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1403template <
typename _Tp,
typename =
void_t<>>
1406template <
typename _Tp>
1407 struct __is_vector_type<
1408 _Tp,
void_t<typename __vector_type<
1410 : is_same<_Tp, typename __vector_type<
1411 remove_reference_t<decltype(declval<_Tp>()[0])>,
1412 sizeof(_Tp)>::type> {};
1414template <
typename _Tp>
1415 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1419#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1420template <
typename _Tp>
1421 using __is_intrinsic_type = __is_vector_type<_Tp>;
1423template <
typename _Tp,
typename =
void_t<>>
1426template <
typename _Tp>
1427 struct __is_intrinsic_type<
1428 _Tp,
void_t<typename __intrinsic_type<
1430 : is_same<_Tp, typename __intrinsic_type<
1431 remove_reference_t<decltype(declval<_Tp>()[0])>,
1432 sizeof(_Tp)>::type> {};
1435template <
typename _Tp>
1436 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1440template <
typename _Tp,
typename =
void_t<>>
1441 struct _VectorTraitsImpl;
1443template <
typename _Tp>
1444 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>
1445 || __is_intrinsic_type_v<_Tp>>>
1448 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1449 static constexpr int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1450 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1451 template <
typename _Up,
int _W = _S_full_size>
1452 static constexpr bool _S_is
1453 = is_same_v<value_type, _Up> && _W == _S_full_size;
1456template <
typename _Tp,
size_t _Np>
1457 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1458 void_t<__vector_type_t<_Tp, _Np>>>
1460 using type = __vector_type_t<_Tp, _Np>;
1461 using value_type = _Tp;
1462 static constexpr int _S_full_size =
sizeof(type) /
sizeof(value_type);
1463 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1464 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1465 static constexpr int _S_partial_width = _Np;
1466 template <
typename _Up,
int _W = _S_full_size>
1467 static constexpr bool _S_is
1468 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1471template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1472 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1476template <
typename _V>
1477 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1480 if constexpr (__is_vector_type_v<_V>)
1482 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1483 return __data(__x)._M_data;
1484 else if constexpr (__is_vectorizable_v<_V>)
1485 return __vector_type_t<_V, 2>{__x};
1492template <
size_t _Np = 0,
typename _V>
1493 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1494 __as_wrapper(_V __x)
1496 if constexpr (__is_vector_type_v<_V>)
1497 return _SimdWrapper<
typename _VectorTraits<_V>::value_type,
1498 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1499 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1506 static_assert(_V::_S_size == _Np);
1513template <
typename _To,
typename _From>
1514 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1515 __intrin_bitcast(_From __v)
1517 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1518 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1519 if constexpr (
sizeof(_To) ==
sizeof(_From))
1520 return reinterpret_cast<_To
>(__v);
1521 else if constexpr (
sizeof(_From) >
sizeof(_To))
1522 if constexpr (
sizeof(_To) >= 16)
1523 return reinterpret_cast<const __may_alias<_To>&
>(__v);
1527 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1530#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1531 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1532 return reinterpret_cast<_To
>(__builtin_ia32_ps256_ps(
1533 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1534 else if constexpr (__have_avx512f &&
sizeof(_From) == 16
1535 &&
sizeof(_To) == 64)
1536 return reinterpret_cast<_To
>(__builtin_ia32_ps512_ps(
1537 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1538 else if constexpr (__have_avx512f &&
sizeof(_From) == 32
1539 &&
sizeof(_To) == 64)
1540 return reinterpret_cast<_To
>(__builtin_ia32_ps512_256ps(
1541 reinterpret_cast<__vector_type_t<float, 8>
>(__v)));
1543 else if constexpr (
sizeof(__v) <= 8)
1544 return reinterpret_cast<_To
>(
1545 __vector_type_t<__int_for_sizeof_t<_From>,
sizeof(_To) /
sizeof(_From)>{
1546 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1549 static_assert(
sizeof(_To) >
sizeof(_From));
1551 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1558template <
typename _To,
size_t _NN = 0,
typename _From,
1559 typename _FromVT = _VectorTraits<_From>,
1560 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1561 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1562 __vector_bitcast(_From __x)
1564 using _R = __vector_type_t<_To, _Np>;
1565 return __intrin_bitcast<_R>(__x);
1568template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1570 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1571 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1572 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1574 static_assert(_Np > 1);
1575 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1580#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1581template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1582 _To __convert_x86(_Tp);
1584template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1585 _To __convert_x86(_Tp, _Tp);
1587template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1588 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1590template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1591 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1593template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1594 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1595 _Tp, _Tp, _Tp, _Tp);
1600template <
typename _To,
typename _From>
1601 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1602 __bit_cast(
const _From __x)
1605 static_assert(
sizeof(_To) ==
sizeof(_From));
1606 constexpr bool __to_is_vectorizable
1607 = is_arithmetic_v<_To> || is_enum_v<_To>;
1608 constexpr bool __from_is_vectorizable
1609 = is_arithmetic_v<_From> || is_enum_v<_From>;
1610 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1611 return reinterpret_cast<_To
>(__x);
1612 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1614 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1615 return reinterpret_cast<_To
>(_FV{__x});
1617 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1619 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1620 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1621 return reinterpret_cast<_TV
>(_FV{__x})[0];
1623 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1625 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1626 return reinterpret_cast<_TV
>(__x)[0];
1631 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1632 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1639template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1641 = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1642 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
1643 __to_intrin(_Tp __x)
1645 static_assert(
sizeof(__x) <=
sizeof(_R),
1646 "__to_intrin may never drop values off the end");
1647 if constexpr (
sizeof(__x) ==
sizeof(_R))
1648 return reinterpret_cast<_R
>(__as_vector(__x));
1651 using _Up = __int_for_sizeof_t<_Tp>;
1652 return reinterpret_cast<_R
>(
1653 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1659template <
typename _Tp,
typename... _Args>
1660 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1661 __make_vector(
const _Args&... __args)
1663 return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...};
1668template <
size_t _Np,
typename _Tp,
size_t... _I>
1669 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1670 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1671 {
return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1673template <
size_t _Np,
typename _Tp>
1674 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1675 __vector_broadcast(_Tp __x)
1676 {
return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1680 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1681 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1682 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1684 return __vector_type_t<_Tp, _Np>{
1685 static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...};
1688template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1689 _GLIBCXX_SIMD_INTRINSIC
constexpr _V
1690 __generate_vector(_Gp&& __gen)
1692 if constexpr (__is_vector_type_v<_V>)
1693 return __generate_vector_impl<
typename _VVT::value_type,
1694 _VVT::_S_full_size>(
1695 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1697 return __generate_vector_impl<
typename _VVT::value_type,
1698 _VVT::_S_partial_width>(
1699 static_cast<_Gp&&
>(__gen),
1700 make_index_sequence<_VVT::_S_partial_width>());
1703template <
typename _Tp,
size_t _Np,
typename _Gp>
1704 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1705 __generate_vector(_Gp&& __gen)
1707 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1708 make_index_sequence<_Np>());
1713template <
typename _TW>
1714 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1715 __xor(_TW __a, _TW __b)
noexcept
1717 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1719 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1720 _VectorTraitsImpl<_TW>>::value_type;
1721 if constexpr (is_floating_point_v<_Tp>)
1723 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1724 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1725 ^ __vector_bitcast<_Ip>(__b));
1727 else if constexpr (__is_vector_type_v<_TW>)
1730 return __a._M_data ^ __b._M_data;
1738template <
typename _TW>
1739 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1740 __or(_TW __a, _TW __b)
noexcept
1742 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1744 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1745 _VectorTraitsImpl<_TW>>::value_type;
1746 if constexpr (is_floating_point_v<_Tp>)
1748 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1749 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1750 | __vector_bitcast<_Ip>(__b));
1752 else if constexpr (__is_vector_type_v<_TW>)
1755 return __a._M_data | __b._M_data;
1763template <
typename _TW>
1764 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1765 __and(_TW __a, _TW __b)
noexcept
1767 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1769 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1770 _VectorTraitsImpl<_TW>>::value_type;
1771 if constexpr (is_floating_point_v<_Tp>)
1773 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1774 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1775 & __vector_bitcast<_Ip>(__b));
1777 else if constexpr (__is_vector_type_v<_TW>)
1780 return __a._M_data & __b._M_data;
1788#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1789static constexpr struct
1791 _GLIBCXX_SIMD_INTRINSIC __v4sf
1792 operator()(__v4sf __a, __v4sf __b)
const noexcept
1793 {
return __builtin_ia32_andnps(__a, __b); }
1795 _GLIBCXX_SIMD_INTRINSIC __v2df
1796 operator()(__v2df __a, __v2df __b)
const noexcept
1797 {
return __builtin_ia32_andnpd(__a, __b); }
1799 _GLIBCXX_SIMD_INTRINSIC __v2di
1800 operator()(__v2di __a, __v2di __b)
const noexcept
1801 {
return __builtin_ia32_pandn128(__a, __b); }
1803 _GLIBCXX_SIMD_INTRINSIC __v8sf
1804 operator()(__v8sf __a, __v8sf __b)
const noexcept
1805 {
return __builtin_ia32_andnps256(__a, __b); }
1807 _GLIBCXX_SIMD_INTRINSIC __v4df
1808 operator()(__v4df __a, __v4df __b)
const noexcept
1809 {
return __builtin_ia32_andnpd256(__a, __b); }
1811 _GLIBCXX_SIMD_INTRINSIC __v4di
1812 operator()(__v4di __a, __v4di __b)
const noexcept
1814 if constexpr (__have_avx2)
1815 return __builtin_ia32_andnotsi256(__a, __b);
1817 return reinterpret_cast<__v4di
>(
1818 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1819 reinterpret_cast<__v4df
>(__b)));
1822 _GLIBCXX_SIMD_INTRINSIC __v16sf
1823 operator()(__v16sf __a, __v16sf __b)
const noexcept
1825 if constexpr (__have_avx512dq)
1826 return _mm512_andnot_ps(__a, __b);
1828 return reinterpret_cast<__v16sf
>(
1829 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1830 reinterpret_cast<__v8di
>(__b)));
1833 _GLIBCXX_SIMD_INTRINSIC __v8df
1834 operator()(__v8df __a, __v8df __b)
const noexcept
1836 if constexpr (__have_avx512dq)
1837 return _mm512_andnot_pd(__a, __b);
1839 return reinterpret_cast<__v8df
>(
1840 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1841 reinterpret_cast<__v8di
>(__b)));
1844 _GLIBCXX_SIMD_INTRINSIC __v8di
1845 operator()(__v8di __a, __v8di __b)
const noexcept
1846 {
return _mm512_andnot_si512(__a, __b); }
1850template <
typename _TW>
1851 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1852 __andnot(_TW __a, _TW __b)
noexcept
1854 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1856 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1857 _VectorTraitsImpl<_TW>>;
1858 using _Tp =
typename _TVT::value_type;
1859#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1860 if constexpr (
sizeof(_TW) >= 16)
1862 const auto __ai = __to_intrin(__a);
1863 const auto __bi = __to_intrin(__b);
1864 if (!__builtin_is_constant_evaluated()
1865 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1867 const auto __r = _S_x86_andnot(__ai, __bi);
1868 if constexpr (is_convertible_v<
decltype(__r), _TW>)
1871 return reinterpret_cast<typename _TVT::type
>(__r);
1875 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1876 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1877 & __vector_bitcast<_Ip>(__b));
1885template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1886 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
1887 __not(_Tp __a)
noexcept
1889 if constexpr (is_floating_point_v<typename _TVT::value_type>)
1890 return reinterpret_cast<typename _TVT::type
>(
1891 ~__vector_bitcast<unsigned>(__a));
1898template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1899 typename _R = __vector_type_t<
typename _TVT::value_type,
1900 _TVT::_S_full_size * 2>>
1902 __concat(_Tp a_, _Tp b_)
1904#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1906 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1908 long long,
typename _TVT::value_type>>;
1909 constexpr int input_width =
sizeof(_Tp) /
sizeof(_W);
1910 const auto __a = __vector_bitcast<_W>(a_);
1911 const auto __b = __vector_bitcast<_W>(b_);
1912 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
1914 constexpr int input_width = _TVT::_S_full_size;
1915 const _Tp& __a = a_;
1916 const _Tp& __b = b_;
1919 if constexpr (input_width == 2)
1920 return reinterpret_cast<_R
>(_Up{__a[0], __a[1], __b[0], __b[1]});
1921 else if constexpr (input_width == 4)
1922 return reinterpret_cast<_R
>(
1923 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1924 else if constexpr (input_width == 8)
1925 return reinterpret_cast<_R
>(
1926 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1927 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1928 else if constexpr (input_width == 16)
1929 return reinterpret_cast<_R
>(
1930 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1931 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1932 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1933 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1934 __b[12], __b[13], __b[14], __b[15]});
1935 else if constexpr (input_width == 32)
1936 return reinterpret_cast<_R
>(
1937 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1938 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1939 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1940 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1941 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1942 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1943 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1944 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1945 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1951template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1952 struct _ZeroExtendProxy
1954 using value_type =
typename _TVT::value_type;
1955 static constexpr size_t _Np = _TVT::_S_full_size;
1958 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
1960 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
1961 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
1963 constexpr size_t _ToN = _ToVT::_S_full_size;
1964 if constexpr (_ToN == _Np)
1966 else if constexpr (_ToN == 2 * _Np)
1968#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1969 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1970 return __vector_bitcast<value_type>(
1971 _mm256_insertf128_ps(__m256(), __x, 0));
1972 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1973 return __vector_bitcast<value_type>(
1974 _mm256_insertf128_pd(__m256d(), __x, 0));
1975 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
1976 return __vector_bitcast<value_type>(
1977 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1978 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1980 if constexpr (__have_avx512dq)
1981 return __vector_bitcast<value_type>(
1982 _mm512_insertf32x8(__m512(), __x, 0));
1984 return reinterpret_cast<__m512
>(
1985 _mm512_insertf64x4(__m512d(),
1986 reinterpret_cast<__m256d
>(__x), 0));
1988 else if constexpr (__have_avx512f
1989 && _TVT::template _S_is<double, 4>)
1990 return __vector_bitcast<value_type>(
1991 _mm512_insertf64x4(__m512d(), __x, 0));
1992 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
1993 return __vector_bitcast<value_type>(
1994 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1996 return __concat(__x, _Tp());
1998 else if constexpr (_ToN == 4 * _Np)
2000#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2001 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2003 return __vector_bitcast<value_type>(
2004 _mm512_insertf64x2(__m512d(), __x, 0));
2006 else if constexpr (__have_avx512f
2007 && is_floating_point_v<value_type>)
2009 return __vector_bitcast<value_type>(
2010 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
2013 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
2015 return __vector_bitcast<value_type>(
2016 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2019 return __concat(__concat(__x, _Tp()),
2020 __vector_type_t<value_type, _Np * 2>());
2022 else if constexpr (_ToN == 8 * _Np)
2023 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
2024 __vector_type_t<value_type, _Np * 4>());
2025 else if constexpr (_ToN == 16 * _Np)
2026 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
2027 __vector_type_t<value_type, _Np * 8>());
2029 __assert_unreachable<_Tp>();
2033template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2034 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2035 __zero_extend(_Tp __x)
2040template <
int _Offset,
2043 typename _TVT = _VectorTraits<_Tp>,
2044 typename _R = __vector_type_t<
typename _TVT::value_type,
2045 _TVT::_S_full_size / _SplitBy>>
2046 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2049 using value_type =
typename _TVT::value_type;
2050#if _GLIBCXX_SIMD_X86INTRIN
2051 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2053 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2054 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2055 else if constexpr (is_floating_point_v<value_type>)
2056 return __vector_bitcast<value_type>(
2057 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2059 return reinterpret_cast<_R
>(
2060 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2066#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2068 is_floating_point_v<value_type>, double,
2070 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2071 constexpr int __return_width =
sizeof(_R) /
sizeof(_W);
2072 using _Up = __vector_type_t<_W, __return_width>;
2073 const auto __x = __vector_bitcast<_W>(__in);
2075 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2077 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2080 constexpr int _O = _Offset * __return_width;
2081 return __call_with_subscripts<__return_width, _O>(
2082 __x, [](
auto... __entries) {
2083 return reinterpret_cast<_R
>(_Up{__entries...});
2090template <
typename _Tp,
2092 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2093 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2097 __builtin_memcpy(&__r, &__x, 8);
2101template <
typename _Tp,
2103 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2104 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2107 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2109 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2113template <
typename _Tp,
2115 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2116 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2117 __hi64z([[maybe_unused]] _Tp __x)
2120 if constexpr (
sizeof(_Tp) == 16)
2121 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2127template <
typename _Tp>
2128 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2130 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2132template <
typename _Tp>
2133 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2136 static_assert(
sizeof(__x) == 32);
2137 return __extract<1, 2>(__x);
2142template <
typename _Tp>
2143 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2146 static_assert(
sizeof(__x) == 64);
2147 return __extract<0, 2>(__x);
2150template <
typename _Tp>
2151 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2154 static_assert(
sizeof(__x) == 64);
2155 return __extract<1, 2>(__x);
2160template <
typename _Tp>
2163 static_assert(__is_vector_type_v<_Tp>);
2167 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2168 _GLIBCXX_SIMD_INTRINSIC
constexpr operator _Up()
const
2169 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2172template <
typename _Tp>
2173 _GLIBCXX_SIMD_INTRINSIC
constexpr _AutoCast<_Tp>
2174 __auto_bitcast(
const _Tp& __x)
2177template <
typename _Tp,
size_t _Np>
2178 _GLIBCXX_SIMD_INTRINSIC
constexpr
2179 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2180 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2181 {
return {__x._M_data}; }
2186#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2188#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2189template <
size_t _Size>
2190 struct __bool_storage_member_type
2192 static_assert((_Size & (_Size - 1)) != 0,
2193 "This trait may only be used for non-power-of-2 sizes. "
2194 "Power-of-2 sizes must be specialized.");
2196 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2200 struct __bool_storage_member_type<1> {
using type = bool; };
2203 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2206 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2209 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2212 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2215 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2218 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2224#if _GLIBCXX_SIMD_HAVE_SSE
2225template <
typename _Tp,
size_t _Bytes>
2226 struct __intrinsic_type<_Tp, _Bytes,
2227 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2229 static_assert(!is_same_v<_Tp, long double>,
2230 "no __intrinsic_type support for long double on x86");
2232 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16
2236 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2237 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2244#if _GLIBCXX_SIMD_HAVE_NEON
2246 struct __intrinsic_type<float, 8, void>
2247 {
using type = float32x2_t; };
2250 struct __intrinsic_type<float, 16, void>
2251 {
using type = float32x4_t; };
2253#if _GLIBCXX_SIMD_HAVE_NEON_A64
2255 struct __intrinsic_type<double, 8, void>
2256 {
using type = float64x1_t; };
2259 struct __intrinsic_type<double, 16, void>
2260 {
using type = float64x2_t; };
2263#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2265 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2266 _Np * _Bits / 8, void> \
2267 { using type = int##_Bits##x##_Np##_t; }; \
2269 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2270 _Np * _Bits / 8, void> \
2271 { using type = uint##_Bits##x##_Np##_t; }
2272_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2273_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2274_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2275_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2276_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2277_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2278_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2279_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2280#undef _GLIBCXX_SIMD_ARM_INTRIN
2282template <
typename _Tp,
size_t _Bytes>
2283 struct __intrinsic_type<_Tp, _Bytes,
2284 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2286 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2287 using _Ip = __int_for_sizeof_t<_Tp>;
2289 is_floating_point_v<_Tp>, _Tp,
2290 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2291 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2292 "should use explicit specialization above");
2293 using type =
typename __intrinsic_type<_Up, _SVecBytes>::type;
2300template <
typename _Tp>
2301 struct __intrinsic_type_impl;
2303#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2305 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2306_GLIBCXX_SIMD_PPC_INTRIN(
float);
2308_GLIBCXX_SIMD_PPC_INTRIN(
double);
2310_GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2311_GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2312_GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2313_GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2314_GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2315_GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2316#if defined __VSX__ || __SIZEOF_LONG__ == 4
2317_GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2318_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2321_GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2322_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2324#undef _GLIBCXX_SIMD_PPC_INTRIN
2326template <
typename _Tp,
size_t _Bytes>
2327 struct __intrinsic_type<_Tp, _Bytes,
2328 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2330 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2332 static_assert(!(_S_is_ldouble &&
sizeof(
long double) >
sizeof(
double)),
2333 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2335 static_assert(!(is_same_v<_Tp, double>
2336 || (_S_is_ldouble &&
sizeof(
long double) ==
sizeof(
double))),
2337 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2340 typename __intrinsic_type_impl<
2341 conditional_t<is_floating_point_v<_Tp>,
2342 conditional_t<_S_is_ldouble, double, _Tp>,
2343 __int_for_sizeof_t<_Tp>>>::type;
2349template <
size_t _W
idth>
2350 struct _SimdWrapper<bool, _Width,
2351 void_t<typename __bool_storage_member_type<_Width>::type>>
2353 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2354 using value_type = bool;
2356 static constexpr size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2358 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<bool, _S_full_size>
2359 __as_full_vector()
const {
return _M_data; }
2361 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper() =
default;
2362 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper(_BuiltinType __k)
2365 _GLIBCXX_SIMD_INTRINSIC
operator const _BuiltinType&()
const
2368 _GLIBCXX_SIMD_INTRINSIC
operator _BuiltinType&()
2371 _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin()
const
2374 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type operator[](
size_t __i)
const
2375 {
return _M_data & (_BuiltinType(1) << __i); }
2377 template <
size_t __i>
2378 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2379 operator[](_SizeConstant<__i>)
const
2380 {
return _M_data & (_BuiltinType(1) << __i); }
2382 _GLIBCXX_SIMD_INTRINSIC
constexpr void _M_set(
size_t __i, value_type __x)
2385 _M_data |= (_BuiltinType(1) << __i);
2387 _M_data &= ~(_BuiltinType(1) << __i);
2390 _GLIBCXX_SIMD_INTRINSIC
2391 constexpr bool _M_is_constprop()
const
2392 {
return __builtin_constant_p(_M_data); }
2394 _GLIBCXX_SIMD_INTRINSIC
constexpr bool _M_is_constprop_none_of()
const
2396 if (__builtin_constant_p(_M_data))
2398 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2399 constexpr _BuiltinType __active_mask
2400 = ~_BuiltinType() >> (__nbits - _Width);
2401 return (_M_data & __active_mask) == 0;
2406 _GLIBCXX_SIMD_INTRINSIC
constexpr bool _M_is_constprop_all_of()
const
2408 if (__builtin_constant_p(_M_data))
2410 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2411 constexpr _BuiltinType __active_mask
2412 = ~_BuiltinType() >> (__nbits - _Width);
2413 return (_M_data & __active_mask) == __active_mask;
2418 _BuiltinType _M_data;
2422template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2423 struct _SimdWrapperBase;
2425template <
typename _BuiltinType>
2426 struct _SimdWrapperBase<false, _BuiltinType>
2428 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapperBase() =
default;
2429 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapperBase(_BuiltinType __init)
2433 _BuiltinType _M_data;
2436template <
typename _BuiltinType>
2437 struct _SimdWrapperBase<true, _BuiltinType>
2440 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapperBase() : _M_data() {}
2441 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapperBase(_BuiltinType __init)
2445 _BuiltinType _M_data;
2450template <
typename _Tp,
size_t _W
idth>
2451 struct _SimdWrapper<
2453 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2454 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2455 && sizeof(_Tp) * _Width
2456 == sizeof(__vector_type_t<_Tp, _Width>),
2457 __vector_type_t<_Tp, _Width>>
2460 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2461 &&
sizeof(_Tp) * _Width
2462 ==
sizeof(__vector_type_t<_Tp, _Width>),
2463 __vector_type_t<_Tp, _Width>>;
2465 static_assert(__is_vectorizable_v<_Tp>);
2466 static_assert(_Width >= 2);
2468 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2469 using value_type = _Tp;
2471 static inline constexpr size_t _S_full_size
2472 =
sizeof(_BuiltinType) /
sizeof(value_type);
2473 static inline constexpr int _S_size = _Width;
2474 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2476 using _Base::_M_data;
2478 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<_Tp, _S_full_size>
2479 __as_full_vector()
const
2482 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper(initializer_list<_Tp> __init)
2483 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2484 [&](auto __i) {
return __init.begin()[__i.value]; })) {}
2486 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper() =
default;
2487 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper(
const _SimdWrapper&)
2489 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper(_SimdWrapper&&) =
default;
2491 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2492 operator=(
const _SimdWrapper&) =
default;
2493 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2494 operator=(_SimdWrapper&&) =
default;
2496 template <
typename _V,
typename =
enable_if_t<disjunction_v<
2497 is_same<_V, __vector_type_t<_Tp, _Width>>,
2498 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2499 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper(_V __x)
2501 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2503 template <
typename... _As,
2504 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2505 &&
sizeof...(_As) <= _Width)>>
2506 _GLIBCXX_SIMD_INTRINSIC
constexpr
2507 operator _SimdTuple<_Tp, _As...>()
const
2509 const auto& dd = _M_data;
2510 return __generate_from_n_evaluations<
sizeof...(_As),
2511 _SimdTuple<_Tp, _As...>>([&](
2512 auto __i)
constexpr {
return dd[int(__i)]; });
2515 _GLIBCXX_SIMD_INTRINSIC
constexpr operator const _BuiltinType&()
const
2518 _GLIBCXX_SIMD_INTRINSIC
constexpr operator _BuiltinType&()
2521 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp operator[](
size_t __i)
const
2522 {
return _M_data[__i]; }
2524 template <
size_t __i>
2525 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp operator[](_SizeConstant<__i>)
const
2526 {
return _M_data[__i]; }
2528 _GLIBCXX_SIMD_INTRINSIC
constexpr void _M_set(
size_t __i, _Tp __x)
2529 { _M_data[__i] = __x; }
2531 _GLIBCXX_SIMD_INTRINSIC
2532 constexpr bool _M_is_constprop()
const
2533 {
return __builtin_constant_p(_M_data); }
2535 _GLIBCXX_SIMD_INTRINSIC
constexpr bool _M_is_constprop_none_of()
const
2537 if (__builtin_constant_p(_M_data))
2540 if constexpr (is_floating_point_v<_Tp>)
2542 using _Ip = __int_for_sizeof_t<_Tp>;
2543 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2544 __execute_n_times<_Width>(
2545 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2548 __execute_n_times<_Width>(
2549 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2550 if (__builtin_constant_p(__r))
2556 _GLIBCXX_SIMD_INTRINSIC
constexpr bool _M_is_constprop_all_of()
const
2558 if (__builtin_constant_p(_M_data))
2561 if constexpr (is_floating_point_v<_Tp>)
2563 using _Ip = __int_for_sizeof_t<_Tp>;
2564 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2565 __execute_n_times<_Width>(
2566 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2569 __execute_n_times<_Width>(
2570 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2571 if (__builtin_constant_p(__r))
2581template <
typename _Tp>
2583 __vectorized_sizeof()
2585 if constexpr (!__is_vectorizable_v<_Tp>)
2588 if constexpr (
sizeof(_Tp) <= 8)
2591 if constexpr (__have_avx512bw)
2593 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2595 if constexpr (__have_avx2)
2597 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2599 if constexpr (__have_sse2)
2601 if constexpr (__have_sse && is_same_v<_Tp, float>)
2611 if constexpr (__have_power8vec
2612 || (__have_power_vmx && (
sizeof(_Tp) < 8))
2613 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2617 if constexpr (__have_neon_a64
2618 || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2620 if constexpr (__have_neon
2625 && (__support_neon_float || !is_floating_point_v<_Tp>))
2635template <
typename _Tp>
2636 inline constexpr int max_fixed_size
2637 = (__have_avx512bw &&
sizeof(_Tp) == 1) ? 64 : 32;
2640#if defined __x86_64__ || defined __aarch64__
2641template <
typename _Tp>
2642 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2643#elif defined __ARM_NEON
2646template <
typename _Tp>
2649 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2650 _VecBuiltin<16>, scalar>;
2653 using compatible = scalar;
2658template <
typename _Tp>
2660 __determine_native_abi()
2662 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2663 if constexpr (__bytes ==
sizeof(_Tp))
2664 return static_cast<scalar*
>(
nullptr);
2665 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2666 return static_cast<_VecBltnBtmsk<__bytes>*
>(
nullptr);
2668 return static_cast<_VecBuiltin<__bytes>*
>(
nullptr);
2671template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2672 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2676#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2677template <
typename _Tp>
2678 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2680template <
typename _Tp>
2681 using __default_abi = compatible<_Tp>;
2689template <
typename _Tp,
typename =
void_t<>>
2692template <
typename _Tp>
2693 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
2694 :
public _Tp::_IsValidAbiTag {};
2696template <
typename _Tp>
2697 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2700template <
typename _Tp>
2703template <
typename _Tp>
2704 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2706template <
typename _Tp>
2709template <
typename _Tp>
2710inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2713template <
typename _Tp,
typename _Abi,
typename =
void>
2714 struct __simd_size_impl {};
2716template <
typename _Tp,
typename _Abi>
2717 struct __simd_size_impl<
2719 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2720 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2722template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2723 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2725template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2726 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2729template <
typename _Tp,
size_t _Np,
typename =
void>
2730 struct __deduce_impl;
2741template <
typename _Tp,
size_t _Np,
typename...>
2744template <
typename _Tp,
size_t _Np,
typename... _Abis>
2745 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
2750template <
typename _Tp,
typename _V,
typename =
void>
2753template <
typename _Tp,
typename _Up,
typename _Abi>
2755 _Tp, simd<_Up, _Abi>,
2756 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2759 = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2762template <
typename _Tp,
typename _Up,
typename _Abi>
2764 _Tp, simd_mask<_Up, _Abi>,
2765 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2768 = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2771template <
typename _Tp,
typename _V>
2772 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
2775template <
int _Np,
typename _V,
typename =
void>
2778template <
int _Np,
typename _Tp,
typename _Abi>
2779 struct resize_simd<_Np, simd<_Tp, _Abi>,
2780 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2781 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2783template <
int _Np,
typename _Tp,
typename _Abi>
2784 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2785 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2786 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2788template <
int _Np,
typename _V>
2789 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
2793template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2794 struct memory_alignment
2795 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2797template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2798 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2801template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2804template <
typename _Tp,
typename _Abi>
2805 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
2807template <
typename _Tp>
2808 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2810template <
typename _Tp,
int _Np>
2811 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2813template <
typename _Tp,
size_t _Np>
2814 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2817template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2820template <
typename _Tp,
typename _Abi>
2821 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
2823template <
typename _Tp>
2824 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2826template <
typename _Tp,
int _Np>
2827 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2829template <
typename _Tp,
size_t _Np>
2830 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2834template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
2836 struct __static_simd_cast_return_type;
2838template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
2839 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2841 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2843template <
typename _Tp,
typename _Up,
typename _Ap>
2844 struct __static_simd_cast_return_type<
2845 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::
size() == simd_size_v<_Up, _Ap>>>
2846 {
using type = _Tp; };
2848template <
typename _Tp,
typename _Ap>
2849 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2850#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2856 {
using type = simd<_Tp, _Ap>; };
2858template <
typename _Tp,
typename =
void>
2859 struct __safe_make_signed {
using type = _Tp;};
2861template <
typename _Tp>
2862 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
2865 using type = make_signed_t<make_unsigned_t<_Tp>>;
2868template <
typename _Tp>
2869 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
2871template <
typename _Tp,
typename _Up,
typename _Ap>
2872 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2873#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2881 (is_integral_v<_Up> && is_integral_v<_Tp> &&
2882#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2883 is_signed_v<_Up> != is_signed_v<_Tp> &&
2885 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2886 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2889template <
typename _Tp,
typename _Up,
typename _Ap,
2891 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2892 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2893 static_simd_cast(
const simd<_Up, _Ap>& __x)
2895 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2899 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2901 return _R(__private_init, __c(__data(__x)));
2905namespace __proposed {
2906template <
typename _Tp,
typename _Up,
typename _Ap,
2908 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2909 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
2910 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
2912 using _RM =
typename _R::mask_type;
2913 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2914 typename _RM::simd_type::value_type>(__x)};
2919template <
typename _Tp,
typename _Up,
typename _Ap,
2920 typename _To = __value_type_or_identity_t<_Tp>>
2921 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
2922 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
2923 ->
decltype(static_simd_cast<_Tp>(__x))
2924 {
return static_simd_cast<_Tp>(__x); }
2926namespace __proposed {
2927template <
typename _Tp,
typename _Up,
typename _Ap,
2928 typename _To = __value_type_or_identity_t<_Tp>>
2929 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
2930 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
2931 ->
decltype(static_simd_cast<_Tp>(__x))
2932 {
return static_simd_cast<_Tp>(__x); }
2937namespace __proposed {
2964template <
typename _Tp,
typename _Up,
typename _Ap>
2965 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
2966 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
2967 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
2969 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
2971 else if constexpr (simd_size_v<_Up, _Ap> == 1)
2979 else if constexpr (
sizeof(_Tp) ==
sizeof(__x)
2980 && !__is_fixed_size_abi_v<_Ap>)
2981 return {__private_init,
2982 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
2983 _Ap::_S_masked(__data(__x))._M_data)};
2987 __builtin_memcpy(&__data(__r), &__data(__x),
2994template <
typename _Tp,
typename _Up,
typename _Ap>
2995 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2996 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
2997 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
2999 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3000 typename _Tp::simd_type::value_type>(__x)};
3006template <
typename _Tp,
int _Np>
3007 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3008 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
3011template <
typename _Tp,
int _Np>
3012 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3013 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3016template <
typename _Tp,
typename _Ap>
3017 _GLIBCXX_SIMD_INTRINSIC
auto
3018 to_fixed_size(
const simd<_Tp, _Ap>& __x)
3020 return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3021 auto __i)
constexpr {
return __x[__i]; });
3024template <
typename _Tp,
typename _Ap>
3025 _GLIBCXX_SIMD_INTRINSIC
auto
3026 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
3029 fixed_size_simd_mask<_Tp, _Np> __r;
3030 __execute_n_times<_Np>([&](
auto __i)
constexpr { __r[__i] = __x[__i]; });
3035template <
typename _Tp,
int _Np>
3036 _GLIBCXX_SIMD_INTRINSIC
3038 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
3040 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3041 __x.copy_to(__mem, vector_aligned);
3042 return {__mem, vector_aligned};
3045template <
typename _Tp,
size_t _Np>
3046 _GLIBCXX_SIMD_INTRINSIC
3048 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3050 return native_simd_mask<_Tp>([&](
auto __i)
constexpr {
return __x[__i]; });
3054template <
typename _Tp,
size_t _Np>
3056 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3058 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3059 __x.copy_to(__mem, vector_aligned);
3060 return {__mem, vector_aligned};
3063template <
typename _Tp,
size_t _Np>
3064 _GLIBCXX_SIMD_INTRINSIC
3066 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3067 {
return simd_mask<_Tp>([&](
auto __i)
constexpr {
return __x[__i]; }); }
3073template <
typename _M,
typename _Tp>
3074 class const_where_expression
3077 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3079 struct _Wrapper {
using value_type = _V; };
3082 using _Impl =
typename _V::_Impl;
3085 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3087 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3088 __get_mask(
const const_where_expression& __x)
3089 {
return __x._M_k; }
3091 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3092 __get_lvalue(
const const_where_expression& __x)
3093 {
return __x._M_value; }
3099 const_where_expression(
const const_where_expression&) =
delete;
3100 const_where_expression& operator=(
const const_where_expression&) =
delete;
3102 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const _M& __kk,
const _Tp& dd)
3103 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3105 _GLIBCXX_SIMD_INTRINSIC _V
3108 return {__private_init,
3109 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3113 template <
typename _Up,
typename _Flags>
3114 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3115 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3117 return {__private_init,
3118 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3119 _Flags::template _S_apply<_V>(__mem))};
3122 template <
typename _Up,
typename _Flags>
3123 _GLIBCXX_SIMD_INTRINSIC
void
3124 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3126 _Impl::_S_masked_store(__data(_M_value),
3127 _Flags::template _S_apply<_V>(__mem),
3133template <
typename _Tp>
3134 class const_where_expression<bool, _Tp>
3139 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3141 struct _Wrapper {
using value_type = _V; };
3145 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3147 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3148 __get_mask(
const const_where_expression& __x)
3149 {
return __x._M_k; }
3151 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3152 __get_lvalue(
const const_where_expression& __x)
3153 {
return __x._M_value; }
3159 const_where_expression(
const const_where_expression&) =
delete;
3160 const_where_expression& operator=(
const const_where_expression&) =
delete;
3162 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const bool __kk,
const _Tp& dd)
3163 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3165 _GLIBCXX_SIMD_INTRINSIC _V
operator-() const&&
3166 {
return _M_k ? -_M_value : _M_value; }
3168 template <
typename _Up,
typename _Flags>
3169 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3170 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3171 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3173 template <
typename _Up,
typename _Flags>
3174 _GLIBCXX_SIMD_INTRINSIC
void
3175 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3178 __mem[0] = _M_value;
3183template <
typename _M,
typename _Tp>
3184 class where_expression :
public const_where_expression<_M, _Tp>
3186 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3188 static_assert(!is_const<_Tp>::value,
3189 "where_expression may only be instantiated with __a non-const "
3192 using typename const_where_expression<_M, _Tp>::value_type;
3193 using const_where_expression<_M, _Tp>::_M_k;
3194 using const_where_expression<_M, _Tp>::_M_value;
3197 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3200 _GLIBCXX_SIMD_INTRINSIC
friend _Tp& __get_lvalue(where_expression& __x)
3201 {
return __x._M_value; }
3204 where_expression(
const where_expression&) =
delete;
3205 where_expression& operator=(
const where_expression&) =
delete;
3207 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3208 : const_where_expression<_M, _Tp>(__kk, dd) {}
3210 template <
typename _Up>
3211 _GLIBCXX_SIMD_INTRINSIC
void operator=(_Up&& __x) &&
3213 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3214 __to_value_type_or_member_type<_Tp>(
3215 static_cast<_Up&&
>(__x)));
3218#define _GLIBCXX_SIMD_OP_(__op, __name) \
3219 template <typename _Up> \
3220 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3222 _Impl::template _S_masked_cassign( \
3223 __data(_M_k), __data(_M_value), \
3224 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3225 [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3226 return __impl.__name(__lhs, __rhs); \
3230 _GLIBCXX_SIMD_OP_(+, _S_plus);
3231 _GLIBCXX_SIMD_OP_(-, _S_minus);
3232 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3233 _GLIBCXX_SIMD_OP_(/, _S_divides);
3234 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3235 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3236 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3237 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3238 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3239 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3240#undef _GLIBCXX_SIMD_OP_
3242 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3245 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3249 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3252 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3256 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3259 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3263 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3266 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3271 template <
typename _Up,
typename _Flags>
3272 _GLIBCXX_SIMD_INTRINSIC
void
3273 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3276 = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3277 _Flags::template _S_apply<_Tp>(__mem));
3282template <
typename _Tp>
3283 class where_expression<bool, _Tp> :
public const_where_expression<bool, _Tp>
3286 using typename const_where_expression<_M, _Tp>::value_type;
3287 using const_where_expression<_M, _Tp>::_M_k;
3288 using const_where_expression<_M, _Tp>::_M_value;
3291 where_expression(
const where_expression&) =
delete;
3292 where_expression& operator=(
const where_expression&) =
delete;
3294 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3295 : const_where_expression<_M, _Tp>(__kk, dd) {}
3297#define _GLIBCXX_SIMD_OP_(__op) \
3298 template <typename _Up> \
3299 _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3300 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3302 _GLIBCXX_SIMD_OP_(=)
3303 _GLIBCXX_SIMD_OP_(+=)
3304 _GLIBCXX_SIMD_OP_(-=)
3305 _GLIBCXX_SIMD_OP_(*=)
3306 _GLIBCXX_SIMD_OP_(/=)
3307 _GLIBCXX_SIMD_OP_(%=)
3308 _GLIBCXX_SIMD_OP_(&=)
3309 _GLIBCXX_SIMD_OP_(|=)
3310 _GLIBCXX_SIMD_OP_(^=)
3311 _GLIBCXX_SIMD_OP_(<<=)
3312 _GLIBCXX_SIMD_OP_(>>=)
3313 #undef _GLIBCXX_SIMD_OP_
3315 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3316 {
if (_M_k) ++_M_value; }
3318 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3319 {
if (_M_k) ++_M_value; }
3321 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3322 {
if (_M_k) --_M_value; }
3324 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3325 {
if (_M_k) --_M_value; }
3328 template <
typename _Up,
typename _Flags>
3329 _GLIBCXX_SIMD_INTRINSIC
void
3330 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3331 {
if (_M_k) _M_value = __mem[0]; }
3335template <
typename _Tp,
typename _Ap>
3336 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3337 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3338 {
return {__k, __value}; }
3340template <
typename _Tp,
typename _Ap>
3341 _GLIBCXX_SIMD_INTRINSIC
3342 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3343 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
3344 const simd<_Tp, _Ap>& __value)
3345 {
return {__k, __value}; }
3347template <
typename _Tp,
typename _Ap>
3348 _GLIBCXX_SIMD_INTRINSIC
3349 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3351 simd_mask<_Tp, _Ap>& __value)
3352 {
return {__k, __value}; }
3354template <
typename _Tp,
typename _Ap>
3355 _GLIBCXX_SIMD_INTRINSIC
3356 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3358 const simd_mask<_Tp, _Ap>& __value)
3359 {
return {__k, __value}; }
3361template <
typename _Tp>
3362 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3363 where(_ExactBool __k, _Tp& __value)
3364 {
return {__k, __value}; }
3366template <
typename _Tp>
3367 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3368 where(_ExactBool __k,
const _Tp& __value)
3369 {
return {__k, __value}; }
3371 template <
typename _Tp,
typename _Ap>
3372 void where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3374 template <
typename _Tp,
typename _Ap>
3375 void where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3378namespace __proposed {
3379template <
size_t _Np>
3382 const bitset<_Np> __bits;
3385 where_range(bitset<_Np> __b) : __bits(__b) {}
3392 _GLIBCXX_SIMD_INTRINSIC
void __next_bit()
3393 { __bit = __builtin_ctzl(__mask); }
3395 _GLIBCXX_SIMD_INTRINSIC
void __reset_lsb()
3398 __mask &= (__mask - 1);
3403 iterator(
decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3404 iterator(
const iterator&) =
default;
3405 iterator(iterator&&) =
default;
3407 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator->()
const
3410 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator*()
const
3413 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3420 _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(
int)
3422 iterator __tmp = *
this;
3428 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator==(
const iterator& __rhs)
const
3429 {
return __mask == __rhs.__mask; }
3431 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator!=(
const iterator& __rhs)
const
3432 {
return __mask != __rhs.__mask; }
3435 iterator
begin()
const
3436 {
return __bits.to_ullong(); }
3438 iterator
end()
const
3442template <
typename _Tp,
typename _Ap>
3443 where_range<simd_size_v<_Tp, _Ap>>
3444 where(
const simd_mask<_Tp, _Ap>& __k)
3445 {
return __k.__to_bitset(); }
3451template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3452 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3453 reduce(
const simd<_Tp, _Abi>& __v,
3454 _BinaryOperation __binary_op = _BinaryOperation())
3455 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3457template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3458 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3459 reduce(
const const_where_expression<_M, _V>& __x,
3460 typename _V::value_type __identity_element,
3461 _BinaryOperation __binary_op)
3463 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3464 return __identity_element;
3466 _V __tmp = __identity_element;
3467 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3468 __data(__get_lvalue(__x)));
3469 return reduce(__tmp, __binary_op);
3472template <
typename _M,
typename _V>
3473 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3474 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3475 {
return reduce(__x, 0, __binary_op); }
3477template <
typename _M,
typename _V>
3478 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3479 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3480 {
return reduce(__x, 1, __binary_op); }
3482template <
typename _M,
typename _V>
3483 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3484 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3485 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3487template <
typename _M,
typename _V>
3488 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3489 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3490 {
return reduce(__x, 0, __binary_op); }
3492template <
typename _M,
typename _V>
3493 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3494 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3495 {
return reduce(__x, 0, __binary_op); }
3497template <
typename _Tp,
typename _Abi>
3498 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3499 hmin(
const simd<_Tp, _Abi>& __v)
noexcept
3501 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3504template <
typename _Tp,
typename _Abi>
3505 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3506 hmax(
const simd<_Tp, _Abi>& __v)
noexcept
3508 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3511template <
typename _M,
typename _V>
3512 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3513 typename _V::value_type
3514 hmin(
const const_where_expression<_M, _V>& __x)
noexcept
3516 using _Tp =
typename _V::value_type;
3517 constexpr _Tp __id_elem =
3518#ifdef __FINITE_MATH_ONLY__
3519 __finite_max_v<_Tp>;
3521 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3523 _V __tmp = __id_elem;
3524 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3525 __data(__get_lvalue(__x)));
3526 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3529template <
typename _M,
typename _V>
3530 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3531 typename _V::value_type
3532 hmax(
const const_where_expression<_M, _V>& __x)
noexcept
3534 using _Tp =
typename _V::value_type;
3535 constexpr _Tp __id_elem =
3536#ifdef __FINITE_MATH_ONLY__
3537 __finite_min_v<_Tp>;
3540 if constexpr (__value_exists_v<__infinity, _Tp>)
3541 return -__infinity_v<_Tp>;
3543 return __finite_min_v<_Tp>;
3546 _V __tmp = __id_elem;
3547 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3548 __data(__get_lvalue(__x)));
3549 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3554template <
typename _Tp,
typename _Ap>
3555 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3556 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3557 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3559template <
typename _Tp,
typename _Ap>
3560 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3561 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3562 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3564template <
typename _Tp,
typename _Ap>
3565 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3566 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3567 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3569 const auto pair_of_members
3570 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3571 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3572 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3575template <
typename _Tp,
typename _Ap>
3576 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3577 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
3578 const simd<_Tp, _Ap>& __hi)
3580 using _Impl =
typename _Ap::_SimdImpl;
3581 return {__private_init,
3582 _Impl::_S_min(__data(__hi),
3583 _Impl::_S_max(__data(__lo), __data(__v)))};
3588template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3590 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3591 split(
const simd<_Tp, _Ap>&);
3594template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3595 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3596 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3597 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3599template <
int _Index,
int _Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
3601 _GLIBCXX_SIMD_INTRINSIC
auto
3602 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3606template <
size_t _V0,
size_t... _Values>
3609 template <
size_t _I>
3610 static constexpr size_t _S_at(_SizeConstant<_I> = {})
3612 if constexpr (_I == 0)
3615 return _SizeList<_Values...>::template _S_at<_I - 1>();
3618 template <
size_t _I>
3619 static constexpr auto _S_before(_SizeConstant<_I> = {})
3621 if constexpr (_I == 0)
3622 return _SizeConstant<0>();
3624 return _SizeConstant<
3625 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3628 template <
size_t _Np>
3629 static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3631 if constexpr (_Np == 0)
3634 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3640template <
typename _Tp,
size_t _Np>
3641 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3642 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3644 static_assert(_Np >= 4);
3645 static_assert(_Np % 4 == 0);
3646#if _GLIBCXX_SIMD_X86INTRIN
3647 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
3649 const auto __intrin = __to_intrin(__x);
3650 if constexpr (is_integral_v<_Tp>)
3651 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3652 _mm512_shuffle_i32x4(__intrin, __intrin,
3653 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3654 else if constexpr (
sizeof(_Tp) == 4)
3655 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3656 _mm512_shuffle_f32x4(__intrin, __intrin,
3657 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3658 else if constexpr (
sizeof(_Tp) == 8)
3659 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3660 _mm512_shuffle_f64x2(__intrin, __intrin,
3661 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3663 __assert_unreachable<_Tp>();
3665 else if constexpr (
sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3666 return __vector_bitcast<_Tp>(
3667 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3668 __hi128(__vector_bitcast<double>(__x)), 1));
3669 else if constexpr (
sizeof(__x) == 32 &&
sizeof(_Tp) * _Np <= 32)
3670 return __vector_bitcast<_Tp>(
3671 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3672 __lo128(__vector_bitcast<_LLong>(__x)),
3673 sizeof(_Tp) * _Np / 4));
3677 __vector_type_t<_Tp, _Np / 2> __r;
3678 __builtin_memcpy(&__r,
3679 reinterpret_cast<const char*
>(&__x)
3680 +
sizeof(_Tp) * _Np / 4,
3681 sizeof(_Tp) * _Np / 2);
3686template <
typename _Tp,
typename _A0,
typename... _As>
3687 _GLIBCXX_SIMD_INTRINSIC
3688 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3689 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
3691 if constexpr (
sizeof...(_As) == 0)
3692 return __extract_center(__x.first);
3694 return __extract_part<1, 4, 2>(__x);
3699template <
size_t... _Sizes,
typename _Tp,
typename... _As>
3701 __split_wrapper(_SizeList<_Sizes...>,
const _SimdTuple<_Tp, _As...>& __x)
3703 return split<_Sizes...>(
3704 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3711template <
typename _V,
typename _Ap,
3712 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> /
_V::size()>
3713 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts *
_V::size()
3714 && is_simd_v<_V>, array<_V, _Parts>>
3715 split(
const simd<typename _V::value_type, _Ap>& __x)
3717 using _Tp =
typename _V::value_type;
3718 if constexpr (_Parts == 1)
3720 return {simd_cast<_V>(__x)};
3722 else if (__x._M_is_constprop())
3724 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3725 auto __i)
constexpr {
3726 return _V([&](
auto __j)
constexpr {
3727 return __x[__i *
_V::size() + __j];
3732 __is_fixed_size_abi_v<_Ap>
3733 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3734 || (__is_fixed_size_abi_v<typename _V::abi_type>
3735 &&
sizeof(_V) ==
sizeof(_Tp) *
_V::size()
3739#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3740 const __may_alias<_Tp>*
const __element_ptr
3741 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
3742 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3743 auto __i)
constexpr {
3744 return _V(__element_ptr + __i *
_V::size(), vector_aligned);
3747 const auto& __xx = __data(__x);
3748 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3749 auto __i)
constexpr {
3750 [[maybe_unused]]
constexpr size_t __offset
3751 =
decltype(__i)::value *
_V::size();
3752 return _V([&](
auto __j)
constexpr {
3753 constexpr _SizeConstant<__j + __offset> __k;
3759 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3762 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3763 auto __i)
constexpr {
return __x[__i]; });
3767 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3768 auto __i)
constexpr {
3769 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3770 return _V([&](
auto __j)
constexpr {
3771 return __x[__i *
_V::size() + __j];
3774 return _V(__private_init,
3775 __extract_part<
decltype(__i)::value, _Parts>(__data(__x)));
3782template <
typename _V,
typename _Ap,
3784 = simd_size_v<typename _V::simd_type::value_type, _Ap> /
_V::size()>
3785 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
3786 _V::simd_type::value_type, _Ap> == _Parts *
_V::size(), array<_V, _Parts>>
3787 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3789 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3791 else if constexpr (_Parts == 1)
3792 return {__proposed::static_simd_cast<_V>(__x)};
3793 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3794 && __is_avx_abi<_Ap>())
3795 return {_V(__private_init, __lo128(__data(__x))),
3796 _V(__private_init, __hi128(__data(__x)))};
3797 else if constexpr (
_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
3799 const bitset __bits = __x.__to_bitset();
3800 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3801 auto __i)
constexpr {
3802 constexpr size_t __offset = __i *
_V::size();
3803 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3808 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3809 auto __i)
constexpr {
3810 constexpr size_t __offset = __i *
_V::size();
3812 __private_init, [&](
auto __j)
constexpr {
3813 return __x[__j + __offset];
3821template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
3822 _GLIBCXX_SIMD_ALWAYS_INLINE
3823 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3824 split(
const simd<_Tp, _Ap>& __x)
3826 using _SL = _SizeList<_Sizes...>;
3827 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3828 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3829 constexpr size_t _N0 = _SL::template _S_at<0>();
3830 using _V = __deduced_simd<_Tp, _N0>;
3832 if (__x._M_is_constprop())
3833 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3834 auto __i)
constexpr {
3835 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3836 constexpr size_t __offset = _SL::_S_before(__i);
3837 return _Vi([&](
auto __j)
constexpr {
return __x[__offset + __j]; });
3839 else if constexpr (_Np == _N0)
3841 static_assert(
sizeof...(_Sizes) == 1);
3842 return {simd_cast<_V>(__x)};
3845 (__is_fixed_size_abi_v<
3846 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3849 !__is_fixed_size_abi_v<typename _V::abi_type>,
3850 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3855 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3856 __split_wrapper(_SL::template _S_pop_front<1>(),
3857 __data(__x).second));
3859 else if constexpr ((!is_same_v<simd_abi::scalar,
3860 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3861 && (!__is_fixed_size_abi_v<
3862 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3864 if constexpr (((_Sizes * 2 == _Np) && ...))
3865 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3866 {__private_init, __extract_part<1, 2>(__data(__x))}};
3867 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3868 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3869 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3870 {__private_init, __extract_part<1, 3>(__data(__x))},
3871 {__private_init, __extract_part<2, 3>(__data(__x))}};
3872 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3873 _SizeList<2 * _Np / 3, _Np / 3>>)
3874 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3875 {__private_init, __extract_part<2, 3>(__data(__x))}};
3876 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3877 _SizeList<_Np / 3, 2 * _Np / 3>>)
3878 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3879 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3880 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3881 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3882 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3883 {__private_init, __extract_part<2, 4>(__data(__x))},
3884 {__private_init, __extract_part<3, 4>(__data(__x))}};
3885 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3886 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3887 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3888 {__private_init, __extract_part<1, 4>(__data(__x))},
3889 {__private_init, __extract_part<1, 2>(__data(__x))}};
3890 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3891 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3892 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3893 {__private_init, __extract_center(__data(__x))},
3894 {__private_init, __extract_part<3, 4>(__data(__x))}};
3895 else if constexpr (((_Sizes * 4 == _Np) && ...))
3896 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3897 {__private_init, __extract_part<1, 4>(__data(__x))},
3898 {__private_init, __extract_part<2, 4>(__data(__x))},
3899 {__private_init, __extract_part<3, 4>(__data(__x))}};
3902#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3903 const __may_alias<_Tp>*
const __element_ptr
3904 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
3905 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3906 auto __i)
constexpr {
3907 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3908 constexpr size_t __offset = _SL::_S_before(__i);
3909 constexpr size_t __base_align =
alignof(simd<_Tp, _Ap>);
3910 constexpr size_t __a
3911 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
3912 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
3913 constexpr size_t __alignment = __b == 0 ? __a : __b;
3914 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
3917 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3918 auto __i)
constexpr {
3919 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3920 const auto& __xx = __data(__x);
3921 using _Offset =
decltype(_SL::_S_before(__i));
3922 return _Vi([&](
auto __j)
constexpr {
3923 constexpr _SizeConstant<_Offset::value + __j> __k;
3933template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
3934 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
3935 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
3937 if constexpr (_I < simd_size_v<_Tp, _Ap>)
3940 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
3945template <
typename _Tp,
typename _A0,
typename... _As>
3946 _GLIBCXX_SIMD_INTRINSIC
void
3947 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
3948 const simd<_Tp, _As>&... __xs)
3950 constexpr size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
3951 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
3952 if constexpr (
sizeof...(__xs) > 0)
3953 __store_pack_of_simd(__mem + __n_bytes, __xs...);
3958template <
typename _Tp,
typename... _As>
3959 inline _GLIBCXX_SIMD_CONSTEXPR
3960 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
3961 concat(
const simd<_Tp, _As>&... __xs)
3963 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
3964 if constexpr (
sizeof...(__xs) == 1)
3965 return simd_cast<_Rp>(__xs...);
3966 else if ((... && __xs._M_is_constprop()))
3968 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
3969 auto __i)
constexpr {
return __subscript_in_pack<__i>(__xs...); });
3973 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
3980template <
typename _Tp,
typename _Abi,
size_t _Np>
3981 _GLIBCXX_SIMD_ALWAYS_INLINE
3982 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
3983 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
3985 return __call_with_subscripts<_Np>(__x, [](
const auto&... __xs) {
3986 return concat(__xs...);
3994template <
typename _Up,
typename _Accessor = _Up,
3995 typename _ValueType =
typename _Up::value_type>
3996 class _SmartReference
4002 _GLIBCXX_SIMD_INTRINSIC
constexpr _ValueType _M_read() const noexcept
4004 if constexpr (is_arithmetic_v<_Up>)
4007 return _M_obj[_M_index];
4010 template <
typename _Tp>
4011 _GLIBCXX_SIMD_INTRINSIC
constexpr void _M_write(_Tp&& __x)
const
4012 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
4015 _GLIBCXX_SIMD_INTRINSIC
constexpr
4016 _SmartReference(_Up& __o,
int __i) noexcept
4017 : _M_index(__i), _M_obj(__o) {}
4019 using value_type = _ValueType;
4021 _GLIBCXX_SIMD_INTRINSIC _SmartReference(
const _SmartReference&) =
delete;
4023 _GLIBCXX_SIMD_INTRINSIC
constexpr operator value_type() const noexcept
4024 {
return _M_read(); }
4026 template <
typename _Tp,
4028 = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4029 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference operator=(_Tp&& __x) &&
4031 _M_write(
static_cast<_Tp&&
>(__x));
4032 return {_M_obj, _M_index};
4035#define _GLIBCXX_SIMD_OP_(__op) \
4036 template <typename _Tp, \
4038 = decltype(declval<value_type>() __op declval<_Tp>()), \
4039 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4040 typename = _ValuePreservingOrInt<_TT, value_type>> \
4041 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4042 operator __op##=(_Tp&& __x) && \
4044 const value_type& __lhs = _M_read(); \
4045 _M_write(__lhs __op __x); \
4046 return {_M_obj, _M_index}; \
4048 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4049 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4050 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4051#undef _GLIBCXX_SIMD_OP_
4053 template <
typename _Tp = void,
4055 =
decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4056 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference operator++() &&
4058 value_type __x = _M_read();
4060 return {_M_obj, _M_index};
4063 template <
typename _Tp = void,
4065 =
decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4066 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type operator++(
int) &&
4068 const value_type __r = _M_read();
4069 value_type __x = __r;
4074 template <
typename _Tp = void,
4076 =
decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4077 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference operator--() &&
4079 value_type __x = _M_read();
4081 return {_M_obj, _M_index};
4084 template <
typename _Tp = void,
4086 =
decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4087 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type operator--(
int) &&
4089 const value_type __r = _M_read();
4090 value_type __x = __r;
4095 _GLIBCXX_SIMD_INTRINSIC
friend void
4096 swap(_SmartReference&& __a, _SmartReference&& __b)
noexcept(
4098 is_nothrow_constructible<value_type, _SmartReference&&>,
4099 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4101 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
4102 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
4103 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4106 _GLIBCXX_SIMD_INTRINSIC
friend void
4107 swap(value_type& __a, _SmartReference&& __b)
noexcept(
4109 is_nothrow_constructible<value_type, value_type&&>,
4110 is_nothrow_assignable<value_type&, value_type&&>,
4111 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4114 __a =
static_cast<value_type
>(__b);
4115 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4118 _GLIBCXX_SIMD_INTRINSIC
friend void
4119 swap(_SmartReference&& __a, value_type& __b)
noexcept(
4121 is_nothrow_constructible<value_type, _SmartReference&&>,
4122 is_nothrow_assignable<value_type&, value_type&&>,
4123 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4125 value_type __tmp(__a);
4126 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
4133template <
int _Bytes>
4134 struct __scalar_abi_wrapper
4136 template <
typename _Tp>
static constexpr size_t _S_full_size = 1;
4137 template <
typename _Tp>
static constexpr size_t _S_size = 1;
4138 template <
typename _Tp>
static constexpr size_t _S_is_partial =
false;
4140 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
4141 static constexpr bool _S_is_valid_v
4142 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
4147template <
typename _Tp>
4148 struct __decay_abi {
using type = _Tp; };
4150template <
int _Bytes>
4151 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4152 {
using type = simd_abi::scalar; };
4160template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
4161 struct __find_next_valid_abi
4163 static constexpr auto _S_choose()
4165 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4166 using _NextAbi = _Abi<_NextBytes>;
4167 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4168 return _Abi<_Bytes>();
4169 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4170 && _NextAbi::template _S_is_valid_v<_Tp>)
4173 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4176 using type =
decltype(_S_choose());
4179template <
int _Bytes,
typename _Tp>
4180 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4181 {
using type = simd_abi::scalar; };
4184template <
template <
int>
class...>
4187 template <
typename,
int>
static constexpr bool _S_has_valid_abi =
false;
4188 template <
typename,
int>
using _FirstValidAbi = void;
4189 template <
typename,
int>
using _BestAbi = void;
4192template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4193 struct _AbiList<_A0, _Rest...>
4195 template <
typename _Tp,
int _Np>
4196 static constexpr bool _S_has_valid_abi
4197 = _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<
4198 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4200 template <
typename _Tp,
int _Np>
4202 _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4203 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4204 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4206 template <
typename _Tp,
int _Np>
4207 static constexpr auto _S_determine_best_abi()
4209 static_assert(_Np >= 1);
4210 constexpr int _Bytes =
sizeof(_Tp) * _Np;
4211 if constexpr (_Np == 1)
4212 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4215 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4220 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4221 _Tp> && __fullsize / 2 < _Np)
4222 return typename __decay_abi<_A0<_Bytes>>::type{};
4226 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4227 if constexpr (_Bp::template _S_is_valid_v<
4228 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4232 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4237 template <
typename _Tp,
int _Np>
4238 using _BestAbi =
decltype(_S_determine_best_abi<_Tp, _Np>());
4246using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4247 __scalar_abi_wrapper>;
4250template <
typename _Tp,
typename _Abi>
4251 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4252 : _Abi::template __traits<_Tp> {};
4256template <
typename _Tp,
size_t _Np>
4257 struct __deduce_impl<
4258 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4259 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4262template <
typename _Tp,
size_t _Np,
typename =
void>
4263 struct __deduce_fixed_size_fallback {};
4265template <
typename _Tp,
size_t _Np>
4266 struct __deduce_fixed_size_fallback<_Tp, _Np,
4267 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4268 {
using type = simd_abi::fixed_size<_Np>; };
4270template <
typename _Tp,
size_t _Np,
typename>
4271 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4277template <
typename _Tp,
typename _Abi>
4278 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4281 using _Traits = _SimdTraits<_Tp, _Abi>;
4282 using _MemberType =
typename _Traits::_MaskMember;
4288 using _Ip = __int_for_sizeof_t<_Tp>;
4289 static constexpr _Ip* _S_type_tag =
nullptr;
4291 friend typename _Traits::_MaskBase;
4292 friend class simd<_Tp, _Abi>;
4293 friend typename _Traits::_SimdImpl;
4296 using _Impl =
typename _Traits::_MaskImpl;
4301 using value_type = bool;
4302 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4303 using simd_type = simd<_Tp, _Abi>;
4304 using abi_type = _Abi;
4307 static constexpr size_t size()
4308 {
return __size_or_zero_v<_Tp, _Abi>; }
4312 simd_mask() =
default;
4313 simd_mask(
const simd_mask&) =
default;
4314 simd_mask(simd_mask&&) =
default;
4315 simd_mask& operator=(
const simd_mask&) =
default;
4316 simd_mask& operator=(simd_mask&&) =
default;
4320 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit
4321 simd_mask(
typename _Traits::_MaskCastType __init)
4322 : _M_data{__init} {}
4331 _GLIBCXX_SIMD_ALWAYS_INLINE
static simd_mask
4332 __from_bitset(bitset<
size()> bs)
4333 {
return {__bitset_init, bs}; }
4335 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<
size()>
4337 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4341 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4342 simd_mask(value_type __x)
4343 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4347 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4349 template <
typename _Up,
typename _A2,
4350 typename = enable_if_t<simd_size_v<_Up, _A2> ==
size()>>
4351 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4352 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4353 simd_mask(
const simd_mask<_Up, _A2>& __x)
4354 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4357 template <
typename _Up,
typename =
enable_if_t<conjunction<
4358 is_same<abi_type, simd_abi::fixed_size<
size()>>,
4359 is_same<_Up, _Up>>::value>>
4360 _GLIBCXX_SIMD_ALWAYS_INLINE
4361 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<
size()>>& __x)
4362 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4367 template <
typename _Flags>
4368 _GLIBCXX_SIMD_ALWAYS_INLINE
4369 simd_mask(
const value_type* __mem, _Flags)
4370 : _M_data(_Impl::template _S_load<_Ip>(
4371 _Flags::template _S_apply<simd_mask>(__mem))) {}
4373 template <
typename _Flags>
4374 _GLIBCXX_SIMD_ALWAYS_INLINE
4375 simd_mask(
const value_type* __mem, simd_mask __k, _Flags)
4379 = _Impl::_S_masked_load(_M_data, __k._M_data,
4380 _Flags::template _S_apply<simd_mask>(__mem));
4385 template <
typename _Flags>
4386 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4387 copy_from(
const value_type* __mem, _Flags)
4389 _M_data = _Impl::template _S_load<_Ip>(
4390 _Flags::template _S_apply<simd_mask>(__mem));
4395 template <
typename _Flags>
4396 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4397 copy_to(value_type* __mem, _Flags)
const
4398 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4402 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4403 operator[](
size_t __i)
4406 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i,
size() - 1);
4407 return {_M_data, int(__i)};
4410 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4411 operator[](
size_t __i)
const
4414 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i,
size() - 1);
4415 if constexpr (__is_scalar_abi<_Abi>())
4418 return static_cast<bool>(_M_data[__i]);
4423 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4425 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4429 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4431 template <
typename _Up,
typename _A2,
4433 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4434 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4435 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4437 return {__private_init,
4438 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4441 template <
typename _Up,
typename _A2,
4443 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4444 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4445 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4447 return {__private_init,
4448 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4452 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4453 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4455 return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4458 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4459 operator||(
const simd_mask& __x,
const simd_mask& __y)
4461 return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4464 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4465 operator&(
const simd_mask& __x,
const simd_mask& __y)
4466 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4468 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4469 operator|(
const simd_mask& __x,
const simd_mask& __y)
4470 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4472 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4473 operator^(
const simd_mask& __x,
const simd_mask& __y)
4474 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4476 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4477 operator&=(simd_mask& __x,
const simd_mask& __y)
4479 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4483 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4484 operator|=(simd_mask& __x,
const simd_mask& __y)
4486 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4490 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4491 operator^=(simd_mask& __x,
const simd_mask& __y)
4493 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4499 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4500 operator==(
const simd_mask& __x,
const simd_mask& __y)
4501 {
return !operator!=(__x, __y); }
4503 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4504 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4505 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4509 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4510 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4511 : _M_data(__init) {}
4515 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4516 _GLIBCXX_SIMD_INTRINSIC
constexpr
4517 simd_mask(_PrivateInit, _Fp&& __gen)
4520 __execute_n_times<size()>([&](
auto __i)
constexpr {
4521 _Impl::_S_set(_M_data, __i, __gen(__i));
4527 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<
size()> __init)
4529 _Impl::_S_from_bitmask(_SanitizedBitMask<
size()>(__init), _S_type_tag))
4540 template <
typename _Up,
typename _A2,
4542 = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4543 operator simd_mask<_Up, _A2>() &&
4545 using namespace std::experimental::__proposed;
4546 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4549 const simd_mask<_Tp, _Abi>& _M_data;
4552 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4558 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4559 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4560 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4561 const simd_mask& __where_false)
4563 auto __ret = __where_false;
4564 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4568 template <
typename _U1,
typename _U2,
4569 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4571 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4572 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4573 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4574 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4575 const _U2& __where_false)
4577 _Rp __ret = __where_false;
4578 _Rp::_Impl::_S_masked_assign(
4579 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4580 __data(
static_cast<_Rp
>(__where_true)));
4584 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4585 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4587 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4588 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4589 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4590 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4591 const simd_mask<_Up, _Au>& __where_false)
4593 simd_mask __ret = __where_false;
4594 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4595 __where_true._M_data);
4603 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
4604 _M_is_constprop()
const
4606 if constexpr (__is_scalar_abi<_Abi>())
4607 return __builtin_constant_p(_M_data);
4609 return _M_data._M_is_constprop();
4615 friend const auto& __data<_Tp, abi_type>(
const simd_mask&);
4616 friend auto& __data<_Tp, abi_type>(simd_mask&);
4617 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4624template <
typename _Tp,
typename _Ap>
4625 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
4626 __data(
const simd_mask<_Tp, _Ap>& __x)
4627 {
return __x._M_data; }
4629template <
typename _Tp,
typename _Ap>
4630 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
4631 __data(simd_mask<_Tp, _Ap>& __x)
4632 {
return __x._M_data; }
4638template <
typename _Tp,
typename _Abi>
4639 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4640 all_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4642 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4644 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4650 return _Abi::_MaskImpl::_S_all_of(__k);
4653template <
typename _Tp,
typename _Abi>
4654 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4655 any_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4657 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4659 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4665 return _Abi::_MaskImpl::_S_any_of(__k);
4668template <
typename _Tp,
typename _Abi>
4669 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4670 none_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4672 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4674 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4680 return _Abi::_MaskImpl::_S_none_of(__k);
4683template <
typename _Tp,
typename _Abi>
4684 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4685 some_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4687 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4689 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4690 if (__k[__i] != __k[__i - 1])
4695 return _Abi::_MaskImpl::_S_some_of(__k);
4698template <
typename _Tp,
typename _Abi>
4699 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4700 popcount(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4702 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4704 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4705 __k, [](
auto... __elements) {
return ((__elements != 0) + ...); });
4706 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4709 return _Abi::_MaskImpl::_S_popcount(__k);
4712template <
typename _Tp,
typename _Abi>
4713 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4714 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
4716 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4718 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4719 const size_t _Idx = __call_with_n_evaluations<_Np>(
4720 [](
auto... __indexes) {
return std::min({__indexes...}); },
4721 [&](
auto __i) {
return __k[__i] ? +__i : _Np; });
4723 __invoke_ub(
"find_first_set(empty mask) is UB");
4724 if (__builtin_constant_p(_Idx))
4727 return _Abi::_MaskImpl::_S_find_first_set(__k);
4730template <
typename _Tp,
typename _Abi>
4731 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4732 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
4734 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4736 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4737 const int _Idx = __call_with_n_evaluations<_Np>(
4738 [](
auto... __indexes) {
return std::max({__indexes...}); },
4739 [&](
auto __i) {
return __k[__i] ? int(__i) : -1; });
4741 __invoke_ub(
"find_first_set(empty mask) is UB");
4742 if (__builtin_constant_p(_Idx))
4745 return _Abi::_MaskImpl::_S_find_last_set(__k);
4748_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4749all_of(_ExactBool __x)
noexcept
4752_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4753any_of(_ExactBool __x)
noexcept
4756_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4757none_of(_ExactBool __x)
noexcept
4760_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4761some_of(_ExactBool)
noexcept
4764_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4765popcount(_ExactBool __x)
noexcept
4768_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4769find_first_set(_ExactBool)
4772_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4773find_last_set(_ExactBool)
4780template <
typename _V,
typename _Impl,
bool>
4781 class _SimdIntOperators {};
4783template <
typename _V,
typename _Impl>
4784 class _SimdIntOperators<_V, _Impl, true>
4786 _GLIBCXX_SIMD_INTRINSIC
const _V& __derived()
const
4787 {
return *
static_cast<const _V*
>(
this); }
4789 template <
typename _Tp>
4790 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
4791 _S_make_derived(_Tp&& __d)
4792 {
return {__private_init,
static_cast<_Tp&&
>(__d)}; }
4795 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator%=(_V& __lhs,
const _V& __x)
4796 {
return __lhs = __lhs % __x; }
4798 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator&=(_V& __lhs,
const _V& __x)
4799 {
return __lhs = __lhs & __x; }
4801 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator|=(_V& __lhs,
const _V& __x)
4802 {
return __lhs = __lhs | __x; }
4804 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator^=(_V& __lhs,
const _V& __x)
4805 {
return __lhs = __lhs ^ __x; }
4807 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator<<=(_V& __lhs,
const _V& __x)
4808 {
return __lhs = __lhs << __x; }
4810 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator>>=(_V& __lhs,
const _V& __x)
4811 {
return __lhs = __lhs >> __x; }
4813 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator<<=(_V& __lhs,
int __x)
4814 {
return __lhs = __lhs << __x; }
4816 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator>>=(_V& __lhs,
int __x)
4817 {
return __lhs = __lhs >> __x; }
4819 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator%(
const _V& __x,
const _V& __y)
4821 return _SimdIntOperators::_S_make_derived(
4822 _Impl::_S_modulus(__data(__x), __data(__y)));
4825 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator&(
const _V& __x,
const _V& __y)
4827 return _SimdIntOperators::_S_make_derived(
4828 _Impl::_S_bit_and(__data(__x), __data(__y)));
4831 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator|(
const _V& __x,
const _V& __y)
4833 return _SimdIntOperators::_S_make_derived(
4834 _Impl::_S_bit_or(__data(__x), __data(__y)));
4837 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator^(
const _V& __x,
const _V& __y)
4839 return _SimdIntOperators::_S_make_derived(
4840 _Impl::_S_bit_xor(__data(__x), __data(__y)));
4843 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator<<(
const _V& __x,
const _V& __y)
4845 return _SimdIntOperators::_S_make_derived(
4846 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4849 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator>>(
const _V& __x,
const _V& __y)
4851 return _SimdIntOperators::_S_make_derived(
4852 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4855 template <
typename _VV = _V>
4856 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator<<(
const _V& __x,
int __y)
4858 using _Tp =
typename _VV::value_type;
4860 __invoke_ub(
"The behavior is undefined if the right operand of a "
4861 "shift operation is negative. [expr.shift]\nA shift by "
4864 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4866 "The behavior is undefined if the right operand of a "
4867 "shift operation is greater than or equal to the width of the "
4868 "promoted left operand. [expr.shift]\nA shift by %d was requested",
4870 return _SimdIntOperators::_S_make_derived(
4871 _Impl::_S_bit_shift_left(__data(__x), __y));
4874 template <
typename _VV = _V>
4875 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator>>(
const _V& __x,
int __y)
4877 using _Tp =
typename _VV::value_type;
4880 "The behavior is undefined if the right operand of a shift "
4881 "operation is negative. [expr.shift]\nA shift by %d was requested",
4883 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4885 "The behavior is undefined if the right operand of a shift "
4886 "operation is greater than or equal to the width of the promoted "
4887 "left operand. [expr.shift]\nA shift by %d was requested",
4889 return _SimdIntOperators::_S_make_derived(
4890 _Impl::_S_bit_shift_right(__data(__x), __y));
4894 _GLIBCXX_SIMD_CONSTEXPR _V operator~()
const
4895 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
4902template <
typename _Tp,
typename _Abi>
4903 class simd :
public _SimdIntOperators<
4904 simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
4905 conjunction<is_integral<_Tp>,
4906 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
4907 public _SimdTraits<_Tp, _Abi>::_SimdBase
4909 using _Traits = _SimdTraits<_Tp, _Abi>;
4910 using _MemberType =
typename _Traits::_SimdMember;
4911 using _CastType =
typename _Traits::_SimdCastType;
4912 static constexpr _Tp* _S_type_tag =
nullptr;
4913 friend typename _Traits::_SimdBase;
4916 using _Impl =
typename _Traits::_SimdImpl;
4918 friend _SimdIntOperators<simd, _Impl, true>;
4920 using value_type = _Tp;
4921 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4922 using mask_type = simd_mask<_Tp, _Abi>;
4923 using abi_type = _Abi;
4925 static constexpr size_t size()
4926 {
return __size_or_zero_v<_Tp, _Abi>; }
4928 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
4929 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
4930 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
4931 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
4932 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
4935 template <typename _Up,
4936 typename =
enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
4937 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4938 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
4940 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
4945 template <
typename _Up>
4946 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4947 simd(
const simd<_Up, simd_abi::fixed_size<
size()>>& __x,
4950 is_same<simd_abi::fixed_size<
size()>, abi_type>,
4951 negation<__is_narrowing_conversion<_Up, value_type>>,
4952 __converts_to_higher_integer_rank<_Up, value_type>>::value,
4954 : simd{static_cast<array<_Up,
size()>>(__x).
data(), vector_aligned} {}
4957#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
4958 template <
typename _Up,
typename _A2,
4959 typename =
decltype(static_simd_cast<simd>(
4960 declval<
const simd<_Up, _A2>&>()))>
4961 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4962 simd(
const simd<_Up, _A2>& __x)
4963 : simd(static_simd_cast<simd>(__x)) {}
4967 template <
typename _Fp>
4968 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4969 simd(_Fp&& __gen, _ValuePreservingOrInt<
decltype(declval<_Fp>()(
4970 declval<_SizeConstant<0>&>())),
4971 value_type>* =
nullptr)
4972 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
4975 template <
typename _Up,
typename _Flags>
4976 _GLIBCXX_SIMD_ALWAYS_INLINE
4977 simd(
const _Up* __mem, _Flags)
4979 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
4983 template <
typename _Up,
typename _Flags>
4984 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4985 copy_from(
const _Vectorizable<_Up>* __mem, _Flags)
4987 _M_data =
static_cast<decltype(_M_data)
>(
4988 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
4992 template <
typename _Up,
typename _Flags>
4993 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4994 copy_to(_Vectorizable<_Up>* __mem, _Flags)
const
4996 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5001 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5002 operator[](
size_t __i)
5003 {
return {_M_data, int(__i)}; }
5005 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5006 operator[]([[maybe_unused]]
size_t __i)
const
5008 if constexpr (__is_scalar_abi<_Abi>())
5010 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5014 return _M_data[__i];
5018 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5021 _Impl::_S_increment(_M_data);
5025 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5029 _Impl::_S_increment(_M_data);
5033 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5036 _Impl::_S_decrement(_M_data);
5040 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5044 _Impl::_S_decrement(_M_data);
5049 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5051 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
5053 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5057 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5059 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5062 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5063 simd(_CastType __init) : _M_data(__init) {}
5066 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5067 operator+=(simd& __lhs,
const simd& __x)
5068 {
return __lhs = __lhs + __x; }
5070 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5071 operator-=(simd& __lhs,
const simd& __x)
5072 {
return __lhs = __lhs - __x; }
5074 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5075 operator*=(simd& __lhs,
const simd& __x)
5076 {
return __lhs = __lhs * __x; }
5078 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5079 operator/=(simd& __lhs,
const simd& __x)
5080 {
return __lhs = __lhs / __x; }
5083 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5084 operator+(
const simd& __x,
const simd& __y)
5085 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5087 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5088 operator-(
const simd& __x,
const simd& __y)
5089 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5091 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5092 operator*(
const simd& __x,
const simd& __y)
5093 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5095 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5096 operator/(
const simd& __x,
const simd& __y)
5097 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5100 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5101 operator==(
const simd& __x,
const simd& __y)
5102 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5104 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5105 operator!=(
const simd& __x,
const simd& __y)
5107 return simd::_S_make_mask(
5108 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5111 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5112 operator<(
const simd& __x,
const simd& __y)
5113 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5115 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5116 operator<=(
const simd& __x,
const simd& __y)
5118 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5121 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5122 operator>(
const simd& __x,
const simd& __y)
5123 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5125 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5126 operator>=(
const simd& __x,
const simd& __y)
5128 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5132#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5133 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5134 operator?:(
const mask_type& __k,
const simd& __where_true,
5135 const simd& __where_false)
5137 auto __ret = __where_false;
5138 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5146 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5147 simd(_PrivateInit,
const _MemberType& __init)
5148 : _M_data(__init) {}
5151 _GLIBCXX_SIMD_INTRINSIC
5152 simd(_BitsetInit, bitset<
size()> __init) : _M_data()
5153 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
5155 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
5156 _M_is_constprop()
const
5158 if constexpr (__is_scalar_abi<_Abi>())
5159 return __builtin_constant_p(_M_data);
5161 return _M_data._M_is_constprop();
5165 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
static mask_type
5166 _S_make_mask(
typename mask_type::_MemberType __k)
5167 {
return {__private_init, __k}; }
5169 friend const auto& __data<value_type, abi_type>(
const simd&);
5170 friend auto& __data<value_type, abi_type>(simd&);
5171 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5177template <
typename _Tp,
typename _Ap>
5178 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
5179 __data(
const simd<_Tp, _Ap>& __x)
5180 {
return __x._M_data; }
5182template <
typename _Tp,
typename _Ap>
5183 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
5184 __data(simd<_Tp, _Ap>& __x)
5185 {
return __x._M_data; }
5188namespace __float_bitwise_operators {
5189template <
typename _Tp,
typename _Ap>
5190 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5191 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5193 return {__private_init,
5194 _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5197template <
typename _Tp,
typename _Ap>
5198 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5199 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5201 return {__private_init,
5202 _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5205template <
typename _Tp,
typename _Ap>
5206 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5207 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5209 return {__private_init,
5210 _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5216_GLIBCXX_SIMD_END_NAMESPACE
constexpr duration< __common_rep_t< _Rep2, _Rep1 >, _Period > operator*(const _Rep1 &__s, const duration< _Rep2, _Period > &__d)
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
constexpr time_point< _Clock, typename common_type< duration< _Rep1, _Period1 >, _Dur2 >::type > operator+(const duration< _Rep1, _Period1 > &__lhs, const time_point< _Clock, _Dur2 > &__rhs)
Adjust a time point forwards by the given duration.
constexpr common_type< duration< _Rep1, _Period1 >, duration< _Rep2, _Period2 > >::type operator-(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
The difference between two durations.
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator/(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
void swap(any &__x, any &__y) noexcept
Exchange the states of two any objects.
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.