libstdc++
simd.h
1// Definition of the public simd interfaces -*- C++ -*-
2
3// Copyright (C) 2020-2021 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
27
28#if __cplusplus >= 201703L
29
30#include "simd_detail.h"
31#include "numeric_traits.h"
32#include <bit>
33#include <bitset>
34#ifdef _GLIBCXX_DEBUG_UB
35#include <cstdio> // for stderr
36#endif
37#include <cstring>
38#include <functional>
39#include <iosfwd>
40#include <utility>
41
42#if _GLIBCXX_SIMD_X86INTRIN
43#include <x86intrin.h>
44#elif _GLIBCXX_SIMD_HAVE_NEON
45#include <arm_neon.h>
46#endif
47
48/** @ingroup ts_simd
49 * @{
50 */
51/* There are several closely related types, with the following naming
52 * convention:
53 * _Tp: vectorizable (arithmetic) type (or any type)
54 * _TV: __vector_type_t<_Tp, _Np>
55 * _TW: _SimdWrapper<_Tp, _Np>
56 * _TI: __intrinsic_type_t<_Tp, _Np>
57 * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
58 * If one additional type is needed use _U instead of _T.
59 * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
60 *
61 * More naming conventions:
62 * _Ap or _Abi: An ABI tag from the simd_abi namespace
63 * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
64 * _IV, _IW as for _TV, _TW
65 * _Np: number of elements (not bytes)
66 * _Bytes: number of bytes
67 *
68 * Variable names:
69 * __k: mask object (vector- or bitmask)
70 */
71_GLIBCXX_SIMD_BEGIN_NAMESPACE
72
73#if !_GLIBCXX_SIMD_X86INTRIN
74using __m128 [[__gnu__::__vector_size__(16)]] = float;
75using __m128d [[__gnu__::__vector_size__(16)]] = double;
76using __m128i [[__gnu__::__vector_size__(16)]] = long long;
77using __m256 [[__gnu__::__vector_size__(32)]] = float;
78using __m256d [[__gnu__::__vector_size__(32)]] = double;
79using __m256i [[__gnu__::__vector_size__(32)]] = long long;
80using __m512 [[__gnu__::__vector_size__(64)]] = float;
81using __m512d [[__gnu__::__vector_size__(64)]] = double;
82using __m512i [[__gnu__::__vector_size__(64)]] = long long;
83#endif
84
85namespace simd_abi {
86// simd_abi forward declarations {{{
87// implementation details:
88struct _Scalar;
89
90template <int _Np>
91 struct _Fixed;
92
93// There are two major ABIs that appear on different architectures.
94// Both have non-boolean values packed into an N Byte register
95// -> #elements = N / sizeof(T)
96// Masks differ:
97// 1. Use value vector registers for masks (all 0 or all 1)
98// 2. Use bitmasks (mask registers) with one bit per value in the corresponding
99// value vector
100//
101// Both can be partially used, masking off the rest when doing horizontal
102// operations or operations that can trap (e.g. FP_INVALID or integer division
103// by 0). This is encoded as the number of used bytes.
104template <int _UsedBytes>
105 struct _VecBuiltin;
106
107template <int _UsedBytes>
108 struct _VecBltnBtmsk;
109
110template <typename _Tp, int _Np>
111 using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
112
113template <int _UsedBytes = 16>
114 using _Sse = _VecBuiltin<_UsedBytes>;
115
116template <int _UsedBytes = 32>
117 using _Avx = _VecBuiltin<_UsedBytes>;
118
119template <int _UsedBytes = 64>
120 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
121
122template <int _UsedBytes = 16>
123 using _Neon = _VecBuiltin<_UsedBytes>;
124
125// implementation-defined:
126using __sse = _Sse<>;
127using __avx = _Avx<>;
128using __avx512 = _Avx512<>;
129using __neon = _Neon<>;
130using __neon128 = _Neon<16>;
131using __neon64 = _Neon<8>;
132
133// standard:
134template <typename _Tp, size_t _Np, typename...>
135 struct deduce;
136
137template <int _Np>
138 using fixed_size = _Fixed<_Np>;
139
140using scalar = _Scalar;
141
142// }}}
143} // namespace simd_abi
144// forward declarations is_simd(_mask), simd(_mask), simd_size {{{
145template <typename _Tp>
146 struct is_simd;
147
148template <typename _Tp>
149 struct is_simd_mask;
150
151template <typename _Tp, typename _Abi>
152 class simd;
153
154template <typename _Tp, typename _Abi>
155 class simd_mask;
156
157template <typename _Tp, typename _Abi>
158 struct simd_size;
159
160// }}}
161// load/store flags {{{
162struct element_aligned_tag
163{
164 template <typename _Tp, typename _Up = typename _Tp::value_type>
165 static constexpr size_t _S_alignment = alignof(_Up);
166
167 template <typename _Tp, typename _Up>
168 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
169 _S_apply(_Up* __ptr)
170 { return __ptr; }
171};
172
173struct vector_aligned_tag
174{
175 template <typename _Tp, typename _Up = typename _Tp::value_type>
176 static constexpr size_t _S_alignment
177 = std::__bit_ceil(sizeof(_Up) * _Tp::size());
178
179 template <typename _Tp, typename _Up>
180 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
181 _S_apply(_Up* __ptr)
182 {
183 return static_cast<_Up*>(
184 __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
185 }
186};
187
188template <size_t _Np> struct overaligned_tag
189{
190 template <typename _Tp, typename _Up = typename _Tp::value_type>
191 static constexpr size_t _S_alignment = _Np;
192
193 template <typename _Tp, typename _Up>
194 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
195 _S_apply(_Up* __ptr)
196 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
197};
198
199inline constexpr element_aligned_tag element_aligned = {};
200
201inline constexpr vector_aligned_tag vector_aligned = {};
202
203template <size_t _Np>
204 inline constexpr overaligned_tag<_Np> overaligned = {};
205
206// }}}
207template <size_t _Xp>
208 using _SizeConstant = integral_constant<size_t, _Xp>;
209
210namespace __detail
211{
212 struct _Minimum
213 {
214 template <typename _Tp>
215 _GLIBCXX_SIMD_INTRINSIC constexpr
216 _Tp
217 operator()(_Tp __a, _Tp __b) const
218 {
219 using std::min;
220 return min(__a, __b);
221 }
222 };
223
224 struct _Maximum
225 {
226 template <typename _Tp>
227 _GLIBCXX_SIMD_INTRINSIC constexpr
228 _Tp
229 operator()(_Tp __a, _Tp __b) const
230 {
231 using std::max;
232 return max(__a, __b);
233 }
234 };
235} // namespace __detail
236
237// unrolled/pack execution helpers
238// __execute_n_times{{{
239template <typename _Fp, size_t... _I>
240 _GLIBCXX_SIMD_INTRINSIC constexpr void
241 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
242 { ((void)__f(_SizeConstant<_I>()), ...); }
243
244template <typename _Fp>
245 _GLIBCXX_SIMD_INTRINSIC constexpr void
246 __execute_on_index_sequence(_Fp&&, index_sequence<>)
247 { }
248
249template <size_t _Np, typename _Fp>
250 _GLIBCXX_SIMD_INTRINSIC constexpr void
251 __execute_n_times(_Fp&& __f)
252 {
253 __execute_on_index_sequence(static_cast<_Fp&&>(__f),
254 make_index_sequence<_Np>{});
255 }
256
257// }}}
258// __generate_from_n_evaluations{{{
259template <typename _R, typename _Fp, size_t... _I>
260 _GLIBCXX_SIMD_INTRINSIC constexpr _R
261 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
262 { return _R{__f(_SizeConstant<_I>())...}; }
263
264template <size_t _Np, typename _R, typename _Fp>
265 _GLIBCXX_SIMD_INTRINSIC constexpr _R
266 __generate_from_n_evaluations(_Fp&& __f)
267 {
268 return __execute_on_index_sequence_with_return<_R>(
269 static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
270 }
271
272// }}}
273// __call_with_n_evaluations{{{
274template <size_t... _I, typename _F0, typename _FArgs>
275 _GLIBCXX_SIMD_INTRINSIC constexpr auto
276 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
277 { return __f0(__fargs(_SizeConstant<_I>())...); }
278
279template <size_t _Np, typename _F0, typename _FArgs>
280 _GLIBCXX_SIMD_INTRINSIC constexpr auto
281 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
282 {
283 return __call_with_n_evaluations(make_index_sequence<_Np>{},
284 static_cast<_F0&&>(__f0),
285 static_cast<_FArgs&&>(__fargs));
286 }
287
288// }}}
289// __call_with_subscripts{{{
290template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
291 _GLIBCXX_SIMD_INTRINSIC constexpr auto
292 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
293 { return __fun(__x[_First + _It]...); }
294
295template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
296 _GLIBCXX_SIMD_INTRINSIC constexpr auto
297 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
298 {
299 return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
300 make_index_sequence<_Np>(),
301 static_cast<_Fp&&>(__fun));
302 }
303
304// }}}
305
306// vvv ---- type traits ---- vvv
307// integer type aliases{{{
308using _UChar = unsigned char;
309using _SChar = signed char;
310using _UShort = unsigned short;
311using _UInt = unsigned int;
312using _ULong = unsigned long;
313using _ULLong = unsigned long long;
314using _LLong = long long;
315
316//}}}
317// __first_of_pack{{{
318template <typename _T0, typename...>
319 struct __first_of_pack
320 { using type = _T0; };
321
322template <typename... _Ts>
323 using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
324
325//}}}
326// __value_type_or_identity_t {{{
327template <typename _Tp>
328 typename _Tp::value_type
329 __value_type_or_identity_impl(int);
330
331template <typename _Tp>
332 _Tp
333 __value_type_or_identity_impl(float);
334
335template <typename _Tp>
336 using __value_type_or_identity_t
337 = decltype(__value_type_or_identity_impl<_Tp>(int()));
338
339// }}}
340// __is_vectorizable {{{
341template <typename _Tp>
342 struct __is_vectorizable : public is_arithmetic<_Tp> {};
343
344template <>
345 struct __is_vectorizable<bool> : public false_type {};
346
347template <typename _Tp>
348 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
349
350// Deduces to a vectorizable type
351template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
352 using _Vectorizable = _Tp;
353
354// }}}
355// _LoadStorePtr / __is_possible_loadstore_conversion {{{
356template <typename _Ptr, typename _ValueType>
357 struct __is_possible_loadstore_conversion
358 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
359
360template <>
361 struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
362
363// Deduces to a type allowed for load/store with the given value type.
364template <typename _Ptr, typename _ValueType,
365 typename = enable_if_t<
366 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
367 using _LoadStorePtr = _Ptr;
368
369// }}}
370// __is_bitmask{{{
371template <typename _Tp, typename = void_t<>>
372 struct __is_bitmask : false_type {};
373
374template <typename _Tp>
375 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
376
377// the __mmaskXX case:
378template <typename _Tp>
379 struct __is_bitmask<_Tp,
380 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
381 : true_type {};
382
383// }}}
384// __int_for_sizeof{{{
385#pragma GCC diagnostic push
386#pragma GCC diagnostic ignored "-Wpedantic"
387template <size_t _Bytes>
388 constexpr auto
389 __int_for_sizeof()
390 {
391 if constexpr (_Bytes == sizeof(int))
392 return int();
393 #ifdef __clang__
394 else if constexpr (_Bytes == sizeof(char))
395 return char();
396 #else
397 else if constexpr (_Bytes == sizeof(_SChar))
398 return _SChar();
399 #endif
400 else if constexpr (_Bytes == sizeof(short))
401 return short();
402 #ifndef __clang__
403 else if constexpr (_Bytes == sizeof(long))
404 return long();
405 #endif
406 else if constexpr (_Bytes == sizeof(_LLong))
407 return _LLong();
408 #ifdef __SIZEOF_INT128__
409 else if constexpr (_Bytes == sizeof(__int128))
410 return __int128();
411 #endif // __SIZEOF_INT128__
412 else if constexpr (_Bytes % sizeof(int) == 0)
413 {
414 constexpr size_t _Np = _Bytes / sizeof(int);
415 struct _Ip
416 {
417 int _M_data[_Np];
418
419 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
420 operator&(_Ip __rhs) const
421 {
422 return __generate_from_n_evaluations<_Np, _Ip>(
423 [&](auto __i) { return __rhs._M_data[__i] & _M_data[__i]; });
424 }
425
426 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
427 operator|(_Ip __rhs) const
428 {
429 return __generate_from_n_evaluations<_Np, _Ip>(
430 [&](auto __i) { return __rhs._M_data[__i] | _M_data[__i]; });
431 }
432
433 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
434 operator^(_Ip __rhs) const
435 {
436 return __generate_from_n_evaluations<_Np, _Ip>(
437 [&](auto __i) { return __rhs._M_data[__i] ^ _M_data[__i]; });
438 }
439
440 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
441 operator~() const
442 {
443 return __generate_from_n_evaluations<_Np, _Ip>(
444 [&](auto __i) { return ~_M_data[__i]; });
445 }
446 };
447 return _Ip{};
448 }
449 else
450 static_assert(_Bytes != _Bytes, "this should be unreachable");
451 }
452#pragma GCC diagnostic pop
453
454template <typename _Tp>
455 using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
456
457template <size_t _Np>
458 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
459
460// }}}
461// __is_fixed_size_abi{{{
462template <typename _Tp>
463 struct __is_fixed_size_abi : false_type {};
464
465template <int _Np>
466 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
467
468template <typename _Tp>
469 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
470
471// }}}
472// constexpr feature detection{{{
473constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
474constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
475constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
476constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
477constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
478constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
479constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
480constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
481constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
482constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
483constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
484constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
485constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
486constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
487constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
488constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
489constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
490constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
491constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
492constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
493constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
494constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
495constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
496constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
497
498constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
499constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
500constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
501constexpr inline bool __support_neon_float =
502#if defined __GCC_IEC_559
503 __GCC_IEC_559 == 0;
504#elif defined __FAST_MATH__
505 true;
506#else
507 false;
508#endif
509
510#ifdef _ARCH_PWR10
511constexpr inline bool __have_power10vec = true;
512#else
513constexpr inline bool __have_power10vec = false;
514#endif
515#ifdef __POWER9_VECTOR__
516constexpr inline bool __have_power9vec = true;
517#else
518constexpr inline bool __have_power9vec = false;
519#endif
520#if defined __POWER8_VECTOR__
521constexpr inline bool __have_power8vec = true;
522#else
523constexpr inline bool __have_power8vec = __have_power9vec;
524#endif
525#if defined __VSX__
526constexpr inline bool __have_power_vsx = true;
527#else
528constexpr inline bool __have_power_vsx = __have_power8vec;
529#endif
530#if defined __ALTIVEC__
531constexpr inline bool __have_power_vmx = true;
532#else
533constexpr inline bool __have_power_vmx = __have_power_vsx;
534#endif
535
536// }}}
537// __is_scalar_abi {{{
538template <typename _Abi>
539 constexpr bool
540 __is_scalar_abi()
541 { return is_same_v<simd_abi::scalar, _Abi>; }
542
543// }}}
544// __abi_bytes_v {{{
545template <template <int> class _Abi, int _Bytes>
546 constexpr int
547 __abi_bytes_impl(_Abi<_Bytes>*)
548 { return _Bytes; }
549
550template <typename _Tp>
551 constexpr int
552 __abi_bytes_impl(_Tp*)
553 { return -1; }
554
555template <typename _Abi>
556 inline constexpr int __abi_bytes_v
557 = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
558
559// }}}
560// __is_builtin_bitmask_abi {{{
561template <typename _Abi>
562 constexpr bool
563 __is_builtin_bitmask_abi()
564 { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
565
566// }}}
567// __is_sse_abi {{{
568template <typename _Abi>
569 constexpr bool
570 __is_sse_abi()
571 {
572 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
573 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
574 }
575
576// }}}
577// __is_avx_abi {{{
578template <typename _Abi>
579 constexpr bool
580 __is_avx_abi()
581 {
582 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
583 return _Bytes > 16 && _Bytes <= 32
584 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
585 }
586
587// }}}
588// __is_avx512_abi {{{
589template <typename _Abi>
590 constexpr bool
591 __is_avx512_abi()
592 {
593 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
594 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
595 }
596
597// }}}
598// __is_neon_abi {{{
599template <typename _Abi>
600 constexpr bool
601 __is_neon_abi()
602 {
603 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
604 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
605 }
606
607// }}}
608// __make_dependent_t {{{
609template <typename, typename _Up>
610 struct __make_dependent
611 { using type = _Up; };
612
613template <typename _Tp, typename _Up>
614 using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
615
616// }}}
617// ^^^ ---- type traits ---- ^^^
618
619// __invoke_ub{{{
620template <typename... _Args>
621 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
622 __invoke_ub([[maybe_unused]] const char* __msg,
623 [[maybe_unused]] const _Args&... __args)
624 {
625#ifdef _GLIBCXX_DEBUG_UB
626 __builtin_fprintf(stderr, __msg, __args...);
627 __builtin_trap();
628#else
629 __builtin_unreachable();
630#endif
631 }
632
633// }}}
634// __assert_unreachable{{{
635template <typename _Tp>
636 struct __assert_unreachable
637 { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
638
639// }}}
640// __size_or_zero_v {{{
641template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
642 constexpr size_t
643 __size_or_zero_dispatch(int)
644 { return _Np; }
645
646template <typename _Tp, typename _Ap>
647 constexpr size_t
648 __size_or_zero_dispatch(float)
649 { return 0; }
650
651template <typename _Tp, typename _Ap>
652 inline constexpr size_t __size_or_zero_v
653 = __size_or_zero_dispatch<_Tp, _Ap>(0);
654
655// }}}
656// __div_roundup {{{
657inline constexpr size_t
658__div_roundup(size_t __a, size_t __b)
659{ return (__a + __b - 1) / __b; }
660
661// }}}
662// _ExactBool{{{
663class _ExactBool
664{
665 const bool _M_data;
666
667public:
668 _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(bool __b) : _M_data(__b) {}
669
670 _ExactBool(int) = delete;
671
672 _GLIBCXX_SIMD_INTRINSIC constexpr operator bool() const { return _M_data; }
673};
674
675// }}}
676// __may_alias{{{
677/**@internal
678 * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
679 * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
680 * that support it).
681 */
682template <typename _Tp>
683 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
684
685// }}}
686// _UnsupportedBase {{{
687// simd and simd_mask base for unsupported <_Tp, _Abi>
688struct _UnsupportedBase
689{
690 _UnsupportedBase() = delete;
691 _UnsupportedBase(const _UnsupportedBase&) = delete;
692 _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
693 ~_UnsupportedBase() = delete;
694};
695
696// }}}
697// _InvalidTraits {{{
698/**
699 * @internal
700 * Defines the implementation of __a given <_Tp, _Abi>.
701 *
702 * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
703 * possible. Static assertions in the type definition do not suffice. It is
704 * important that SFINAE works.
705 */
706struct _InvalidTraits
707{
708 using _IsValid = false_type;
709 using _SimdBase = _UnsupportedBase;
710 using _MaskBase = _UnsupportedBase;
711
712 static constexpr size_t _S_full_size = 0;
713 static constexpr bool _S_is_partial = false;
714
715 static constexpr size_t _S_simd_align = 1;
716 struct _SimdImpl;
717 struct _SimdMember {};
718 struct _SimdCastType;
719
720 static constexpr size_t _S_mask_align = 1;
721 struct _MaskImpl;
722 struct _MaskMember {};
723 struct _MaskCastType;
724};
725
726// }}}
727// _SimdTraits {{{
728template <typename _Tp, typename _Abi, typename = void_t<>>
729 struct _SimdTraits : _InvalidTraits {};
730
731// }}}
732// __private_init, __bitset_init{{{
733/**
734 * @internal
735 * Tag used for private init constructor of simd and simd_mask
736 */
737inline constexpr struct _PrivateInit {} __private_init = {};
738
739inline constexpr struct _BitsetInit {} __bitset_init = {};
740
741// }}}
742// __is_narrowing_conversion<_From, _To>{{{
743template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
744 bool = is_arithmetic_v<_To>>
745 struct __is_narrowing_conversion;
746
747// ignore "signed/unsigned mismatch" in the following trait.
748// The implicit conversions will do the right thing here.
749template <typename _From, typename _To>
750 struct __is_narrowing_conversion<_From, _To, true, true>
751 : public __bool_constant<(
752 __digits_v<_From> > __digits_v<_To>
753 || __finite_max_v<_From> > __finite_max_v<_To>
754 || __finite_min_v<_From> < __finite_min_v<_To>
755 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
756
757template <typename _Tp>
758 struct __is_narrowing_conversion<_Tp, bool, true, true>
759 : public true_type {};
760
761template <>
762 struct __is_narrowing_conversion<bool, bool, true, true>
763 : public false_type {};
764
765template <typename _Tp>
766 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
767 : public false_type {};
768
769template <typename _From, typename _To>
770 struct __is_narrowing_conversion<_From, _To, false, true>
771 : public negation<is_convertible<_From, _To>> {};
772
773// }}}
774// __converts_to_higher_integer_rank{{{
775template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
776 struct __converts_to_higher_integer_rank : public true_type {};
777
778// this may fail for char -> short if sizeof(char) == sizeof(short)
779template <typename _From, typename _To>
780 struct __converts_to_higher_integer_rank<_From, _To, false>
781 : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
782
783// }}}
784// __data(simd/simd_mask) {{{
785template <typename _Tp, typename _Ap>
786 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
787 __data(const simd<_Tp, _Ap>& __x);
788
789template <typename _Tp, typename _Ap>
790 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
791 __data(simd<_Tp, _Ap>& __x);
792
793template <typename _Tp, typename _Ap>
794 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
795 __data(const simd_mask<_Tp, _Ap>& __x);
796
797template <typename _Tp, typename _Ap>
798 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
799 __data(simd_mask<_Tp, _Ap>& __x);
800
801// }}}
802// _SimdConverter {{{
803template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
804 typename = void>
805 struct _SimdConverter;
806
807template <typename _Tp, typename _Ap>
808 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
809 {
810 template <typename _Up>
811 _GLIBCXX_SIMD_INTRINSIC const _Up&
812 operator()(const _Up& __x)
813 { return __x; }
814 };
815
816// }}}
817// __to_value_type_or_member_type {{{
818template <typename _V>
819 _GLIBCXX_SIMD_INTRINSIC constexpr auto
820 __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
821 { return __data(__x); }
822
823template <typename _V>
824 _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
825 __to_value_type_or_member_type(const typename _V::value_type& __x)
826 { return __x; }
827
828// }}}
829// __bool_storage_member_type{{{
830template <size_t _Size>
831 struct __bool_storage_member_type;
832
833template <size_t _Size>
834 using __bool_storage_member_type_t =
835 typename __bool_storage_member_type<_Size>::type;
836
837// }}}
838// _SimdTuple {{{
839// why not tuple?
840// 1. tuple gives no guarantee about the storage order, but I require
841// storage
842// equivalent to array<_Tp, _Np>
843// 2. direct access to the element type (first template argument)
844// 3. enforces equal element type, only different _Abi types are allowed
845template <typename _Tp, typename... _Abis>
846 struct _SimdTuple;
847
848//}}}
849// __fixed_size_storage_t {{{
850template <typename _Tp, int _Np>
851 struct __fixed_size_storage;
852
853template <typename _Tp, int _Np>
854 using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
855
856// }}}
857// _SimdWrapper fwd decl{{{
858template <typename _Tp, size_t _Size, typename = void_t<>>
859 struct _SimdWrapper;
860
861template <typename _Tp>
862 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
863template <typename _Tp>
864 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
865template <typename _Tp>
866 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
867template <typename _Tp>
868 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
869
870// }}}
871// __is_simd_wrapper {{{
872template <typename _Tp>
873 struct __is_simd_wrapper : false_type {};
874
875template <typename _Tp, size_t _Np>
876 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
877
878template <typename _Tp>
879 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
880
881// }}}
882// _BitOps {{{
883struct _BitOps
884{
885 // _S_bit_iteration {{{
886 template <typename _Tp, typename _Fp>
887 static void
888 _S_bit_iteration(_Tp __mask, _Fp&& __f)
889 {
890 static_assert(sizeof(_ULLong) >= sizeof(_Tp));
891 conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
892 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
893 __k = __mask;
894 else
895 __k = __mask.to_ullong();
896 while(__k)
897 {
898 __f(std::__countr_zero(__k));
899 __k &= (__k - 1);
900 }
901 }
902
903 //}}}
904};
905
906//}}}
907// __increment, __decrement {{{
908template <typename _Tp = void>
909 struct __increment
910 { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
911
912template <>
913 struct __increment<void>
914 {
915 template <typename _Tp>
916 constexpr _Tp
917 operator()(_Tp __a) const
918 { return ++__a; }
919 };
920
921template <typename _Tp = void>
922 struct __decrement
923 { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
924
925template <>
926 struct __decrement<void>
927 {
928 template <typename _Tp>
929 constexpr _Tp
930 operator()(_Tp __a) const
931 { return --__a; }
932 };
933
934// }}}
935// _ValuePreserving(OrInt) {{{
936template <typename _From, typename _To,
937 typename = enable_if_t<negation<
938 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
939 using _ValuePreserving = _From;
940
941template <typename _From, typename _To,
942 typename _DecayedFrom = __remove_cvref_t<_From>,
943 typename = enable_if_t<conjunction<
944 is_convertible<_From, _To>,
945 disjunction<
946 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
947 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
948 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
949 using _ValuePreservingOrInt = _From;
950
951// }}}
952// __intrinsic_type {{{
953template <typename _Tp, size_t _Bytes, typename = void_t<>>
954 struct __intrinsic_type;
955
956template <typename _Tp, size_t _Size>
957 using __intrinsic_type_t =
958 typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
959
960template <typename _Tp>
961 using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
962template <typename _Tp>
963 using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
964template <typename _Tp>
965 using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
966template <typename _Tp>
967 using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
968template <typename _Tp>
969 using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
970template <typename _Tp>
971 using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
972
973// }}}
974// _BitMask {{{
975template <size_t _Np, bool _Sanitized = false>
976 struct _BitMask;
977
978template <size_t _Np, bool _Sanitized>
979 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
980
981template <size_t _Np>
982 using _SanitizedBitMask = _BitMask<_Np, true>;
983
984template <size_t _Np, bool _Sanitized>
985 struct _BitMask
986 {
987 static_assert(_Np > 0);
988
989 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
990
991 using _Tp = conditional_t<_Np == 1, bool,
992 make_unsigned_t<__int_with_sizeof_t<std::min(
993 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
994
995 static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
996
997 _Tp _M_bits[_S_array_size];
998
999 static constexpr int _S_unused_bits
1000 = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1001
1002 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1003
1004 constexpr _BitMask() noexcept = default;
1005
1006 constexpr _BitMask(unsigned long long __x) noexcept
1007 : _M_bits{static_cast<_Tp>(__x)} {}
1008
1009 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1010
1011 constexpr _BitMask(const _BitMask&) noexcept = default;
1012
1013 template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1014 && _Sanitized == true>>
1015 constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1016 : _BitMask(__rhs._M_sanitized()) {}
1017
1018 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1019 {
1020 static_assert(_S_array_size == 1);
1021 return _M_bits[0];
1022 }
1023
1024 // precondition: is sanitized
1025 constexpr _Tp
1026 _M_to_bits() const noexcept
1027 {
1028 static_assert(_S_array_size == 1);
1029 return _M_bits[0];
1030 }
1031
1032 // precondition: is sanitized
1033 constexpr unsigned long long
1034 to_ullong() const noexcept
1035 {
1036 static_assert(_S_array_size == 1);
1037 return _M_bits[0];
1038 }
1039
1040 // precondition: is sanitized
1041 constexpr unsigned long
1042 to_ulong() const noexcept
1043 {
1044 static_assert(_S_array_size == 1);
1045 return _M_bits[0];
1046 }
1047
1048 constexpr bitset<_Np>
1049 _M_to_bitset() const noexcept
1050 {
1051 static_assert(_S_array_size == 1);
1052 return _M_bits[0];
1053 }
1054
1055 constexpr decltype(auto)
1056 _M_sanitized() const noexcept
1057 {
1058 if constexpr (_Sanitized)
1059 return *this;
1060 else if constexpr (_Np == 1)
1061 return _SanitizedBitMask<_Np>(_M_bits[0]);
1062 else
1063 {
1064 _SanitizedBitMask<_Np> __r = {};
1065 for (int __i = 0; __i < _S_array_size; ++__i)
1066 __r._M_bits[__i] = _M_bits[__i];
1067 if constexpr (_S_unused_bits > 0)
1068 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1069 return __r;
1070 }
1071 }
1072
1073 template <size_t _Mp, bool _LSanitized>
1074 constexpr _BitMask<_Np + _Mp, _Sanitized>
1075 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1076 {
1077 constexpr size_t _RN = _Np + _Mp;
1078 using _Rp = _BitMask<_RN, _Sanitized>;
1079 if constexpr (_Rp::_S_array_size == 1)
1080 {
1081 _Rp __r{{_M_bits[0]}};
1082 __r._M_bits[0] <<= _Mp;
1083 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1084 return __r;
1085 }
1086 else
1087 __assert_unreachable<_Rp>();
1088 }
1089
1090 // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1091 // significant bits. If the operation implicitly produces a sanitized bitmask,
1092 // the result type will have _Sanitized set.
1093 template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1094 constexpr auto
1095 _M_extract() const noexcept
1096 {
1097 static_assert(_Np > _DropLsb);
1098 static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1099 "not implemented for bitmasks larger than one ullong");
1100 if constexpr (_NewSize == 1)
1101 // must sanitize because the return _Tp is bool
1102 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1103 else
1104 return _BitMask<_NewSize,
1105 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1106 && _NewSize + _DropLsb <= _Np)
1107 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1108 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1109 >> _DropLsb);
1110 }
1111
1112 // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1113 constexpr bool
1114 all() const noexcept
1115 {
1116 if constexpr (_Np == 1)
1117 return _M_bits[0];
1118 else if constexpr (!_Sanitized)
1119 return _M_sanitized().all();
1120 else
1121 {
1122 constexpr _Tp __allbits = ~_Tp();
1123 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1124 if (_M_bits[__i] != __allbits)
1125 return false;
1126 return _M_bits[_S_array_size - 1] == _S_bitmask;
1127 }
1128 }
1129
1130 // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1131 // false.
1132 constexpr bool
1133 any() const noexcept
1134 {
1135 if constexpr (_Np == 1)
1136 return _M_bits[0];
1137 else if constexpr (!_Sanitized)
1138 return _M_sanitized().any();
1139 else
1140 {
1141 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1142 if (_M_bits[__i] != 0)
1143 return true;
1144 return _M_bits[_S_array_size - 1] != 0;
1145 }
1146 }
1147
1148 // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1149 constexpr bool
1150 none() const noexcept
1151 {
1152 if constexpr (_Np == 1)
1153 return !_M_bits[0];
1154 else if constexpr (!_Sanitized)
1155 return _M_sanitized().none();
1156 else
1157 {
1158 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1159 if (_M_bits[__i] != 0)
1160 return false;
1161 return _M_bits[_S_array_size - 1] == 0;
1162 }
1163 }
1164
1165 // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1166 // false.
1167 constexpr int
1168 count() const noexcept
1169 {
1170 if constexpr (_Np == 1)
1171 return _M_bits[0];
1172 else if constexpr (!_Sanitized)
1173 return _M_sanitized().none();
1174 else
1175 {
1176 int __result = __builtin_popcountll(_M_bits[0]);
1177 for (int __i = 1; __i < _S_array_size; ++__i)
1178 __result += __builtin_popcountll(_M_bits[__i]);
1179 return __result;
1180 }
1181 }
1182
1183 // Returns the bit at offset __i as bool.
1184 constexpr bool
1185 operator[](size_t __i) const noexcept
1186 {
1187 if constexpr (_Np == 1)
1188 return _M_bits[0];
1189 else if constexpr (_S_array_size == 1)
1190 return (_M_bits[0] >> __i) & 1;
1191 else
1192 {
1193 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1194 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1195 return (_M_bits[__j] >> __shift) & 1;
1196 }
1197 }
1198
1199 template <size_t __i>
1200 constexpr bool
1201 operator[](_SizeConstant<__i>) const noexcept
1202 {
1203 static_assert(__i < _Np);
1204 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1205 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1206 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1207 }
1208
1209 // Set the bit at offset __i to __x.
1210 constexpr void
1211 set(size_t __i, bool __x) noexcept
1212 {
1213 if constexpr (_Np == 1)
1214 _M_bits[0] = __x;
1215 else if constexpr (_S_array_size == 1)
1216 {
1217 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1218 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1219 }
1220 else
1221 {
1222 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1223 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1224 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1225 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1226 }
1227 }
1228
1229 template <size_t __i>
1230 constexpr void
1231 set(_SizeConstant<__i>, bool __x) noexcept
1232 {
1233 static_assert(__i < _Np);
1234 if constexpr (_Np == 1)
1235 _M_bits[0] = __x;
1236 else
1237 {
1238 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1239 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1240 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1241 _M_bits[__j] &= __mask;
1242 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1243 }
1244 }
1245
1246 // Inverts all bits. Sanitized input leads to sanitized output.
1247 constexpr _BitMask
1248 operator~() const noexcept
1249 {
1250 if constexpr (_Np == 1)
1251 return !_M_bits[0];
1252 else
1253 {
1254 _BitMask __result{};
1255 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1256 __result._M_bits[__i] = ~_M_bits[__i];
1257 if constexpr (_Sanitized)
1258 __result._M_bits[_S_array_size - 1]
1259 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1260 else
1261 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1262 return __result;
1263 }
1264 }
1265
1266 constexpr _BitMask&
1267 operator^=(const _BitMask& __b) & noexcept
1268 {
1269 __execute_n_times<_S_array_size>(
1270 [&](auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1271 return *this;
1272 }
1273
1274 constexpr _BitMask&
1275 operator|=(const _BitMask& __b) & noexcept
1276 {
1277 __execute_n_times<_S_array_size>(
1278 [&](auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1279 return *this;
1280 }
1281
1282 constexpr _BitMask&
1283 operator&=(const _BitMask& __b) & noexcept
1284 {
1285 __execute_n_times<_S_array_size>(
1286 [&](auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1287 return *this;
1288 }
1289
1290 friend constexpr _BitMask
1291 operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1292 {
1293 _BitMask __r = __a;
1294 __r ^= __b;
1295 return __r;
1296 }
1297
1298 friend constexpr _BitMask
1299 operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1300 {
1301 _BitMask __r = __a;
1302 __r |= __b;
1303 return __r;
1304 }
1305
1306 friend constexpr _BitMask
1307 operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1308 {
1309 _BitMask __r = __a;
1310 __r &= __b;
1311 return __r;
1312 }
1313
1314 _GLIBCXX_SIMD_INTRINSIC
1315 constexpr bool
1316 _M_is_constprop() const
1317 {
1318 if constexpr (_S_array_size == 0)
1319 return __builtin_constant_p(_M_bits[0]);
1320 else
1321 {
1322 for (int __i = 0; __i < _S_array_size; ++__i)
1323 if (!__builtin_constant_p(_M_bits[__i]))
1324 return false;
1325 return true;
1326 }
1327 }
1328 };
1329
1330// }}}
1331
1332// vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1333// __min_vector_size {{{
1334template <typename _Tp = void>
1335 static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1336
1337#if _GLIBCXX_SIMD_HAVE_NEON
1338template <>
1339 inline constexpr int __min_vector_size<void> = 8;
1340#else
1341template <>
1342 inline constexpr int __min_vector_size<void> = 16;
1343#endif
1344
1345// }}}
1346// __vector_type {{{
1347template <typename _Tp, size_t _Np, typename = void>
1348 struct __vector_type_n {};
1349
1350// substition failure for 0-element case
1351template <typename _Tp>
1352 struct __vector_type_n<_Tp, 0, void> {};
1353
1354// special case 1-element to be _Tp itself
1355template <typename _Tp>
1356 struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1357 { using type = _Tp; };
1358
1359// else, use GNU-style builtin vector types
1360template <typename _Tp, size_t _Np>
1361 struct __vector_type_n<_Tp, _Np,
1362 enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1363 {
1364 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1365
1366 static constexpr size_t _S_Bytes =
1367#ifdef __i386__
1368 // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1369 // those objects are passed via MMX registers and nothing ever calls EMMS.
1370 _S_Np2 == 8 ? 16 :
1371#endif
1372 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1373 : _S_Np2;
1374
1375 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1376 };
1377
1378template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1379 struct __vector_type;
1380
1381template <typename _Tp, size_t _Bytes>
1382 struct __vector_type<_Tp, _Bytes, 0>
1383 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1384
1385template <typename _Tp, size_t _Size>
1386 using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1387
1388template <typename _Tp>
1389 using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1390template <typename _Tp>
1391 using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1392template <typename _Tp>
1393 using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1394template <typename _Tp>
1395 using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1396template <typename _Tp>
1397 using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1398template <typename _Tp>
1399 using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1400
1401// }}}
1402// __is_vector_type {{{
1403template <typename _Tp, typename = void_t<>>
1404 struct __is_vector_type : false_type {};
1405
1406template <typename _Tp>
1407 struct __is_vector_type<
1408 _Tp, void_t<typename __vector_type<
1409 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1410 : is_same<_Tp, typename __vector_type<
1411 remove_reference_t<decltype(declval<_Tp>()[0])>,
1412 sizeof(_Tp)>::type> {};
1413
1414template <typename _Tp>
1415 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1416
1417// }}}
1418// __is_intrinsic_type {{{
1419#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1420template <typename _Tp>
1421 using __is_intrinsic_type = __is_vector_type<_Tp>;
1422#else // not SSE (x86)
1423template <typename _Tp, typename = void_t<>>
1424 struct __is_intrinsic_type : false_type {};
1425
1426template <typename _Tp>
1427 struct __is_intrinsic_type<
1428 _Tp, void_t<typename __intrinsic_type<
1429 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1430 : is_same<_Tp, typename __intrinsic_type<
1431 remove_reference_t<decltype(declval<_Tp>()[0])>,
1432 sizeof(_Tp)>::type> {};
1433#endif
1434
1435template <typename _Tp>
1436 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1437
1438// }}}
1439// _VectorTraits{{{
1440template <typename _Tp, typename = void_t<>>
1441 struct _VectorTraitsImpl;
1442
1443template <typename _Tp>
1444 struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1445 || __is_intrinsic_type_v<_Tp>>>
1446 {
1447 using type = _Tp;
1448 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1449 static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1450 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1451 template <typename _Up, int _W = _S_full_size>
1452 static constexpr bool _S_is
1453 = is_same_v<value_type, _Up> && _W == _S_full_size;
1454 };
1455
1456template <typename _Tp, size_t _Np>
1457 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1458 void_t<__vector_type_t<_Tp, _Np>>>
1459 {
1460 using type = __vector_type_t<_Tp, _Np>;
1461 using value_type = _Tp;
1462 static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1463 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1464 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1465 static constexpr int _S_partial_width = _Np;
1466 template <typename _Up, int _W = _S_full_size>
1467 static constexpr bool _S_is
1468 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1469 };
1470
1471template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1472 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1473
1474// }}}
1475// __as_vector{{{
1476template <typename _V>
1477 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1478 __as_vector(_V __x)
1479 {
1480 if constexpr (__is_vector_type_v<_V>)
1481 return __x;
1482 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1483 return __data(__x)._M_data;
1484 else if constexpr (__is_vectorizable_v<_V>)
1485 return __vector_type_t<_V, 2>{__x};
1486 else
1487 return __x._M_data;
1488 }
1489
1490// }}}
1491// __as_wrapper{{{
1492template <size_t _Np = 0, typename _V>
1493 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1494 __as_wrapper(_V __x)
1495 {
1496 if constexpr (__is_vector_type_v<_V>)
1497 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1498 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1499 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1500 {
1501 static_assert(_V::size() == _Np);
1502 return __data(__x);
1503 }
1504 else
1505 {
1506 static_assert(_V::_S_size == _Np);
1507 return __x;
1508 }
1509 }
1510
1511// }}}
1512// __intrin_bitcast{{{
1513template <typename _To, typename _From>
1514 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1515 __intrin_bitcast(_From __v)
1516 {
1517 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1518 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1519 if constexpr (sizeof(_To) == sizeof(_From))
1520 return reinterpret_cast<_To>(__v);
1521 else if constexpr (sizeof(_From) > sizeof(_To))
1522 if constexpr (sizeof(_To) >= 16)
1523 return reinterpret_cast<const __may_alias<_To>&>(__v);
1524 else
1525 {
1526 _To __r;
1527 __builtin_memcpy(&__r, &__v, sizeof(_To));
1528 return __r;
1529 }
1530#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1531 else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1532 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1533 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1534 else if constexpr (__have_avx512f && sizeof(_From) == 16
1535 && sizeof(_To) == 64)
1536 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1537 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1538 else if constexpr (__have_avx512f && sizeof(_From) == 32
1539 && sizeof(_To) == 64)
1540 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1541 reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1542#endif // _GLIBCXX_SIMD_X86INTRIN
1543 else if constexpr (sizeof(__v) <= 8)
1544 return reinterpret_cast<_To>(
1545 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1546 reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1547 else
1548 {
1549 static_assert(sizeof(_To) > sizeof(_From));
1550 _To __r = {};
1551 __builtin_memcpy(&__r, &__v, sizeof(_From));
1552 return __r;
1553 }
1554 }
1555
1556// }}}
1557// __vector_bitcast{{{
1558template <typename _To, size_t _NN = 0, typename _From,
1559 typename _FromVT = _VectorTraits<_From>,
1560 size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1561 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1562 __vector_bitcast(_From __x)
1563 {
1564 using _R = __vector_type_t<_To, _Np>;
1565 return __intrin_bitcast<_R>(__x);
1566 }
1567
1568template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1569 size_t _Np
1570 = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1571 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1572 __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1573 {
1574 static_assert(_Np > 1);
1575 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1576 }
1577
1578// }}}
1579// __convert_x86 declarations {{{
1580#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1581template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1582 _To __convert_x86(_Tp);
1583
1584template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1585 _To __convert_x86(_Tp, _Tp);
1586
1587template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1588 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1589
1590template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1591 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1592
1593template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1594 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1595 _Tp, _Tp, _Tp, _Tp);
1596#endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1597
1598//}}}
1599// __bit_cast {{{
1600template <typename _To, typename _From>
1601 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1602 __bit_cast(const _From __x)
1603 {
1604 // TODO: implement with / replace by __builtin_bit_cast ASAP
1605 static_assert(sizeof(_To) == sizeof(_From));
1606 constexpr bool __to_is_vectorizable
1607 = is_arithmetic_v<_To> || is_enum_v<_To>;
1608 constexpr bool __from_is_vectorizable
1609 = is_arithmetic_v<_From> || is_enum_v<_From>;
1610 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1611 return reinterpret_cast<_To>(__x);
1612 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1613 {
1614 using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1615 return reinterpret_cast<_To>(_FV{__x});
1616 }
1617 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1618 {
1619 using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1620 using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1621 return reinterpret_cast<_TV>(_FV{__x})[0];
1622 }
1623 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1624 {
1625 using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1626 return reinterpret_cast<_TV>(__x)[0];
1627 }
1628 else
1629 {
1630 _To __r;
1631 __builtin_memcpy(reinterpret_cast<char*>(&__r),
1632 reinterpret_cast<const char*>(&__x), sizeof(_To));
1633 return __r;
1634 }
1635 }
1636
1637// }}}
1638// __to_intrin {{{
1639template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1640 typename _R
1641 = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1642 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1643 __to_intrin(_Tp __x)
1644 {
1645 static_assert(sizeof(__x) <= sizeof(_R),
1646 "__to_intrin may never drop values off the end");
1647 if constexpr (sizeof(__x) == sizeof(_R))
1648 return reinterpret_cast<_R>(__as_vector(__x));
1649 else
1650 {
1651 using _Up = __int_for_sizeof_t<_Tp>;
1652 return reinterpret_cast<_R>(
1653 __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1654 }
1655 }
1656
1657// }}}
1658// __make_vector{{{
1659template <typename _Tp, typename... _Args>
1660 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1661 __make_vector(const _Args&... __args)
1662 {
1663 return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...};
1664 }
1665
1666// }}}
1667// __vector_broadcast{{{
1668template <size_t _Np, typename _Tp>
1669 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1670 __vector_broadcast(_Tp __x)
1671 {
1672 return __call_with_n_evaluations<_Np>(
1673 [](auto... __xx) { return __vector_type_t<_Tp, _Np>{__xx...}; },
1674 [&__x](int) { return __x; });
1675 }
1676
1677// }}}
1678// __generate_vector{{{
1679 template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1680 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1681 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1682 {
1683 return __vector_type_t<_Tp, _Np>{
1684 static_cast<_Tp>(__gen(_SizeConstant<_I>()))...};
1685 }
1686
1687template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1688 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1689 __generate_vector(_Gp&& __gen)
1690 {
1691 if constexpr (__is_vector_type_v<_V>)
1692 return __generate_vector_impl<typename _VVT::value_type,
1693 _VVT::_S_full_size>(
1694 static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1695 else
1696 return __generate_vector_impl<typename _VVT::value_type,
1697 _VVT::_S_partial_width>(
1698 static_cast<_Gp&&>(__gen),
1699 make_index_sequence<_VVT::_S_partial_width>());
1700 }
1701
1702template <typename _Tp, size_t _Np, typename _Gp>
1703 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1704 __generate_vector(_Gp&& __gen)
1705 {
1706 return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1707 make_index_sequence<_Np>());
1708 }
1709
1710// }}}
1711// __xor{{{
1712template <typename _TW>
1713 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1714 __xor(_TW __a, _TW __b) noexcept
1715 {
1716 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1717 {
1718 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1719 _VectorTraitsImpl<_TW>>::value_type;
1720 if constexpr (is_floating_point_v<_Tp>)
1721 {
1722 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1723 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1724 ^ __vector_bitcast<_Ip>(__b));
1725 }
1726 else if constexpr (__is_vector_type_v<_TW>)
1727 return __a ^ __b;
1728 else
1729 return __a._M_data ^ __b._M_data;
1730 }
1731 else
1732 return __a ^ __b;
1733 }
1734
1735// }}}
1736// __or{{{
1737template <typename _TW>
1738 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1739 __or(_TW __a, _TW __b) noexcept
1740 {
1741 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1742 {
1743 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1744 _VectorTraitsImpl<_TW>>::value_type;
1745 if constexpr (is_floating_point_v<_Tp>)
1746 {
1747 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1748 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1749 | __vector_bitcast<_Ip>(__b));
1750 }
1751 else if constexpr (__is_vector_type_v<_TW>)
1752 return __a | __b;
1753 else
1754 return __a._M_data | __b._M_data;
1755 }
1756 else
1757 return __a | __b;
1758 }
1759
1760// }}}
1761// __and{{{
1762template <typename _TW>
1763 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1764 __and(_TW __a, _TW __b) noexcept
1765 {
1766 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1767 {
1768 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1769 _VectorTraitsImpl<_TW>>::value_type;
1770 if constexpr (is_floating_point_v<_Tp>)
1771 {
1772 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1773 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1774 & __vector_bitcast<_Ip>(__b));
1775 }
1776 else if constexpr (__is_vector_type_v<_TW>)
1777 return __a & __b;
1778 else
1779 return __a._M_data & __b._M_data;
1780 }
1781 else
1782 return __a & __b;
1783 }
1784
1785// }}}
1786// __andnot{{{
1787#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1788static constexpr struct
1789{
1790 _GLIBCXX_SIMD_INTRINSIC __v4sf
1791 operator()(__v4sf __a, __v4sf __b) const noexcept
1792 { return __builtin_ia32_andnps(__a, __b); }
1793
1794 _GLIBCXX_SIMD_INTRINSIC __v2df
1795 operator()(__v2df __a, __v2df __b) const noexcept
1796 { return __builtin_ia32_andnpd(__a, __b); }
1797
1798 _GLIBCXX_SIMD_INTRINSIC __v2di
1799 operator()(__v2di __a, __v2di __b) const noexcept
1800 { return __builtin_ia32_pandn128(__a, __b); }
1801
1802 _GLIBCXX_SIMD_INTRINSIC __v8sf
1803 operator()(__v8sf __a, __v8sf __b) const noexcept
1804 { return __builtin_ia32_andnps256(__a, __b); }
1805
1806 _GLIBCXX_SIMD_INTRINSIC __v4df
1807 operator()(__v4df __a, __v4df __b) const noexcept
1808 { return __builtin_ia32_andnpd256(__a, __b); }
1809
1810 _GLIBCXX_SIMD_INTRINSIC __v4di
1811 operator()(__v4di __a, __v4di __b) const noexcept
1812 {
1813 if constexpr (__have_avx2)
1814 return __builtin_ia32_andnotsi256(__a, __b);
1815 else
1816 return reinterpret_cast<__v4di>(
1817 __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1818 reinterpret_cast<__v4df>(__b)));
1819 }
1820
1821 _GLIBCXX_SIMD_INTRINSIC __v16sf
1822 operator()(__v16sf __a, __v16sf __b) const noexcept
1823 {
1824 if constexpr (__have_avx512dq)
1825 return _mm512_andnot_ps(__a, __b);
1826 else
1827 return reinterpret_cast<__v16sf>(
1828 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1829 reinterpret_cast<__v8di>(__b)));
1830 }
1831
1832 _GLIBCXX_SIMD_INTRINSIC __v8df
1833 operator()(__v8df __a, __v8df __b) const noexcept
1834 {
1835 if constexpr (__have_avx512dq)
1836 return _mm512_andnot_pd(__a, __b);
1837 else
1838 return reinterpret_cast<__v8df>(
1839 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1840 reinterpret_cast<__v8di>(__b)));
1841 }
1842
1843 _GLIBCXX_SIMD_INTRINSIC __v8di
1844 operator()(__v8di __a, __v8di __b) const noexcept
1845 { return _mm512_andnot_si512(__a, __b); }
1846} _S_x86_andnot;
1847#endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1848
1849template <typename _TW>
1850 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1851 __andnot(_TW __a, _TW __b) noexcept
1852 {
1853 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1854 {
1855 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1856 _VectorTraitsImpl<_TW>>;
1857 using _Tp = typename _TVT::value_type;
1858#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1859 if constexpr (sizeof(_TW) >= 16)
1860 {
1861 const auto __ai = __to_intrin(__a);
1862 const auto __bi = __to_intrin(__b);
1863 if (!__builtin_is_constant_evaluated()
1864 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1865 {
1866 const auto __r = _S_x86_andnot(__ai, __bi);
1867 if constexpr (is_convertible_v<decltype(__r), _TW>)
1868 return __r;
1869 else
1870 return reinterpret_cast<typename _TVT::type>(__r);
1871 }
1872 }
1873#endif // _GLIBCXX_SIMD_X86INTRIN
1874 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1875 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1876 & __vector_bitcast<_Ip>(__b));
1877 }
1878 else
1879 return ~__a & __b;
1880 }
1881
1882// }}}
1883// __not{{{
1884template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1885 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1886 __not(_Tp __a) noexcept
1887 {
1888 if constexpr (is_floating_point_v<typename _TVT::value_type>)
1889 return reinterpret_cast<typename _TVT::type>(
1890 ~__vector_bitcast<unsigned>(__a));
1891 else
1892 return ~__a;
1893 }
1894
1895// }}}
1896// __concat{{{
1897template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1898 typename _R = __vector_type_t<typename _TVT::value_type,
1899 _TVT::_S_full_size * 2>>
1900 constexpr _R
1901 __concat(_Tp a_, _Tp b_)
1902 {
1903#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1904 using _W
1905 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1906 conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
1907 long long, typename _TVT::value_type>>;
1908 constexpr int input_width = sizeof(_Tp) / sizeof(_W);
1909 const auto __a = __vector_bitcast<_W>(a_);
1910 const auto __b = __vector_bitcast<_W>(b_);
1911 using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
1912#else
1913 constexpr int input_width = _TVT::_S_full_size;
1914 const _Tp& __a = a_;
1915 const _Tp& __b = b_;
1916 using _Up = _R;
1917#endif
1918 if constexpr (input_width == 2)
1919 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1920 else if constexpr (input_width == 4)
1921 return reinterpret_cast<_R>(
1922 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1923 else if constexpr (input_width == 8)
1924 return reinterpret_cast<_R>(
1925 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1926 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1927 else if constexpr (input_width == 16)
1928 return reinterpret_cast<_R>(
1929 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1930 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1931 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1932 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1933 __b[12], __b[13], __b[14], __b[15]});
1934 else if constexpr (input_width == 32)
1935 return reinterpret_cast<_R>(
1936 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1937 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1938 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1939 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1940 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1941 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1942 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1943 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1944 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1945 __b[31]});
1946 }
1947
1948// }}}
1949// __zero_extend {{{
1950template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1951 struct _ZeroExtendProxy
1952 {
1953 using value_type = typename _TVT::value_type;
1954 static constexpr size_t _Np = _TVT::_S_full_size;
1955 const _Tp __x;
1956
1957 template <typename _To, typename _ToVT = _VectorTraits<_To>,
1958 typename
1959 = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
1960 _GLIBCXX_SIMD_INTRINSIC operator _To() const
1961 {
1962 constexpr size_t _ToN = _ToVT::_S_full_size;
1963 if constexpr (_ToN == _Np)
1964 return __x;
1965 else if constexpr (_ToN == 2 * _Np)
1966 {
1967#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1968 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1969 return __vector_bitcast<value_type>(
1970 _mm256_insertf128_ps(__m256(), __x, 0));
1971 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1972 return __vector_bitcast<value_type>(
1973 _mm256_insertf128_pd(__m256d(), __x, 0));
1974 else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
1975 return __vector_bitcast<value_type>(
1976 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1977 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1978 {
1979 if constexpr (__have_avx512dq)
1980 return __vector_bitcast<value_type>(
1981 _mm512_insertf32x8(__m512(), __x, 0));
1982 else
1983 return reinterpret_cast<__m512>(
1984 _mm512_insertf64x4(__m512d(),
1985 reinterpret_cast<__m256d>(__x), 0));
1986 }
1987 else if constexpr (__have_avx512f
1988 && _TVT::template _S_is<double, 4>)
1989 return __vector_bitcast<value_type>(
1990 _mm512_insertf64x4(__m512d(), __x, 0));
1991 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
1992 return __vector_bitcast<value_type>(
1993 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1994#endif
1995 return __concat(__x, _Tp());
1996 }
1997 else if constexpr (_ToN == 4 * _Np)
1998 {
1999#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2000 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2001 {
2002 return __vector_bitcast<value_type>(
2003 _mm512_insertf64x2(__m512d(), __x, 0));
2004 }
2005 else if constexpr (__have_avx512f
2006 && is_floating_point_v<value_type>)
2007 {
2008 return __vector_bitcast<value_type>(
2009 _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2010 0));
2011 }
2012 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2013 {
2014 return __vector_bitcast<value_type>(
2015 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2016 }
2017#endif
2018 return __concat(__concat(__x, _Tp()),
2019 __vector_type_t<value_type, _Np * 2>());
2020 }
2021 else if constexpr (_ToN == 8 * _Np)
2022 return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2023 __vector_type_t<value_type, _Np * 4>());
2024 else if constexpr (_ToN == 16 * _Np)
2025 return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2026 __vector_type_t<value_type, _Np * 8>());
2027 else
2028 __assert_unreachable<_Tp>();
2029 }
2030 };
2031
2032template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2033 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2034 __zero_extend(_Tp __x)
2035 { return {__x}; }
2036
2037// }}}
2038// __extract<_Np, By>{{{
2039template <int _Offset,
2040 int _SplitBy,
2041 typename _Tp,
2042 typename _TVT = _VectorTraits<_Tp>,
2043 typename _R = __vector_type_t<typename _TVT::value_type,
2044 _TVT::_S_full_size / _SplitBy>>
2045 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2046 __extract(_Tp __in)
2047 {
2048 using value_type = typename _TVT::value_type;
2049#if _GLIBCXX_SIMD_X86INTRIN // {{{
2050 if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2051 {
2052 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2053 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2054 else if constexpr (is_floating_point_v<value_type>)
2055 return __vector_bitcast<value_type>(
2056 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2057 else
2058 return reinterpret_cast<_R>(
2059 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2060 _Offset));
2061 }
2062 else
2063#endif // _GLIBCXX_SIMD_X86INTRIN }}}
2064 {
2065#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2066 using _W = conditional_t<
2067 is_floating_point_v<value_type>, double,
2068 conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2069 static_assert(sizeof(_R) % sizeof(_W) == 0);
2070 constexpr int __return_width = sizeof(_R) / sizeof(_W);
2071 using _Up = __vector_type_t<_W, __return_width>;
2072 const auto __x = __vector_bitcast<_W>(__in);
2073#else
2074 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2075 using _Up = _R;
2076 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2077 = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2078#endif
2079 constexpr int _O = _Offset * __return_width;
2080 return __call_with_subscripts<__return_width, _O>(
2081 __x, [](auto... __entries) {
2082 return reinterpret_cast<_R>(_Up{__entries...});
2083 });
2084 }
2085 }
2086
2087// }}}
2088// __lo/__hi64[z]{{{
2089template <typename _Tp,
2090 typename _R
2091 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2092 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2093 __lo64(_Tp __x)
2094 {
2095 _R __r{};
2096 __builtin_memcpy(&__r, &__x, 8);
2097 return __r;
2098 }
2099
2100template <typename _Tp,
2101 typename _R
2102 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2103 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2104 __hi64(_Tp __x)
2105 {
2106 static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2107 _R __r{};
2108 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2109 return __r;
2110 }
2111
2112template <typename _Tp,
2113 typename _R
2114 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2115 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2116 __hi64z([[maybe_unused]] _Tp __x)
2117 {
2118 _R __r{};
2119 if constexpr (sizeof(_Tp) == 16)
2120 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2121 return __r;
2122 }
2123
2124// }}}
2125// __lo/__hi128{{{
2126template <typename _Tp>
2127 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2128 __lo128(_Tp __x)
2129 { return __extract<0, sizeof(_Tp) / 16>(__x); }
2130
2131template <typename _Tp>
2132 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2133 __hi128(_Tp __x)
2134 {
2135 static_assert(sizeof(__x) == 32);
2136 return __extract<1, 2>(__x);
2137 }
2138
2139// }}}
2140// __lo/__hi256{{{
2141template <typename _Tp>
2142 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2143 __lo256(_Tp __x)
2144 {
2145 static_assert(sizeof(__x) == 64);
2146 return __extract<0, 2>(__x);
2147 }
2148
2149template <typename _Tp>
2150 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2151 __hi256(_Tp __x)
2152 {
2153 static_assert(sizeof(__x) == 64);
2154 return __extract<1, 2>(__x);
2155 }
2156
2157// }}}
2158// __auto_bitcast{{{
2159template <typename _Tp>
2160 struct _AutoCast
2161 {
2162 static_assert(__is_vector_type_v<_Tp>);
2163
2164 const _Tp __x;
2165
2166 template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2167 _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2168 { return __intrin_bitcast<typename _UVT::type>(__x); }
2169 };
2170
2171template <typename _Tp>
2172 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2173 __auto_bitcast(const _Tp& __x)
2174 { return {__x}; }
2175
2176template <typename _Tp, size_t _Np>
2177 _GLIBCXX_SIMD_INTRINSIC constexpr
2178 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2179 __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2180 { return {__x._M_data}; }
2181
2182// }}}
2183// ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2184
2185#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2186// __bool_storage_member_type{{{
2187#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2188template <size_t _Size>
2189 struct __bool_storage_member_type
2190 {
2191 static_assert((_Size & (_Size - 1)) != 0,
2192 "This trait may only be used for non-power-of-2 sizes. "
2193 "Power-of-2 sizes must be specialized.");
2194 using type =
2195 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2196 };
2197
2198template <>
2199 struct __bool_storage_member_type<1> { using type = bool; };
2200
2201template <>
2202 struct __bool_storage_member_type<2> { using type = __mmask8; };
2203
2204template <>
2205 struct __bool_storage_member_type<4> { using type = __mmask8; };
2206
2207template <>
2208 struct __bool_storage_member_type<8> { using type = __mmask8; };
2209
2210template <>
2211 struct __bool_storage_member_type<16> { using type = __mmask16; };
2212
2213template <>
2214 struct __bool_storage_member_type<32> { using type = __mmask32; };
2215
2216template <>
2217 struct __bool_storage_member_type<64> { using type = __mmask64; };
2218#endif // _GLIBCXX_SIMD_HAVE_AVX512F
2219
2220// }}}
2221// __intrinsic_type (x86){{{
2222// the following excludes bool via __is_vectorizable
2223#if _GLIBCXX_SIMD_HAVE_SSE
2224template <typename _Tp, size_t _Bytes>
2225 struct __intrinsic_type<_Tp, _Bytes,
2226 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2227 {
2228 static_assert(!is_same_v<_Tp, long double>,
2229 "no __intrinsic_type support for long double on x86");
2230
2231 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16
2232 : _Bytes <= 32 ? 32
2233 : 64;
2234
2235 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2236 = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2237 };
2238#endif // _GLIBCXX_SIMD_HAVE_SSE
2239
2240// }}}
2241#endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2242// __intrinsic_type (ARM){{{
2243#if _GLIBCXX_SIMD_HAVE_NEON
2244template <>
2245 struct __intrinsic_type<float, 8, void>
2246 { using type = float32x2_t; };
2247
2248template <>
2249 struct __intrinsic_type<float, 16, void>
2250 { using type = float32x4_t; };
2251
2252#if _GLIBCXX_SIMD_HAVE_NEON_A64
2253template <>
2254 struct __intrinsic_type<double, 8, void>
2255 { using type = float64x1_t; };
2256
2257template <>
2258 struct __intrinsic_type<double, 16, void>
2259 { using type = float64x2_t; };
2260#endif
2261
2262#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2263template <> \
2264 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2265 _Np * _Bits / 8, void> \
2266 { using type = int##_Bits##x##_Np##_t; }; \
2267template <> \
2268 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2269 _Np * _Bits / 8, void> \
2270 { using type = uint##_Bits##x##_Np##_t; }
2271_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2272_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2273_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2274_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2275_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2276_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2277_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2278_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2279#undef _GLIBCXX_SIMD_ARM_INTRIN
2280
2281template <typename _Tp, size_t _Bytes>
2282 struct __intrinsic_type<_Tp, _Bytes,
2283 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2284 {
2285 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2286 using _Ip = __int_for_sizeof_t<_Tp>;
2287 using _Up = conditional_t<
2288 is_floating_point_v<_Tp>, _Tp,
2289 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2290 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2291 "should use explicit specialization above");
2292 using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2293 };
2294#endif // _GLIBCXX_SIMD_HAVE_NEON
2295
2296// }}}
2297// __intrinsic_type (PPC){{{
2298#ifdef __ALTIVEC__
2299template <typename _Tp>
2300 struct __intrinsic_type_impl;
2301
2302#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2303 template <> \
2304 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2305_GLIBCXX_SIMD_PPC_INTRIN(float);
2306_GLIBCXX_SIMD_PPC_INTRIN(double);
2307_GLIBCXX_SIMD_PPC_INTRIN(signed char);
2308_GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2309_GLIBCXX_SIMD_PPC_INTRIN(signed short);
2310_GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2311_GLIBCXX_SIMD_PPC_INTRIN(signed int);
2312_GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2313_GLIBCXX_SIMD_PPC_INTRIN(signed long);
2314_GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2315_GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2316_GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2317#undef _GLIBCXX_SIMD_PPC_INTRIN
2318
2319template <typename _Tp, size_t _Bytes>
2320 struct __intrinsic_type<_Tp, _Bytes,
2321 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2322 {
2323 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2324 // allow _Tp == long double with -mlong-double-64
2325 static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2326 "no __intrinsic_type support for long double on PPC");
2327#ifndef __VSX__
2328 static_assert(!is_same_v<_Tp, double>,
2329 "no __intrinsic_type support for double on PPC w/o VSX");
2330#endif
2331 using type =
2332 typename __intrinsic_type_impl<
2333 conditional_t<is_floating_point_v<_Tp>,
2334 conditional_t<_S_is_ldouble, double, _Tp>,
2335 __int_for_sizeof_t<_Tp>>>::type;
2336 };
2337#endif // __ALTIVEC__
2338
2339// }}}
2340// _SimdWrapper<bool>{{{1
2341template <size_t _Width>
2342 struct _SimdWrapper<bool, _Width,
2343 void_t<typename __bool_storage_member_type<_Width>::type>>
2344 {
2345 using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2346 using value_type = bool;
2347
2348 static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2349
2350 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2351 __as_full_vector() const { return _M_data; }
2352
2353 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2354 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2355 : _M_data(__k) {};
2356
2357 _GLIBCXX_SIMD_INTRINSIC operator const _BuiltinType&() const
2358 { return _M_data; }
2359
2360 _GLIBCXX_SIMD_INTRINSIC operator _BuiltinType&()
2361 { return _M_data; }
2362
2363 _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin() const
2364 { return _M_data; }
2365
2366 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](size_t __i) const
2367 { return _M_data & (_BuiltinType(1) << __i); }
2368
2369 template <size_t __i>
2370 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2371 operator[](_SizeConstant<__i>) const
2372 { return _M_data & (_BuiltinType(1) << __i); }
2373
2374 _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, value_type __x)
2375 {
2376 if (__x)
2377 _M_data |= (_BuiltinType(1) << __i);
2378 else
2379 _M_data &= ~(_BuiltinType(1) << __i);
2380 }
2381
2382 _GLIBCXX_SIMD_INTRINSIC
2383 constexpr bool _M_is_constprop() const
2384 { return __builtin_constant_p(_M_data); }
2385
2386 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2387 {
2388 if (__builtin_constant_p(_M_data))
2389 {
2390 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2391 constexpr _BuiltinType __active_mask
2392 = ~_BuiltinType() >> (__nbits - _Width);
2393 return (_M_data & __active_mask) == 0;
2394 }
2395 return false;
2396 }
2397
2398 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2399 {
2400 if (__builtin_constant_p(_M_data))
2401 {
2402 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2403 constexpr _BuiltinType __active_mask
2404 = ~_BuiltinType() >> (__nbits - _Width);
2405 return (_M_data & __active_mask) == __active_mask;
2406 }
2407 return false;
2408 }
2409
2410 _BuiltinType _M_data;
2411 };
2412
2413// _SimdWrapperBase{{{1
2414template <bool _MustZeroInitPadding, typename _BuiltinType>
2415 struct _SimdWrapperBase;
2416
2417template <typename _BuiltinType>
2418 struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2419 {
2420 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
2421 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2422 : _M_data(__init)
2423 {}
2424
2425 _BuiltinType _M_data;
2426 };
2427
2428template <typename _BuiltinType>
2429 struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2430 // never become SNaN
2431 {
2432 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2433 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2434 : _M_data(__init)
2435 {}
2436
2437 _BuiltinType _M_data;
2438 };
2439
2440// }}}
2441// _SimdWrapper{{{
2442template <typename _Tp, size_t _Width>
2443 struct _SimdWrapper<
2444 _Tp, _Width,
2445 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2446 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2447 && sizeof(_Tp) * _Width
2448 == sizeof(__vector_type_t<_Tp, _Width>),
2449 __vector_type_t<_Tp, _Width>>
2450 {
2451 using _Base
2452 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2453 && sizeof(_Tp) * _Width
2454 == sizeof(__vector_type_t<_Tp, _Width>),
2455 __vector_type_t<_Tp, _Width>>;
2456
2457 static_assert(__is_vectorizable_v<_Tp>);
2458 static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2459
2460 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2461 using value_type = _Tp;
2462
2463 static inline constexpr size_t _S_full_size
2464 = sizeof(_BuiltinType) / sizeof(value_type);
2465 static inline constexpr int _S_size = _Width;
2466 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2467
2468 using _Base::_M_data;
2469
2470 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2471 __as_full_vector() const
2472 { return _M_data; }
2473
2474 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2475 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2476 [&](auto __i) { return __init.begin()[__i.value]; })) {}
2477
2478 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2479 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(const _SimdWrapper&)
2480 = default;
2481 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) = default;
2482
2483 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2484 operator=(const _SimdWrapper&) = default;
2485 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2486 operator=(_SimdWrapper&&) = default;
2487
2488 template <typename _V, typename = enable_if_t<disjunction_v<
2489 is_same<_V, __vector_type_t<_Tp, _Width>>,
2490 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2491 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2492 // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2493 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2494
2495 template <typename... _As,
2496 typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2497 && sizeof...(_As) <= _Width)>>
2498 _GLIBCXX_SIMD_INTRINSIC constexpr
2499 operator _SimdTuple<_Tp, _As...>() const
2500 {
2501 const auto& dd = _M_data; // workaround for GCC7 ICE
2502 return __generate_from_n_evaluations<sizeof...(_As),
2503 _SimdTuple<_Tp, _As...>>([&](
2504 auto __i) constexpr { return dd[int(__i)]; });
2505 }
2506
2507 _GLIBCXX_SIMD_INTRINSIC constexpr operator const _BuiltinType&() const
2508 { return _M_data; }
2509
2510 _GLIBCXX_SIMD_INTRINSIC constexpr operator _BuiltinType&()
2511 { return _M_data; }
2512
2513 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](size_t __i) const
2514 { return _M_data[__i]; }
2515
2516 template <size_t __i>
2517 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>) const
2518 { return _M_data[__i]; }
2519
2520 _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, _Tp __x)
2521 { _M_data[__i] = __x; }
2522
2523 _GLIBCXX_SIMD_INTRINSIC
2524 constexpr bool _M_is_constprop() const
2525 { return __builtin_constant_p(_M_data); }
2526
2527 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2528 {
2529 if (__builtin_constant_p(_M_data))
2530 {
2531 bool __r = true;
2532 if constexpr (is_floating_point_v<_Tp>)
2533 {
2534 using _Ip = __int_for_sizeof_t<_Tp>;
2535 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2536 __execute_n_times<_Width>(
2537 [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2538 }
2539 else
2540 __execute_n_times<_Width>(
2541 [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2542 return __r;
2543 }
2544 return false;
2545 }
2546
2547 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2548 {
2549 if (__builtin_constant_p(_M_data))
2550 {
2551 bool __r = true;
2552 if constexpr (is_floating_point_v<_Tp>)
2553 {
2554 using _Ip = __int_for_sizeof_t<_Tp>;
2555 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2556 __execute_n_times<_Width>(
2557 [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2558 }
2559 else
2560 __execute_n_times<_Width>(
2561 [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2562 return __r;
2563 }
2564 return false;
2565 }
2566 };
2567
2568// }}}
2569
2570// __vectorized_sizeof {{{
2571template <typename _Tp>
2572 constexpr size_t
2573 __vectorized_sizeof()
2574 {
2575 if constexpr (!__is_vectorizable_v<_Tp>)
2576 return 0;
2577
2578 if constexpr (sizeof(_Tp) <= 8)
2579 {
2580 // X86:
2581 if constexpr (__have_avx512bw)
2582 return 64;
2583 if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2584 return 64;
2585 if constexpr (__have_avx2)
2586 return 32;
2587 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2588 return 32;
2589 if constexpr (__have_sse2)
2590 return 16;
2591 if constexpr (__have_sse && is_same_v<_Tp, float>)
2592 return 16;
2593 /* The following is too much trouble because of mixed MMX and x87 code.
2594 * While nothing here explicitly calls MMX instructions of registers,
2595 * they are still emitted but no EMMS cleanup is done.
2596 if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2597 return 8;
2598 */
2599
2600 // PowerPC:
2601 if constexpr (__have_power8vec
2602 || (__have_power_vmx && (sizeof(_Tp) < 8))
2603 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2604 return 16;
2605
2606 // ARM:
2607 if constexpr (__have_neon_a64
2608 || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2609 return 16;
2610 if constexpr (__have_neon
2611 && sizeof(_Tp) < 8
2612 // Only allow fp if the user allows non-ICE559 fp (e.g.
2613 // via -ffast-math). ARMv7 NEON fp is not conforming to
2614 // IEC559.
2615 && (__support_neon_float || !is_floating_point_v<_Tp>))
2616 return 16;
2617 }
2618
2619 return sizeof(_Tp);
2620 }
2621
2622// }}}
2623namespace simd_abi {
2624// most of simd_abi is defined in simd_detail.h
2625template <typename _Tp>
2626 inline constexpr int max_fixed_size
2627 = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2628
2629// compatible {{{
2630#if defined __x86_64__ || defined __aarch64__
2631template <typename _Tp>
2632 using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2633#elif defined __ARM_NEON
2634// FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2635// ABI?)
2636template <typename _Tp>
2637 using compatible
2638 = conditional_t<(sizeof(_Tp) < 8
2639 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2640 _VecBuiltin<16>, scalar>;
2641#else
2642template <typename>
2643 using compatible = scalar;
2644#endif
2645
2646// }}}
2647// native {{{
2648template <typename _Tp>
2649 constexpr auto
2650 __determine_native_abi()
2651 {
2652 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2653 if constexpr (__bytes == sizeof(_Tp))
2654 return static_cast<scalar*>(nullptr);
2655 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2656 return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2657 else
2658 return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2659 }
2660
2661template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2662 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2663
2664// }}}
2665// __default_abi {{{
2666#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2667template <typename _Tp>
2668 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2669#else
2670template <typename _Tp>
2671 using __default_abi = compatible<_Tp>;
2672#endif
2673
2674// }}}
2675} // namespace simd_abi
2676
2677// traits {{{1
2678// is_abi_tag {{{2
2679template <typename _Tp, typename = void_t<>>
2680 struct is_abi_tag : false_type {};
2681
2682template <typename _Tp>
2683 struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2684 : public _Tp::_IsValidAbiTag {};
2685
2686template <typename _Tp>
2687 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2688
2689// is_simd(_mask) {{{2
2690template <typename _Tp>
2691 struct is_simd : public false_type {};
2692
2693template <typename _Tp>
2694 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2695
2696template <typename _Tp>
2697 struct is_simd_mask : public false_type {};
2698
2699template <typename _Tp>
2700inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2701
2702// simd_size {{{2
2703template <typename _Tp, typename _Abi, typename = void>
2704 struct __simd_size_impl {};
2705
2706template <typename _Tp, typename _Abi>
2707 struct __simd_size_impl<
2708 _Tp, _Abi,
2709 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2710 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2711
2712template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2713 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2714
2715template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2716 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2717
2718// simd_abi::deduce {{{2
2719template <typename _Tp, size_t _Np, typename = void>
2720 struct __deduce_impl;
2721
2722namespace simd_abi {
2723/**
2724 * @tparam _Tp The requested `value_type` for the elements.
2725 * @tparam _Np The requested number of elements.
2726 * @tparam _Abis This parameter is ignored, since this implementation cannot
2727 * make any use of it. Either __a good native ABI is matched and used as `type`
2728 * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2729 * the best matching native ABIs.
2730 */
2731template <typename _Tp, size_t _Np, typename...>
2732 struct deduce : __deduce_impl<_Tp, _Np> {};
2733
2734template <typename _Tp, size_t _Np, typename... _Abis>
2735 using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2736} // namespace simd_abi
2737
2738// }}}2
2739// rebind_simd {{{2
2740template <typename _Tp, typename _V, typename = void>
2741 struct rebind_simd;
2742
2743template <typename _Tp, typename _Up, typename _Abi>
2744 struct rebind_simd<
2745 _Tp, simd<_Up, _Abi>,
2746 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2747 {
2748 using type
2749 = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2750 };
2751
2752template <typename _Tp, typename _Up, typename _Abi>
2753 struct rebind_simd<
2754 _Tp, simd_mask<_Up, _Abi>,
2755 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2756 {
2757 using type
2758 = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2759 };
2760
2761template <typename _Tp, typename _V>
2762 using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2763
2764// resize_simd {{{2
2765template <int _Np, typename _V, typename = void>
2766 struct resize_simd;
2767
2768template <int _Np, typename _Tp, typename _Abi>
2769 struct resize_simd<_Np, simd<_Tp, _Abi>,
2770 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2771 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2772
2773template <int _Np, typename _Tp, typename _Abi>
2774 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2775 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2776 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2777
2778template <int _Np, typename _V>
2779 using resize_simd_t = typename resize_simd<_Np, _V>::type;
2780
2781// }}}2
2782// memory_alignment {{{2
2783template <typename _Tp, typename _Up = typename _Tp::value_type>
2784 struct memory_alignment
2785 : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2786
2787template <typename _Tp, typename _Up = typename _Tp::value_type>
2788 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2789
2790// class template simd [simd] {{{1
2791template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2792 class simd;
2793
2794template <typename _Tp, typename _Abi>
2795 struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2796
2797template <typename _Tp>
2798 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2799
2800template <typename _Tp, int _Np>
2801 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2802
2803template <typename _Tp, size_t _Np>
2804 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2805
2806// class template simd_mask [simd_mask] {{{1
2807template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2808 class simd_mask;
2809
2810template <typename _Tp, typename _Abi>
2811 struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2812
2813template <typename _Tp>
2814 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2815
2816template <typename _Tp, int _Np>
2817 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2818
2819template <typename _Tp, size_t _Np>
2820 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2821
2822// casts [simd.casts] {{{1
2823// static_simd_cast {{{2
2824template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>,
2825 typename = void>
2826 struct __static_simd_cast_return_type;
2827
2828template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2829 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2830 void>
2831 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2832
2833template <typename _Tp, typename _Up, typename _Ap>
2834 struct __static_simd_cast_return_type<
2835 _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2836 { using type = _Tp; };
2837
2838template <typename _Tp, typename _Ap>
2839 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2840#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2841 enable_if_t<__is_vectorizable_v<_Tp>>
2842#else
2843 void
2844#endif
2845 >
2846 { using type = simd<_Tp, _Ap>; };
2847
2848template <typename _Tp, typename = void>
2849 struct __safe_make_signed { using type = _Tp;};
2850
2851template <typename _Tp>
2852 struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2853 {
2854 // the extra make_unsigned_t is because of PR85951
2855 using type = make_signed_t<make_unsigned_t<_Tp>>;
2856 };
2857
2858template <typename _Tp>
2859 using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2860
2861template <typename _Tp, typename _Up, typename _Ap>
2862 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2863#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2864 enable_if_t<__is_vectorizable_v<_Tp>>
2865#else
2866 void
2867#endif
2868 >
2869 {
2870 using type = conditional_t<
2871 (is_integral_v<_Up> && is_integral_v<_Tp> &&
2872#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2873 is_signed_v<_Up> != is_signed_v<_Tp> &&
2874#endif
2875 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2876 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2877 };
2878
2879template <typename _Tp, typename _Up, typename _Ap,
2880 typename _R
2881 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2882 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2883 static_simd_cast(const simd<_Up, _Ap>& __x)
2884 {
2885 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2886 return __x;
2887 else
2888 {
2889 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2890 __c;
2891 return _R(__private_init, __c(__data(__x)));
2892 }
2893 }
2894
2895namespace __proposed {
2896template <typename _Tp, typename _Up, typename _Ap,
2897 typename _R
2898 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2899 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
2900 static_simd_cast(const simd_mask<_Up, _Ap>& __x)
2901 {
2902 using _RM = typename _R::mask_type;
2903 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2904 typename _RM::simd_type::value_type>(__x)};
2905 }
2906} // namespace __proposed
2907
2908// simd_cast {{{2
2909template <typename _Tp, typename _Up, typename _Ap,
2910 typename _To = __value_type_or_identity_t<_Tp>>
2911 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
2912 simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
2913 -> decltype(static_simd_cast<_Tp>(__x))
2914 { return static_simd_cast<_Tp>(__x); }
2915
2916namespace __proposed {
2917template <typename _Tp, typename _Up, typename _Ap,
2918 typename _To = __value_type_or_identity_t<_Tp>>
2919 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
2920 simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
2921 -> decltype(static_simd_cast<_Tp>(__x))
2922 { return static_simd_cast<_Tp>(__x); }
2923} // namespace __proposed
2924
2925// }}}2
2926// resizing_simd_cast {{{
2927namespace __proposed {
2928/* Proposed spec:
2929
2930template <class T, class U, class Abi>
2931T resizing_simd_cast(const simd<U, Abi>& x)
2932
2933p1 Constraints:
2934 - is_simd_v<T> is true and
2935 - T::value_type is the same type as U
2936
2937p2 Returns:
2938 A simd object with the i^th element initialized to x[i] for all i in the
2939 range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
2940 than simd_size_v<U, Abi>, the remaining elements are value-initialized.
2941
2942template <class T, class U, class Abi>
2943T resizing_simd_cast(const simd_mask<U, Abi>& x)
2944
2945p1 Constraints: is_simd_mask_v<T> is true
2946
2947p2 Returns:
2948 A simd_mask object with the i^th element initialized to x[i] for all i in
2949the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
2950 than simd_size_v<U, Abi>, the remaining elements are initialized to false.
2951
2952 */
2953
2954template <typename _Tp, typename _Up, typename _Ap>
2955 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
2956 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
2957 resizing_simd_cast(const simd<_Up, _Ap>& __x)
2958 {
2959 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
2960 return __x;
2961 else if constexpr (simd_size_v<_Up, _Ap> == 1)
2962 {
2963 _Tp __r{};
2964 __r[0] = __x[0];
2965 return __r;
2966 }
2967 else if constexpr (_Tp::size() == 1)
2968 return __x[0];
2969 else if constexpr (sizeof(_Tp) == sizeof(__x)
2970 && !__is_fixed_size_abi_v<_Ap>)
2971 return {__private_init,
2972 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
2973 _Ap::_S_masked(__data(__x))._M_data)};
2974 else
2975 {
2976 _Tp __r{};
2977 __builtin_memcpy(&__data(__r), &__data(__x),
2978 sizeof(_Up)
2979 * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
2980 return __r;
2981 }
2982 }
2983
2984template <typename _Tp, typename _Up, typename _Ap>
2985 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2986 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
2987 resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
2988 {
2989 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
2990 typename _Tp::simd_type::value_type>(__x)};
2991 }
2992} // namespace __proposed
2993
2994// }}}
2995// to_fixed_size {{{2
2996template <typename _Tp, int _Np>
2997 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
2998 to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
2999 { return __x; }
3000
3001template <typename _Tp, int _Np>
3002 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3003 to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3004 { return __x; }
3005
3006template <typename _Tp, typename _Ap>
3007 _GLIBCXX_SIMD_INTRINSIC auto
3008 to_fixed_size(const simd<_Tp, _Ap>& __x)
3009 {
3010 return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3011 auto __i) constexpr { return __x[__i]; });
3012 }
3013
3014template <typename _Tp, typename _Ap>
3015 _GLIBCXX_SIMD_INTRINSIC auto
3016 to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3017 {
3018 constexpr int _Np = simd_mask<_Tp, _Ap>::size();
3019 fixed_size_simd_mask<_Tp, _Np> __r;
3020 __execute_n_times<_Np>([&](auto __i) constexpr { __r[__i] = __x[__i]; });
3021 return __r;
3022 }
3023
3024// to_native {{{2
3025template <typename _Tp, int _Np>
3026 _GLIBCXX_SIMD_INTRINSIC
3027 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3028 to_native(const fixed_size_simd<_Tp, _Np>& __x)
3029 {
3030 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3031 __x.copy_to(__mem, vector_aligned);
3032 return {__mem, vector_aligned};
3033 }
3034
3035template <typename _Tp, size_t _Np>
3036 _GLIBCXX_SIMD_INTRINSIC
3037 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3038 to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3039 {
3040 return native_simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; });
3041 }
3042
3043// to_compatible {{{2
3044template <typename _Tp, size_t _Np>
3045 _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3046 to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3047 {
3048 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3049 __x.copy_to(__mem, vector_aligned);
3050 return {__mem, vector_aligned};
3051 }
3052
3053template <typename _Tp, size_t _Np>
3054 _GLIBCXX_SIMD_INTRINSIC
3055 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3056 to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3057 { return simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; }); }
3058
3059// masked assignment [simd_mask.where] {{{1
3060
3061// where_expression {{{1
3062// const_where_expression<M, T> {{{2
3063template <typename _M, typename _Tp>
3064 class const_where_expression
3065 {
3066 using _V = _Tp;
3067 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3068
3069 struct _Wrapper { using value_type = _V; };
3070
3071 protected:
3072 using _Impl = typename _V::_Impl;
3073
3074 using value_type =
3075 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3076
3077 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3078 __get_mask(const const_where_expression& __x)
3079 { return __x._M_k; }
3080
3081 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3082 __get_lvalue(const const_where_expression& __x)
3083 { return __x._M_value; }
3084
3085 const _M& _M_k;
3086 _Tp& _M_value;
3087
3088 public:
3089 const_where_expression(const const_where_expression&) = delete;
3090 const_where_expression& operator=(const const_where_expression&) = delete;
3091
3092 _GLIBCXX_SIMD_INTRINSIC const_where_expression(const _M& __kk, const _Tp& dd)
3093 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3094
3095 _GLIBCXX_SIMD_INTRINSIC _V
3096 operator-() const&&
3097 {
3098 return {__private_init,
3099 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3100 __data(_M_value))};
3101 }
3102
3103 template <typename _Up, typename _Flags>
3104 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3105 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3106 {
3107 return {__private_init,
3108 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3109 _Flags::template _S_apply<_V>(__mem))};
3110 }
3111
3112 template <typename _Up, typename _Flags>
3113 _GLIBCXX_SIMD_INTRINSIC void
3114 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3115 {
3116 _Impl::_S_masked_store(__data(_M_value),
3117 _Flags::template _S_apply<_V>(__mem),
3118 __data(_M_k));
3119 }
3120 };
3121
3122// const_where_expression<bool, T> {{{2
3123template <typename _Tp>
3124 class const_where_expression<bool, _Tp>
3125 {
3126 using _M = bool;
3127 using _V = _Tp;
3128
3129 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3130
3131 struct _Wrapper { using value_type = _V; };
3132
3133 protected:
3134 using value_type =
3135 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3136
3137 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3138 __get_mask(const const_where_expression& __x)
3139 { return __x._M_k; }
3140
3141 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3142 __get_lvalue(const const_where_expression& __x)
3143 { return __x._M_value; }
3144
3145 const bool _M_k;
3146 _Tp& _M_value;
3147
3148 public:
3149 const_where_expression(const const_where_expression&) = delete;
3150 const_where_expression& operator=(const const_where_expression&) = delete;
3151
3152 _GLIBCXX_SIMD_INTRINSIC const_where_expression(const bool __kk, const _Tp& dd)
3153 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3154
3155 _GLIBCXX_SIMD_INTRINSIC _V operator-() const&&
3156 { return _M_k ? -_M_value : _M_value; }
3157
3158 template <typename _Up, typename _Flags>
3159 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3160 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3161 { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3162
3163 template <typename _Up, typename _Flags>
3164 _GLIBCXX_SIMD_INTRINSIC void
3165 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3166 {
3167 if (_M_k)
3168 __mem[0] = _M_value;
3169 }
3170 };
3171
3172// where_expression<M, T> {{{2
3173template <typename _M, typename _Tp>
3174 class where_expression : public const_where_expression<_M, _Tp>
3175 {
3176 using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3177
3178 static_assert(!is_const<_Tp>::value,
3179 "where_expression may only be instantiated with __a non-const "
3180 "_Tp parameter");
3181
3182 using typename const_where_expression<_M, _Tp>::value_type;
3183 using const_where_expression<_M, _Tp>::_M_k;
3184 using const_where_expression<_M, _Tp>::_M_value;
3185
3186 static_assert(
3187 is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3188 static_assert(_M::size() == _Tp::size(), "");
3189
3190 _GLIBCXX_SIMD_INTRINSIC friend _Tp& __get_lvalue(where_expression& __x)
3191 { return __x._M_value; }
3192
3193 public:
3194 where_expression(const where_expression&) = delete;
3195 where_expression& operator=(const where_expression&) = delete;
3196
3197 _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3198 : const_where_expression<_M, _Tp>(__kk, dd) {}
3199
3200 template <typename _Up>
3201 _GLIBCXX_SIMD_INTRINSIC void operator=(_Up&& __x) &&
3202 {
3203 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3204 __to_value_type_or_member_type<_Tp>(
3205 static_cast<_Up&&>(__x)));
3206 }
3207
3208#define _GLIBCXX_SIMD_OP_(__op, __name) \
3209 template <typename _Up> \
3210 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3211 { \
3212 _Impl::template _S_masked_cassign( \
3213 __data(_M_k), __data(_M_value), \
3214 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3215 [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3216 return __impl.__name(__lhs, __rhs); \
3217 }); \
3218 } \
3219 static_assert(true)
3220 _GLIBCXX_SIMD_OP_(+, _S_plus);
3221 _GLIBCXX_SIMD_OP_(-, _S_minus);
3222 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3223 _GLIBCXX_SIMD_OP_(/, _S_divides);
3224 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3225 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3226 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3227 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3228 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3229 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3230#undef _GLIBCXX_SIMD_OP_
3231
3232 _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3233 {
3234 __data(_M_value)
3235 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3236 __data(_M_value));
3237 }
3238
3239 _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3240 {
3241 __data(_M_value)
3242 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3243 __data(_M_value));
3244 }
3245
3246 _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3247 {
3248 __data(_M_value)
3249 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3250 __data(_M_value));
3251 }
3252
3253 _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3254 {
3255 __data(_M_value)
3256 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3257 __data(_M_value));
3258 }
3259
3260 // intentionally hides const_where_expression::copy_from
3261 template <typename _Up, typename _Flags>
3262 _GLIBCXX_SIMD_INTRINSIC void
3263 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3264 {
3265 __data(_M_value)
3266 = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3267 _Flags::template _S_apply<_Tp>(__mem));
3268 }
3269 };
3270
3271// where_expression<bool, T> {{{2
3272template <typename _Tp>
3273 class where_expression<bool, _Tp> : public const_where_expression<bool, _Tp>
3274 {
3275 using _M = bool;
3276 using typename const_where_expression<_M, _Tp>::value_type;
3277 using const_where_expression<_M, _Tp>::_M_k;
3278 using const_where_expression<_M, _Tp>::_M_value;
3279
3280 public:
3281 where_expression(const where_expression&) = delete;
3282 where_expression& operator=(const where_expression&) = delete;
3283
3284 _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3285 : const_where_expression<_M, _Tp>(__kk, dd) {}
3286
3287#define _GLIBCXX_SIMD_OP_(__op) \
3288 template <typename _Up> \
3289 _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3290 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3291
3292 _GLIBCXX_SIMD_OP_(=)
3293 _GLIBCXX_SIMD_OP_(+=)
3294 _GLIBCXX_SIMD_OP_(-=)
3295 _GLIBCXX_SIMD_OP_(*=)
3296 _GLIBCXX_SIMD_OP_(/=)
3297 _GLIBCXX_SIMD_OP_(%=)
3298 _GLIBCXX_SIMD_OP_(&=)
3299 _GLIBCXX_SIMD_OP_(|=)
3300 _GLIBCXX_SIMD_OP_(^=)
3301 _GLIBCXX_SIMD_OP_(<<=)
3302 _GLIBCXX_SIMD_OP_(>>=)
3303 #undef _GLIBCXX_SIMD_OP_
3304
3305 _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3306 { if (_M_k) ++_M_value; }
3307
3308 _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3309 { if (_M_k) ++_M_value; }
3310
3311 _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3312 { if (_M_k) --_M_value; }
3313
3314 _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3315 { if (_M_k) --_M_value; }
3316
3317 // intentionally hides const_where_expression::copy_from
3318 template <typename _Up, typename _Flags>
3319 _GLIBCXX_SIMD_INTRINSIC void
3320 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3321 { if (_M_k) _M_value = __mem[0]; }
3322 };
3323
3324// where {{{1
3325template <typename _Tp, typename _Ap>
3326 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3327 where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3328 { return {__k, __value}; }
3329
3330template <typename _Tp, typename _Ap>
3331 _GLIBCXX_SIMD_INTRINSIC
3332 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3333 where(const typename simd<_Tp, _Ap>::mask_type& __k,
3334 const simd<_Tp, _Ap>& __value)
3335 { return {__k, __value}; }
3336
3337template <typename _Tp, typename _Ap>
3338 _GLIBCXX_SIMD_INTRINSIC
3339 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3340 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3341 simd_mask<_Tp, _Ap>& __value)
3342 { return {__k, __value}; }
3343
3344template <typename _Tp, typename _Ap>
3345 _GLIBCXX_SIMD_INTRINSIC
3346 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3347 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3348 const simd_mask<_Tp, _Ap>& __value)
3349 { return {__k, __value}; }
3350
3351template <typename _Tp>
3352 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3353 where(_ExactBool __k, _Tp& __value)
3354 { return {__k, __value}; }
3355
3356template <typename _Tp>
3357 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3358 where(_ExactBool __k, const _Tp& __value)
3359 { return {__k, __value}; }
3360
3361 template <typename _Tp, typename _Ap>
3362 void where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3363
3364 template <typename _Tp, typename _Ap>
3365 void where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3366
3367// proposed mask iterations {{{1
3368namespace __proposed {
3369template <size_t _Np>
3370 class where_range
3371 {
3372 const bitset<_Np> __bits;
3373
3374 public:
3375 where_range(bitset<_Np> __b) : __bits(__b) {}
3376
3377 class iterator
3378 {
3379 size_t __mask;
3380 size_t __bit;
3381
3382 _GLIBCXX_SIMD_INTRINSIC void __next_bit()
3383 { __bit = __builtin_ctzl(__mask); }
3384
3385 _GLIBCXX_SIMD_INTRINSIC void __reset_lsb()
3386 {
3387 // 01100100 - 1 = 01100011
3388 __mask &= (__mask - 1);
3389 // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3390 }
3391
3392 public:
3393 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3394 iterator(const iterator&) = default;
3395 iterator(iterator&&) = default;
3396
3397 _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator->() const
3398 { return __bit; }
3399
3400 _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator*() const
3401 { return __bit; }
3402
3403 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3404 {
3405 __reset_lsb();
3406 __next_bit();
3407 return *this;
3408 }
3409
3410 _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(int)
3411 {
3412 iterator __tmp = *this;
3413 __reset_lsb();
3414 __next_bit();
3415 return __tmp;
3416 }
3417
3418 _GLIBCXX_SIMD_ALWAYS_INLINE bool operator==(const iterator& __rhs) const
3419 { return __mask == __rhs.__mask; }
3420
3421 _GLIBCXX_SIMD_ALWAYS_INLINE bool operator!=(const iterator& __rhs) const
3422 { return __mask != __rhs.__mask; }
3423 };
3424
3425 iterator begin() const
3426 { return __bits.to_ullong(); }
3427
3428 iterator end() const
3429 { return 0; }
3430 };
3431
3432template <typename _Tp, typename _Ap>
3433 where_range<simd_size_v<_Tp, _Ap>>
3434 where(const simd_mask<_Tp, _Ap>& __k)
3435 { return __k.__to_bitset(); }
3436
3437} // namespace __proposed
3438
3439// }}}1
3440// reductions [simd.reductions] {{{1
3441template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3442 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3443 reduce(const simd<_Tp, _Abi>& __v,
3444 _BinaryOperation __binary_op = _BinaryOperation())
3445 { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3446
3447template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3448 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3449 reduce(const const_where_expression<_M, _V>& __x,
3450 typename _V::value_type __identity_element,
3451 _BinaryOperation __binary_op)
3452 {
3453 if (__builtin_expect(none_of(__get_mask(__x)), false))
3454 return __identity_element;
3455
3456 _V __tmp = __identity_element;
3457 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3458 __data(__get_lvalue(__x)));
3459 return reduce(__tmp, __binary_op);
3460 }
3461
3462template <typename _M, typename _V>
3463 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3464 reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3465 { return reduce(__x, 0, __binary_op); }
3466
3467template <typename _M, typename _V>
3468 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3469 reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3470 { return reduce(__x, 1, __binary_op); }
3471
3472template <typename _M, typename _V>
3473 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3474 reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3475 { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3476
3477template <typename _M, typename _V>
3478 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3479 reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3480 { return reduce(__x, 0, __binary_op); }
3481
3482template <typename _M, typename _V>
3483 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3484 reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3485 { return reduce(__x, 0, __binary_op); }
3486
3487template <typename _Tp, typename _Abi>
3488 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3489 hmin(const simd<_Tp, _Abi>& __v) noexcept
3490 {
3491 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3492 }
3493
3494template <typename _Tp, typename _Abi>
3495 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3496 hmax(const simd<_Tp, _Abi>& __v) noexcept
3497 {
3498 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3499 }
3500
3501template <typename _M, typename _V>
3502 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3503 typename _V::value_type
3504 hmin(const const_where_expression<_M, _V>& __x) noexcept
3505 {
3506 using _Tp = typename _V::value_type;
3507 constexpr _Tp __id_elem =
3508#ifdef __FINITE_MATH_ONLY__
3509 __finite_max_v<_Tp>;
3510#else
3511 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3512#endif
3513 _V __tmp = __id_elem;
3514 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3515 __data(__get_lvalue(__x)));
3516 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3517 }
3518
3519template <typename _M, typename _V>
3520 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3521 typename _V::value_type
3522 hmax(const const_where_expression<_M, _V>& __x) noexcept
3523 {
3524 using _Tp = typename _V::value_type;
3525 constexpr _Tp __id_elem =
3526#ifdef __FINITE_MATH_ONLY__
3527 __finite_min_v<_Tp>;
3528#else
3529 [] {
3530 if constexpr (__value_exists_v<__infinity, _Tp>)
3531 return -__infinity_v<_Tp>;
3532 else
3533 return __finite_min_v<_Tp>;
3534 }();
3535#endif
3536 _V __tmp = __id_elem;
3537 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3538 __data(__get_lvalue(__x)));
3539 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3540 }
3541
3542// }}}1
3543// algorithms [simd.alg] {{{
3544template <typename _Tp, typename _Ap>
3545 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3546 min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3547 { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3548
3549template <typename _Tp, typename _Ap>
3550 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3551 max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3552 { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3553
3554template <typename _Tp, typename _Ap>
3555 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3556 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3557 minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3558 {
3559 const auto pair_of_members
3560 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3561 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3562 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3563 }
3564
3565template <typename _Tp, typename _Ap>
3566 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3567 clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo,
3568 const simd<_Tp, _Ap>& __hi)
3569 {
3570 using _Impl = typename _Ap::_SimdImpl;
3571 return {__private_init,
3572 _Impl::_S_min(__data(__hi),
3573 _Impl::_S_max(__data(__lo), __data(__v)))};
3574 }
3575
3576// }}}
3577
3578template <size_t... _Sizes, typename _Tp, typename _Ap,
3579 typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3580 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3581 split(const simd<_Tp, _Ap>&);
3582
3583// __extract_part {{{
3584template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3585 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3586 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3587 __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3588
3589template <int Index, int Parts, int _Combine = 1, typename _Tp, typename _A0,
3590 typename... _As>
3591 _GLIBCXX_SIMD_INTRINSIC auto
3592 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3593
3594// }}}
3595// _SizeList {{{
3596template <size_t _V0, size_t... _Values>
3597 struct _SizeList
3598 {
3599 template <size_t _I>
3600 static constexpr size_t _S_at(_SizeConstant<_I> = {})
3601 {
3602 if constexpr (_I == 0)
3603 return _V0;
3604 else
3605 return _SizeList<_Values...>::template _S_at<_I - 1>();
3606 }
3607
3608 template <size_t _I>
3609 static constexpr auto _S_before(_SizeConstant<_I> = {})
3610 {
3611 if constexpr (_I == 0)
3612 return _SizeConstant<0>();
3613 else
3614 return _SizeConstant<
3615 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3616 }
3617
3618 template <size_t _Np>
3619 static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3620 {
3621 if constexpr (_Np == 0)
3622 return _SizeList();
3623 else
3624 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3625 }
3626 };
3627
3628// }}}
3629// __extract_center {{{
3630template <typename _Tp, size_t _Np>
3631 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3632 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3633 {
3634 static_assert(_Np >= 4);
3635 static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3636#if _GLIBCXX_SIMD_X86INTRIN // {{{
3637 if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3638 {
3639 const auto __intrin = __to_intrin(__x);
3640 if constexpr (is_integral_v<_Tp>)
3641 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3642 _mm512_shuffle_i32x4(__intrin, __intrin,
3643 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3644 else if constexpr (sizeof(_Tp) == 4)
3645 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3646 _mm512_shuffle_f32x4(__intrin, __intrin,
3647 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3648 else if constexpr (sizeof(_Tp) == 8)
3649 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3650 _mm512_shuffle_f64x2(__intrin, __intrin,
3651 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3652 else
3653 __assert_unreachable<_Tp>();
3654 }
3655 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3656 return __vector_bitcast<_Tp>(
3657 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3658 __hi128(__vector_bitcast<double>(__x)), 1));
3659 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3660 return __vector_bitcast<_Tp>(
3661 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3662 __lo128(__vector_bitcast<_LLong>(__x)),
3663 sizeof(_Tp) * _Np / 4));
3664 else
3665#endif // _GLIBCXX_SIMD_X86INTRIN }}}
3666 {
3667 __vector_type_t<_Tp, _Np / 2> __r;
3668 __builtin_memcpy(&__r,
3669 reinterpret_cast<const char*>(&__x)
3670 + sizeof(_Tp) * _Np / 4,
3671 sizeof(_Tp) * _Np / 2);
3672 return __r;
3673 }
3674 }
3675
3676template <typename _Tp, typename _A0, typename... _As>
3677 _GLIBCXX_SIMD_INTRINSIC
3678 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3679 __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3680 {
3681 if constexpr (sizeof...(_As) == 0)
3682 return __extract_center(__x.first);
3683 else
3684 return __extract_part<1, 4, 2>(__x);
3685 }
3686
3687// }}}
3688// __split_wrapper {{{
3689template <size_t... _Sizes, typename _Tp, typename... _As>
3690 auto
3691 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3692 {
3693 return split<_Sizes...>(
3694 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3695 __x));
3696 }
3697
3698// }}}
3699
3700// split<simd>(simd) {{{
3701template <typename _V, typename _Ap,
3702 size_t Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3703 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts * _V::size()
3704 && is_simd_v<_V>, array<_V, Parts>>
3705 split(const simd<typename _V::value_type, _Ap>& __x)
3706 {
3707 using _Tp = typename _V::value_type;
3708 if constexpr (Parts == 1)
3709 {
3710 return {simd_cast<_V>(__x)};
3711 }
3712 else if (__x._M_is_constprop())
3713 {
3714 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3715 auto __i) constexpr {
3716 return _V([&](auto __j) constexpr {
3717 return __x[__i * _V::size() + __j];
3718 });
3719 });
3720 }
3721 else if constexpr (
3722 __is_fixed_size_abi_v<_Ap>
3723 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3724 || (__is_fixed_size_abi_v<typename _V::abi_type>
3725 && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3726 )))
3727 {
3728 // fixed_size -> fixed_size (w/o padding) or scalar
3729#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3730 const __may_alias<_Tp>* const __element_ptr
3731 = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3732 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3733 auto __i) constexpr {
3734 return _V(__element_ptr + __i * _V::size(), vector_aligned);
3735 });
3736#else
3737 const auto& __xx = __data(__x);
3738 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3739 auto __i) constexpr {
3740 [[maybe_unused]] constexpr size_t __offset
3741 = decltype(__i)::value * _V::size();
3742 return _V([&](auto __j) constexpr {
3743 constexpr _SizeConstant<__j + __offset> __k;
3744 return __xx[__k];
3745 });
3746 });
3747#endif
3748 }
3749 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3750 {
3751 // normally memcpy should work here as well
3752 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3753 auto __i) constexpr { return __x[__i]; });
3754 }
3755 else
3756 {
3757 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3758 auto __i) constexpr {
3759 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3760 return _V([&](auto __j) constexpr {
3761 return __x[__i * _V::size() + __j];
3762 });
3763 else
3764 return _V(__private_init,
3765 __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3766 });
3767 }
3768 }
3769
3770// }}}
3771// split<simd_mask>(simd_mask) {{{
3772template <typename _V, typename _Ap,
3773 size_t _Parts
3774 = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3775 enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3776 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3777 split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3778 {
3779 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3780 return {__x};
3781 else if constexpr (_Parts == 1)
3782 return {__proposed::static_simd_cast<_V>(__x)};
3783 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3784 && __is_avx_abi<_Ap>())
3785 return {_V(__private_init, __lo128(__data(__x))),
3786 _V(__private_init, __hi128(__data(__x)))};
3787 else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3788 {
3789 const bitset __bits = __x.__to_bitset();
3790 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3791 auto __i) constexpr {
3792 constexpr size_t __offset = __i * _V::size();
3793 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3794 });
3795 }
3796 else
3797 {
3798 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3799 auto __i) constexpr {
3800 constexpr size_t __offset = __i * _V::size();
3801 return _V(
3802 __private_init, [&](auto __j) constexpr {
3803 return __x[__j + __offset];
3804 });
3805 });
3806 }
3807 }
3808
3809// }}}
3810// split<_Sizes...>(simd) {{{
3811template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3812 _GLIBCXX_SIMD_ALWAYS_INLINE
3813 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3814 split(const simd<_Tp, _Ap>& __x)
3815 {
3816 using _SL = _SizeList<_Sizes...>;
3817 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3818 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3819 constexpr size_t _N0 = _SL::template _S_at<0>();
3820 using _V = __deduced_simd<_Tp, _N0>;
3821
3822 if (__x._M_is_constprop())
3823 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3824 auto __i) constexpr {
3825 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3826 constexpr size_t __offset = _SL::_S_before(__i);
3827 return _Vi([&](auto __j) constexpr { return __x[__offset + __j]; });
3828 });
3829 else if constexpr (_Np == _N0)
3830 {
3831 static_assert(sizeof...(_Sizes) == 1);
3832 return {simd_cast<_V>(__x)};
3833 }
3834 else if constexpr // split from fixed_size, such that __x::first.size == _N0
3835 (__is_fixed_size_abi_v<
3836 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3837 {
3838 static_assert(
3839 !__is_fixed_size_abi_v<typename _V::abi_type>,
3840 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3841 "fixed_size_simd "
3842 "when deduced?");
3843 // extract first and recurse (__split_wrapper is needed to deduce a new
3844 // _Sizes pack)
3845 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3846 __split_wrapper(_SL::template _S_pop_front<1>(),
3847 __data(__x).second));
3848 }
3849 else if constexpr ((!is_same_v<simd_abi::scalar,
3850 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3851 && (!__is_fixed_size_abi_v<
3852 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3853 {
3854 if constexpr (((_Sizes * 2 == _Np) && ...))
3855 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3856 {__private_init, __extract_part<1, 2>(__data(__x))}};
3857 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3858 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3859 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3860 {__private_init, __extract_part<1, 3>(__data(__x))},
3861 {__private_init, __extract_part<2, 3>(__data(__x))}};
3862 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3863 _SizeList<2 * _Np / 3, _Np / 3>>)
3864 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3865 {__private_init, __extract_part<2, 3>(__data(__x))}};
3866 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3867 _SizeList<_Np / 3, 2 * _Np / 3>>)
3868 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3869 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3870 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3871 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3872 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3873 {__private_init, __extract_part<2, 4>(__data(__x))},
3874 {__private_init, __extract_part<3, 4>(__data(__x))}};
3875 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3876 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3877 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3878 {__private_init, __extract_part<1, 4>(__data(__x))},
3879 {__private_init, __extract_part<1, 2>(__data(__x))}};
3880 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3881 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3882 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3883 {__private_init, __extract_center(__data(__x))},
3884 {__private_init, __extract_part<3, 4>(__data(__x))}};
3885 else if constexpr (((_Sizes * 4 == _Np) && ...))
3886 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3887 {__private_init, __extract_part<1, 4>(__data(__x))},
3888 {__private_init, __extract_part<2, 4>(__data(__x))},
3889 {__private_init, __extract_part<3, 4>(__data(__x))}};
3890 // else fall through
3891 }
3892#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3893 const __may_alias<_Tp>* const __element_ptr
3894 = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
3895 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3896 auto __i) constexpr {
3897 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3898 constexpr size_t __offset = _SL::_S_before(__i);
3899 constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
3900 constexpr size_t __a
3901 = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
3902 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
3903 constexpr size_t __alignment = __b == 0 ? __a : __b;
3904 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
3905 });
3906#else
3907 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3908 auto __i) constexpr {
3909 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3910 const auto& __xx = __data(__x);
3911 using _Offset = decltype(_SL::_S_before(__i));
3912 return _Vi([&](auto __j) constexpr {
3913 constexpr _SizeConstant<_Offset::value + __j> __k;
3914 return __xx[__k];
3915 });
3916 });
3917#endif
3918 }
3919
3920// }}}
3921
3922// __subscript_in_pack {{{
3923template <size_t _I, typename _Tp, typename _Ap, typename... _As>
3924 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
3925 __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
3926 {
3927 if constexpr (_I < simd_size_v<_Tp, _Ap>)
3928 return __x[_I];
3929 else
3930 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
3931 }
3932
3933// }}}
3934// __store_pack_of_simd {{{
3935template <typename _Tp, typename _A0, typename... _As>
3936 _GLIBCXX_SIMD_INTRINSIC void
3937 __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0,
3938 const simd<_Tp, _As>&... __xs)
3939 {
3940 constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
3941 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
3942 if constexpr (sizeof...(__xs) > 0)
3943 __store_pack_of_simd(__mem + __n_bytes, __xs...);
3944 }
3945
3946// }}}
3947// concat(simd...) {{{
3948template <typename _Tp, typename... _As>
3949 inline _GLIBCXX_SIMD_CONSTEXPR
3950 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
3951 concat(const simd<_Tp, _As>&... __xs)
3952 {
3953 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
3954 if constexpr (sizeof...(__xs) == 1)
3955 return simd_cast<_Rp>(__xs...);
3956 else if ((... && __xs._M_is_constprop()))
3957 return simd<_Tp,
3958 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
3959 auto __i) constexpr { return __subscript_in_pack<__i>(__xs...); });
3960 else
3961 {
3962 _Rp __r{};
3963 __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
3964 return __r;
3965 }
3966 }
3967
3968// }}}
3969// concat(array<simd>) {{{
3970template <typename _Tp, typename _Abi, size_t _Np>
3971 _GLIBCXX_SIMD_ALWAYS_INLINE
3972 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
3973 concat(const array<simd<_Tp, _Abi>, _Np>& __x)
3974 {
3975 return __call_with_subscripts<_Np>(__x, [](const auto&... __xs) {
3976 return concat(__xs...);
3977 });
3978 }
3979
3980// }}}
3981
3982/// @cond undocumented
3983// _SmartReference {{{
3984template <typename _Up, typename _Accessor = _Up,
3985 typename _ValueType = typename _Up::value_type>
3986 class _SmartReference
3987 {
3988 friend _Accessor;
3989 int _M_index;
3990 _Up& _M_obj;
3991
3992 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
3993 {
3994 if constexpr (is_arithmetic_v<_Up>)
3995 return _M_obj;
3996 else
3997 return _M_obj[_M_index];
3998 }
3999
4000 template <typename _Tp>
4001 _GLIBCXX_SIMD_INTRINSIC constexpr void _M_write(_Tp&& __x) const
4002 { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4003
4004 public:
4005 _GLIBCXX_SIMD_INTRINSIC constexpr
4006 _SmartReference(_Up& __o, int __i) noexcept
4007 : _M_index(__i), _M_obj(__o) {}
4008
4009 using value_type = _ValueType;
4010
4011 _GLIBCXX_SIMD_INTRINSIC _SmartReference(const _SmartReference&) = delete;
4012
4013 _GLIBCXX_SIMD_INTRINSIC constexpr operator value_type() const noexcept
4014 { return _M_read(); }
4015
4016 template <typename _Tp,
4017 typename
4018 = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4019 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4020 {
4021 _M_write(static_cast<_Tp&&>(__x));
4022 return {_M_obj, _M_index};
4023 }
4024
4025#define _GLIBCXX_SIMD_OP_(__op) \
4026 template <typename _Tp, \
4027 typename _TT \
4028 = decltype(declval<value_type>() __op declval<_Tp>()), \
4029 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4030 typename = _ValuePreservingOrInt<_TT, value_type>> \
4031 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4032 operator __op##=(_Tp&& __x) && \
4033 { \
4034 const value_type& __lhs = _M_read(); \
4035 _M_write(__lhs __op __x); \
4036 return {_M_obj, _M_index}; \
4037 }
4038 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4039 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4040 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4041#undef _GLIBCXX_SIMD_OP_
4042
4043 template <typename _Tp = void,
4044 typename
4045 = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4046 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4047 {
4048 value_type __x = _M_read();
4049 _M_write(++__x);
4050 return {_M_obj, _M_index};
4051 }
4052
4053 template <typename _Tp = void,
4054 typename
4055 = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4056 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(int) &&
4057 {
4058 const value_type __r = _M_read();
4059 value_type __x = __r;
4060 _M_write(++__x);
4061 return __r;
4062 }
4063
4064 template <typename _Tp = void,
4065 typename
4066 = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4067 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4068 {
4069 value_type __x = _M_read();
4070 _M_write(--__x);
4071 return {_M_obj, _M_index};
4072 }
4073
4074 template <typename _Tp = void,
4075 typename
4076 = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4077 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(int) &&
4078 {
4079 const value_type __r = _M_read();
4080 value_type __x = __r;
4081 _M_write(--__x);
4082 return __r;
4083 }
4084
4085 _GLIBCXX_SIMD_INTRINSIC friend void
4086 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4087 conjunction<
4088 is_nothrow_constructible<value_type, _SmartReference&&>,
4089 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4090 {
4091 value_type __tmp = static_cast<_SmartReference&&>(__a);
4092 static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4093 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4094 }
4095
4096 _GLIBCXX_SIMD_INTRINSIC friend void
4097 swap(value_type& __a, _SmartReference&& __b) noexcept(
4098 conjunction<
4099 is_nothrow_constructible<value_type, value_type&&>,
4100 is_nothrow_assignable<value_type&, value_type&&>,
4101 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4102 {
4103 value_type __tmp(std::move(__a));
4104 __a = static_cast<value_type>(__b);
4105 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4106 }
4107
4108 _GLIBCXX_SIMD_INTRINSIC friend void
4109 swap(_SmartReference&& __a, value_type& __b) noexcept(
4110 conjunction<
4111 is_nothrow_constructible<value_type, _SmartReference&&>,
4112 is_nothrow_assignable<value_type&, value_type&&>,
4113 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4114 {
4115 value_type __tmp(__a);
4116 static_cast<_SmartReference&&>(__a) = std::move(__b);
4117 __b = std::move(__tmp);
4118 }
4119 };
4120
4121// }}}
4122// __scalar_abi_wrapper {{{
4123template <int _Bytes>
4124 struct __scalar_abi_wrapper
4125 {
4126 template <typename _Tp> static constexpr size_t _S_full_size = 1;
4127 template <typename _Tp> static constexpr size_t _S_size = 1;
4128 template <typename _Tp> static constexpr size_t _S_is_partial = false;
4129
4130 template <typename _Tp, typename _Abi = simd_abi::scalar>
4131 static constexpr bool _S_is_valid_v
4132 = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4133 };
4134
4135// }}}
4136// __decay_abi metafunction {{{
4137template <typename _Tp>
4138 struct __decay_abi { using type = _Tp; };
4139
4140template <int _Bytes>
4141 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4142 { using type = simd_abi::scalar; };
4143
4144// }}}
4145// __find_next_valid_abi metafunction {{{1
4146// Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4147// true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4148// recursion at 2 elements in the resulting ABI tag. In this case
4149// type::_S_is_valid_v<_Tp> may be false.
4150template <template <int> class _Abi, int _Bytes, typename _Tp>
4151 struct __find_next_valid_abi
4152 {
4153 static constexpr auto _S_choose()
4154 {
4155 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4156 using _NextAbi = _Abi<_NextBytes>;
4157 if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4158 return _Abi<_Bytes>();
4159 else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4160 && _NextAbi::template _S_is_valid_v<_Tp>)
4161 return _NextAbi();
4162 else
4163 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4164 }
4165
4166 using type = decltype(_S_choose());
4167 };
4168
4169template <int _Bytes, typename _Tp>
4170 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4171 { using type = simd_abi::scalar; };
4172
4173// _AbiList {{{1
4174template <template <int> class...>
4175 struct _AbiList
4176 {
4177 template <typename, int> static constexpr bool _S_has_valid_abi = false;
4178 template <typename, int> using _FirstValidAbi = void;
4179 template <typename, int> using _BestAbi = void;
4180 };
4181
4182template <template <int> class _A0, template <int> class... _Rest>
4183 struct _AbiList<_A0, _Rest...>
4184 {
4185 template <typename _Tp, int _Np>
4186 static constexpr bool _S_has_valid_abi
4187 = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4188 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4189
4190 template <typename _Tp, int _Np>
4191 using _FirstValidAbi = conditional_t<
4192 _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4193 typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4194 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4195
4196 template <typename _Tp, int _Np>
4197 static constexpr auto _S_determine_best_abi()
4198 {
4199 static_assert(_Np >= 1);
4200 constexpr int _Bytes = sizeof(_Tp) * _Np;
4201 if constexpr (_Np == 1)
4202 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4203 else
4204 {
4205 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4206 // _A0<_Bytes> is good if:
4207 // 1. The ABI tag is valid for _Tp
4208 // 2. The storage overhead is no more than padding to fill the next
4209 // power-of-2 number of bytes
4210 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4211 _Tp> && __fullsize / 2 < _Np)
4212 return typename __decay_abi<_A0<_Bytes>>::type{};
4213 else
4214 {
4215 using _Bp =
4216 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4217 if constexpr (_Bp::template _S_is_valid_v<
4218 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4219 return _Bp{};
4220 else
4221 return
4222 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4223 }
4224 }
4225 }
4226
4227 template <typename _Tp, int _Np>
4228 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4229 };
4230
4231// }}}1
4232
4233// the following lists all native ABIs, which makes them accessible to
4234// simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4235// matters: Whatever comes first has higher priority.
4236using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4237 __scalar_abi_wrapper>;
4238
4239// valid _SimdTraits specialization {{{1
4240template <typename _Tp, typename _Abi>
4241 struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4242 : _Abi::template __traits<_Tp> {};
4243
4244// __deduce_impl specializations {{{1
4245// try all native ABIs (including scalar) first
4246template <typename _Tp, size_t _Np>
4247 struct __deduce_impl<
4248 _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4249 { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4250
4251// fall back to fixed_size only if scalar and native ABIs don't match
4252template <typename _Tp, size_t _Np, typename = void>
4253 struct __deduce_fixed_size_fallback {};
4254
4255template <typename _Tp, size_t _Np>
4256 struct __deduce_fixed_size_fallback<_Tp, _Np,
4257 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4258 { using type = simd_abi::fixed_size<_Np>; };
4259
4260template <typename _Tp, size_t _Np, typename>
4261 struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4262
4263//}}}1
4264/// @endcond
4265
4266// simd_mask {{{
4267template <typename _Tp, typename _Abi>
4268 class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4269 {
4270 // types, tags, and friends {{{
4271 using _Traits = _SimdTraits<_Tp, _Abi>;
4272 using _MemberType = typename _Traits::_MaskMember;
4273
4274 // We map all masks with equal element sizeof to a single integer type, the
4275 // one given by __int_for_sizeof_t<_Tp>. This is the approach
4276 // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4277 // template specializations in the implementation classes.
4278 using _Ip = __int_for_sizeof_t<_Tp>;
4279 static constexpr _Ip* _S_type_tag = nullptr;
4280
4281 friend typename _Traits::_MaskBase;
4282 friend class simd<_Tp, _Abi>; // to construct masks on return
4283 friend typename _Traits::_SimdImpl; // to construct masks on return and
4284 // inspect data on masked operations
4285 public:
4286 using _Impl = typename _Traits::_MaskImpl;
4287 friend _Impl;
4288
4289 // }}}
4290 // member types {{{
4291 using value_type = bool;
4292 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4293 using simd_type = simd<_Tp, _Abi>;
4294 using abi_type = _Abi;
4295
4296 // }}}
4297 static constexpr size_t size() // {{{
4298 { return __size_or_zero_v<_Tp, _Abi>; }
4299
4300 // }}}
4301 // constructors & assignment {{{
4302 simd_mask() = default;
4303 simd_mask(const simd_mask&) = default;
4304 simd_mask(simd_mask&&) = default;
4305 simd_mask& operator=(const simd_mask&) = default;
4306 simd_mask& operator=(simd_mask&&) = default;
4307
4308 // }}}
4309 // access to internal representation (optional feature) {{{
4310 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
4311 simd_mask(typename _Traits::_MaskCastType __init)
4312 : _M_data{__init} {}
4313 // conversions to internal type is done in _MaskBase
4314
4315 // }}}
4316 // bitset interface (extension to be proposed) {{{
4317 // TS_FEEDBACK:
4318 // Conversion of simd_mask to and from bitset makes it much easier to
4319 // interface with other facilities. I suggest adding `static
4320 // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4321 _GLIBCXX_SIMD_ALWAYS_INLINE static simd_mask
4322 __from_bitset(bitset<size()> bs)
4323 { return {__bitset_init, bs}; }
4324
4325 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4326 __to_bitset() const
4327 { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4328
4329 // }}}
4330 // explicit broadcast constructor {{{
4331 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4332 simd_mask(value_type __x)
4333 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4334
4335 // }}}
4336 // implicit type conversion constructor {{{
4337 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4338 // proposed improvement
4339 template <typename _Up, typename _A2,
4340 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4341 _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4342 != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4343 simd_mask(const simd_mask<_Up, _A2>& __x)
4344 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4345 #else
4346 // conforming to ISO/IEC 19570:2018
4347 template <typename _Up, typename = enable_if_t<conjunction<
4348 is_same<abi_type, simd_abi::fixed_size<size()>>,
4349 is_same<_Up, _Up>>::value>>
4350 _GLIBCXX_SIMD_ALWAYS_INLINE
4351 simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4352 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4353 #endif
4354
4355 // }}}
4356 // load constructor {{{
4357 template <typename _Flags>
4358 _GLIBCXX_SIMD_ALWAYS_INLINE
4359 simd_mask(const value_type* __mem, _Flags)
4360 : _M_data(_Impl::template _S_load<_Ip>(
4361 _Flags::template _S_apply<simd_mask>(__mem))) {}
4362
4363 template <typename _Flags>
4364 _GLIBCXX_SIMD_ALWAYS_INLINE
4365 simd_mask(const value_type* __mem, simd_mask __k, _Flags)
4366 : _M_data{}
4367 {
4368 _M_data
4369 = _Impl::_S_masked_load(_M_data, __k._M_data,
4370 _Flags::template _S_apply<simd_mask>(__mem));
4371 }
4372
4373 // }}}
4374 // loads [simd_mask.load] {{{
4375 template <typename _Flags>
4376 _GLIBCXX_SIMD_ALWAYS_INLINE void
4377 copy_from(const value_type* __mem, _Flags)
4378 {
4379 _M_data = _Impl::template _S_load<_Ip>(
4380 _Flags::template _S_apply<simd_mask>(__mem));
4381 }
4382
4383 // }}}
4384 // stores [simd_mask.store] {{{
4385 template <typename _Flags>
4386 _GLIBCXX_SIMD_ALWAYS_INLINE void
4387 copy_to(value_type* __mem, _Flags) const
4388 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4389
4390 // }}}
4391 // scalar access {{{
4392 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4393 operator[](size_t __i)
4394 {
4395 if (__i >= size())
4396 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4397 return {_M_data, int(__i)};
4398 }
4399
4400 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4401 operator[](size_t __i) const
4402 {
4403 if (__i >= size())
4404 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4405 if constexpr (__is_scalar_abi<_Abi>())
4406 return _M_data;
4407 else
4408 return static_cast<bool>(_M_data[__i]);
4409 }
4410
4411 // }}}
4412 // negation {{{
4413 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4414 operator!() const
4415 { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4416
4417 // }}}
4418 // simd_mask binary operators [simd_mask.binary] {{{
4419 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4420 // simd_mask<int> && simd_mask<uint> needs disambiguation
4421 template <typename _Up, typename _A2,
4422 typename
4423 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4424 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4425 operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4426 {
4427 return {__private_init,
4428 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4429 }
4430
4431 template <typename _Up, typename _A2,
4432 typename
4433 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4434 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4435 operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4436 {
4437 return {__private_init,
4438 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4439 }
4440 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4441
4442 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4443 operator&&(const simd_mask& __x, const simd_mask& __y)
4444 {
4445 return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4446 }
4447
4448 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4449 operator||(const simd_mask& __x, const simd_mask& __y)
4450 {
4451 return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4452 }
4453
4454 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4455 operator&(const simd_mask& __x, const simd_mask& __y)
4456 { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4457
4458 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4459 operator|(const simd_mask& __x, const simd_mask& __y)
4460 { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4461
4462 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4463 operator^(const simd_mask& __x, const simd_mask& __y)
4464 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4465
4466 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4467 operator&=(simd_mask& __x, const simd_mask& __y)
4468 {
4469 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4470 return __x;
4471 }
4472
4473 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4474 operator|=(simd_mask& __x, const simd_mask& __y)
4475 {
4476 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4477 return __x;
4478 }
4479
4480 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4481 operator^=(simd_mask& __x, const simd_mask& __y)
4482 {
4483 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4484 return __x;
4485 }
4486
4487 // }}}
4488 // simd_mask compares [simd_mask.comparison] {{{
4489 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4490 operator==(const simd_mask& __x, const simd_mask& __y)
4491 { return !operator!=(__x, __y); }
4492
4493 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4494 operator!=(const simd_mask& __x, const simd_mask& __y)
4495 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4496
4497 // }}}
4498 // private_init ctor {{{
4499 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4500 simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4501 : _M_data(__init) {}
4502
4503 // }}}
4504 // private_init generator ctor {{{
4505 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4506 _GLIBCXX_SIMD_INTRINSIC constexpr
4507 simd_mask(_PrivateInit, _Fp&& __gen)
4508 : _M_data()
4509 {
4510 __execute_n_times<size()>([&](auto __i) constexpr {
4511 _Impl::_S_set(_M_data, __i, __gen(__i));
4512 });
4513 }
4514
4515 // }}}
4516 // bitset_init ctor {{{
4517 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4518 : _M_data(
4519 _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4520 {}
4521
4522 // }}}
4523 // __cvt {{{
4524 // TS_FEEDBACK:
4525 // The conversion operator this implements should be a ctor on simd_mask.
4526 // Once you call .__cvt() on a simd_mask it converts conveniently.
4527 // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4528 struct _CvtProxy
4529 {
4530 template <typename _Up, typename _A2,
4531 typename
4532 = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4533 operator simd_mask<_Up, _A2>() &&
4534 {
4535 using namespace std::experimental::__proposed;
4536 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4537 }
4538
4539 const simd_mask<_Tp, _Abi>& _M_data;
4540 };
4541
4542 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4543 __cvt() const
4544 { return {*this}; }
4545
4546 // }}}
4547 // operator?: overloads (suggested extension) {{{
4548 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4549 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4550 operator?:(const simd_mask& __k, const simd_mask& __where_true,
4551 const simd_mask& __where_false)
4552 {
4553 auto __ret = __where_false;
4554 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4555 return __ret;
4556 }
4557
4558 template <typename _U1, typename _U2,
4559 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4560 typename = enable_if_t<conjunction_v<
4561 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4562 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4563 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4564 operator?:(const simd_mask& __k, const _U1& __where_true,
4565 const _U2& __where_false)
4566 {
4567 _Rp __ret = __where_false;
4568 _Rp::_Impl::_S_masked_assign(
4569 __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4570 __data(static_cast<_Rp>(__where_true)));
4571 return __ret;
4572 }
4573
4574 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4575 template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4576 typename = enable_if_t<
4577 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4578 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4579 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4580 operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4581 const simd_mask<_Up, _Au>& __where_false)
4582 {
4583 simd_mask __ret = __where_false;
4584 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4585 __where_true._M_data);
4586 return __ret;
4587 }
4588 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4589 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4590
4591 // }}}
4592 // _M_is_constprop {{{
4593 _GLIBCXX_SIMD_INTRINSIC constexpr bool
4594 _M_is_constprop() const
4595 {
4596 if constexpr (__is_scalar_abi<_Abi>())
4597 return __builtin_constant_p(_M_data);
4598 else
4599 return _M_data._M_is_constprop();
4600 }
4601
4602 // }}}
4603
4604 private:
4605 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4606 friend auto& __data<_Tp, abi_type>(simd_mask&);
4607 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4608 };
4609
4610// }}}
4611
4612/// @cond undocumented
4613// __data(simd_mask) {{{
4614template <typename _Tp, typename _Ap>
4615 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4616 __data(const simd_mask<_Tp, _Ap>& __x)
4617 { return __x._M_data; }
4618
4619template <typename _Tp, typename _Ap>
4620 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4621 __data(simd_mask<_Tp, _Ap>& __x)
4622 { return __x._M_data; }
4623
4624// }}}
4625/// @endcond
4626
4627// simd_mask reductions [simd_mask.reductions] {{{
4628template <typename _Tp, typename _Abi>
4629 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4630 all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4631 {
4632 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4633 {
4634 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4635 if (!__k[__i])
4636 return false;
4637 return true;
4638 }
4639 else
4640 return _Abi::_MaskImpl::_S_all_of(__k);
4641 }
4642
4643template <typename _Tp, typename _Abi>
4644 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4645 any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4646 {
4647 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4648 {
4649 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4650 if (__k[__i])
4651 return true;
4652 return false;
4653 }
4654 else
4655 return _Abi::_MaskImpl::_S_any_of(__k);
4656 }
4657
4658template <typename _Tp, typename _Abi>
4659 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4660 none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4661 {
4662 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4663 {
4664 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4665 if (__k[__i])
4666 return false;
4667 return true;
4668 }
4669 else
4670 return _Abi::_MaskImpl::_S_none_of(__k);
4671 }
4672
4673template <typename _Tp, typename _Abi>
4674 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4675 some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4676 {
4677 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4678 {
4679 for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4680 if (__k[__i] != __k[__i - 1])
4681 return true;
4682 return false;
4683 }
4684 else
4685 return _Abi::_MaskImpl::_S_some_of(__k);
4686 }
4687
4688template <typename _Tp, typename _Abi>
4689 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4690 popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4691 {
4692 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4693 {
4694 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4695 __k, [](auto... __elements) { return ((__elements != 0) + ...); });
4696 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4697 return __r;
4698 }
4699 return _Abi::_MaskImpl::_S_popcount(__k);
4700 }
4701
4702template <typename _Tp, typename _Abi>
4703 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4704 find_first_set(const simd_mask<_Tp, _Abi>& __k)
4705 {
4706 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4707 {
4708 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4709 const size_t _Idx = __call_with_n_evaluations<_Np>(
4710 [](auto... __indexes) { return std::min({__indexes...}); },
4711 [&](auto __i) { return __k[__i] ? +__i : _Np; });
4712 if (_Idx >= _Np)
4713 __invoke_ub("find_first_set(empty mask) is UB");
4714 if (__builtin_constant_p(_Idx))
4715 return _Idx;
4716 }
4717 return _Abi::_MaskImpl::_S_find_first_set(__k);
4718 }
4719
4720template <typename _Tp, typename _Abi>
4721 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4722 find_last_set(const simd_mask<_Tp, _Abi>& __k)
4723 {
4724 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4725 {
4726 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4727 const int _Idx = __call_with_n_evaluations<_Np>(
4728 [](auto... __indexes) { return std::max({__indexes...}); },
4729 [&](auto __i) { return __k[__i] ? int(__i) : -1; });
4730 if (_Idx < 0)
4731 __invoke_ub("find_first_set(empty mask) is UB");
4732 if (__builtin_constant_p(_Idx))
4733 return _Idx;
4734 }
4735 return _Abi::_MaskImpl::_S_find_last_set(__k);
4736 }
4737
4738_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4739all_of(_ExactBool __x) noexcept
4740{ return __x; }
4741
4742_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4743any_of(_ExactBool __x) noexcept
4744{ return __x; }
4745
4746_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4747none_of(_ExactBool __x) noexcept
4748{ return !__x; }
4749
4750_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4751some_of(_ExactBool) noexcept
4752{ return false; }
4753
4754_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4755popcount(_ExactBool __x) noexcept
4756{ return __x; }
4757
4758_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4759find_first_set(_ExactBool)
4760{ return 0; }
4761
4762_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4763find_last_set(_ExactBool)
4764{ return 0; }
4765
4766// }}}
4767
4768/// @cond undocumented
4769// _SimdIntOperators{{{1
4770template <typename _V, typename _Impl, bool>
4771 class _SimdIntOperators {};
4772
4773template <typename _V, typename _Impl>
4774 class _SimdIntOperators<_V, _Impl, true>
4775 {
4776 _GLIBCXX_SIMD_INTRINSIC const _V& __derived() const
4777 { return *static_cast<const _V*>(this); }
4778
4779 template <typename _Tp>
4780 _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4781 _S_make_derived(_Tp&& __d)
4782 { return {__private_init, static_cast<_Tp&&>(__d)}; }
4783
4784 public:
4785 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x)
4786 { return __lhs = __lhs % __x; }
4787
4788 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x)
4789 { return __lhs = __lhs & __x; }
4790
4791 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x)
4792 { return __lhs = __lhs | __x; }
4793
4794 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x)
4795 { return __lhs = __lhs ^ __x; }
4796
4797 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x)
4798 { return __lhs = __lhs << __x; }
4799
4800 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x)
4801 { return __lhs = __lhs >> __x; }
4802
4803 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x)
4804 { return __lhs = __lhs << __x; }
4805
4806 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x)
4807 { return __lhs = __lhs >> __x; }
4808
4809 _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y)
4810 {
4811 return _SimdIntOperators::_S_make_derived(
4812 _Impl::_S_modulus(__data(__x), __data(__y)));
4813 }
4814
4815 _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y)
4816 {
4817 return _SimdIntOperators::_S_make_derived(
4818 _Impl::_S_bit_and(__data(__x), __data(__y)));
4819 }
4820
4821 _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y)
4822 {
4823 return _SimdIntOperators::_S_make_derived(
4824 _Impl::_S_bit_or(__data(__x), __data(__y)));
4825 }
4826
4827 _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y)
4828 {
4829 return _SimdIntOperators::_S_make_derived(
4830 _Impl::_S_bit_xor(__data(__x), __data(__y)));
4831 }
4832
4833 _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y)
4834 {
4835 return _SimdIntOperators::_S_make_derived(
4836 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4837 }
4838
4839 _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y)
4840 {
4841 return _SimdIntOperators::_S_make_derived(
4842 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4843 }
4844
4845 template <typename _VV = _V>
4846 _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y)
4847 {
4848 using _Tp = typename _VV::value_type;
4849 if (__y < 0)
4850 __invoke_ub("The behavior is undefined if the right operand of a "
4851 "shift operation is negative. [expr.shift]\nA shift by "
4852 "%d was requested",
4853 __y);
4854 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4855 __invoke_ub(
4856 "The behavior is undefined if the right operand of a "
4857 "shift operation is greater than or equal to the width of the "
4858 "promoted left operand. [expr.shift]\nA shift by %d was requested",
4859 __y);
4860 return _SimdIntOperators::_S_make_derived(
4861 _Impl::_S_bit_shift_left(__data(__x), __y));
4862 }
4863
4864 template <typename _VV = _V>
4865 _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y)
4866 {
4867 using _Tp = typename _VV::value_type;
4868 if (__y < 0)
4869 __invoke_ub(
4870 "The behavior is undefined if the right operand of a shift "
4871 "operation is negative. [expr.shift]\nA shift by %d was requested",
4872 __y);
4873 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4874 __invoke_ub(
4875 "The behavior is undefined if the right operand of a shift "
4876 "operation is greater than or equal to the width of the promoted "
4877 "left operand. [expr.shift]\nA shift by %d was requested",
4878 __y);
4879 return _SimdIntOperators::_S_make_derived(
4880 _Impl::_S_bit_shift_right(__data(__x), __y));
4881 }
4882
4883 // unary operators (for integral _Tp)
4884 _GLIBCXX_SIMD_CONSTEXPR _V operator~() const
4885 { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
4886 };
4887
4888//}}}1
4889/// @endcond
4890
4891// simd {{{
4892template <typename _Tp, typename _Abi>
4893 class simd : public _SimdIntOperators<
4894 simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
4895 conjunction<is_integral<_Tp>,
4896 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
4897 public _SimdTraits<_Tp, _Abi>::_SimdBase
4898 {
4899 using _Traits = _SimdTraits<_Tp, _Abi>;
4900 using _MemberType = typename _Traits::_SimdMember;
4901 using _CastType = typename _Traits::_SimdCastType;
4902 static constexpr _Tp* _S_type_tag = nullptr;
4903 friend typename _Traits::_SimdBase;
4904
4905 public:
4906 using _Impl = typename _Traits::_SimdImpl;
4907 friend _Impl;
4908 friend _SimdIntOperators<simd, _Impl, true>;
4909
4910 using value_type = _Tp;
4911 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4912 using mask_type = simd_mask<_Tp, _Abi>;
4913 using abi_type = _Abi;
4914
4915 static constexpr size_t size()
4916 { return __size_or_zero_v<_Tp, _Abi>; }
4917
4918 _GLIBCXX_SIMD_CONSTEXPR simd() = default;
4919 _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
4920 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
4921 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
4922 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
4923
4924 // implicit broadcast constructor
4925 template <typename _Up,
4926 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
4927 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4928 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
4929 : _M_data(
4930 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
4931 {}
4932
4933 // implicit type conversion constructor (convert from fixed_size to
4934 // fixed_size)
4935 template <typename _Up>
4936 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4937 simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
4939 conjunction<
4940 is_same<simd_abi::fixed_size<size()>, abi_type>,
4941 negation<__is_narrowing_conversion<_Up, value_type>>,
4942 __converts_to_higher_integer_rank<_Up, value_type>>::value,
4943 void*> = nullptr)
4944 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
4945
4946 // explicit type conversion constructor
4947#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
4948 template <typename _Up, typename _A2,
4949 typename = decltype(static_simd_cast<simd>(
4950 declval<const simd<_Up, _A2>&>()))>
4951 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4952 simd(const simd<_Up, _A2>& __x)
4953 : simd(static_simd_cast<simd>(__x)) {}
4954#endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
4955
4956 // generator constructor
4957 template <typename _Fp>
4958 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4959 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
4960 declval<_SizeConstant<0>&>())),
4961 value_type>* = nullptr)
4962 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
4963
4964 // load constructor
4965 template <typename _Up, typename _Flags>
4966 _GLIBCXX_SIMD_ALWAYS_INLINE
4967 simd(const _Up* __mem, _Flags)
4968 : _M_data(
4969 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
4970 {}
4971
4972 // loads [simd.load]
4973 template <typename _Up, typename _Flags>
4974 _GLIBCXX_SIMD_ALWAYS_INLINE void
4975 copy_from(const _Vectorizable<_Up>* __mem, _Flags)
4976 {
4977 _M_data = static_cast<decltype(_M_data)>(
4978 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
4979 }
4980
4981 // stores [simd.store]
4982 template <typename _Up, typename _Flags>
4983 _GLIBCXX_SIMD_ALWAYS_INLINE void
4984 copy_to(_Vectorizable<_Up>* __mem, _Flags) const
4985 {
4986 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
4987 _S_type_tag);
4988 }
4989
4990 // scalar access
4991 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4992 operator[](size_t __i)
4993 { return {_M_data, int(__i)}; }
4994
4995 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4996 operator[]([[maybe_unused]] size_t __i) const
4997 {
4998 if constexpr (__is_scalar_abi<_Abi>())
4999 {
5000 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5001 return _M_data;
5002 }
5003 else
5004 return _M_data[__i];
5005 }
5006
5007 // increment and decrement:
5008 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5009 operator++()
5010 {
5011 _Impl::_S_increment(_M_data);
5012 return *this;
5013 }
5014
5015 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5016 operator++(int)
5017 {
5018 simd __r = *this;
5019 _Impl::_S_increment(_M_data);
5020 return __r;
5021 }
5022
5023 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5024 operator--()
5025 {
5026 _Impl::_S_decrement(_M_data);
5027 return *this;
5028 }
5029
5030 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5031 operator--(int)
5032 {
5033 simd __r = *this;
5034 _Impl::_S_decrement(_M_data);
5035 return __r;
5036 }
5037
5038 // unary operators (for any _Tp)
5039 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5040 operator!() const
5041 { return {__private_init, _Impl::_S_negate(_M_data)}; }
5042
5043 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5044 operator+() const
5045 { return *this; }
5046
5047 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5048 operator-() const
5049 { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5050
5051 // access to internal representation (suggested extension)
5052 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5053 simd(_CastType __init) : _M_data(__init) {}
5054
5055 // compound assignment [simd.cassign]
5056 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5057 operator+=(simd& __lhs, const simd& __x)
5058 { return __lhs = __lhs + __x; }
5059
5060 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5061 operator-=(simd& __lhs, const simd& __x)
5062 { return __lhs = __lhs - __x; }
5063
5064 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5065 operator*=(simd& __lhs, const simd& __x)
5066 { return __lhs = __lhs * __x; }
5067
5068 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5069 operator/=(simd& __lhs, const simd& __x)
5070 { return __lhs = __lhs / __x; }
5071
5072 // binary operators [simd.binary]
5073 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5074 operator+(const simd& __x, const simd& __y)
5075 { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5076
5077 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5078 operator-(const simd& __x, const simd& __y)
5079 { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5080
5081 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5082 operator*(const simd& __x, const simd& __y)
5083 { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5084
5085 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5086 operator/(const simd& __x, const simd& __y)
5087 { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5088
5089 // compares [simd.comparison]
5090 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5091 operator==(const simd& __x, const simd& __y)
5092 { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5093
5094 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5095 operator!=(const simd& __x, const simd& __y)
5096 {
5097 return simd::_S_make_mask(
5098 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5099 }
5100
5101 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5102 operator<(const simd& __x, const simd& __y)
5103 { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5104
5105 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5106 operator<=(const simd& __x, const simd& __y)
5107 {
5108 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5109 }
5110
5111 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5112 operator>(const simd& __x, const simd& __y)
5113 { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5114
5115 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5116 operator>=(const simd& __x, const simd& __y)
5117 {
5118 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5119 }
5120
5121 // operator?: overloads (suggested extension) {{{
5122#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5123 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5124 operator?:(const mask_type& __k, const simd& __where_true,
5125 const simd& __where_false)
5126 {
5127 auto __ret = __where_false;
5128 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5129 return __ret;
5130 }
5131
5132#endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5133 // }}}
5134
5135 // "private" because of the first arguments's namespace
5136 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5137 simd(_PrivateInit, const _MemberType& __init)
5138 : _M_data(__init) {}
5139
5140 // "private" because of the first arguments's namespace
5141 _GLIBCXX_SIMD_INTRINSIC
5142 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5143 { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5144
5145 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5146 _M_is_constprop() const
5147 {
5148 if constexpr (__is_scalar_abi<_Abi>())
5149 return __builtin_constant_p(_M_data);
5150 else
5151 return _M_data._M_is_constprop();
5152 }
5153
5154 private:
5155 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5156 _S_make_mask(typename mask_type::_MemberType __k)
5157 { return {__private_init, __k}; }
5158
5159 friend const auto& __data<value_type, abi_type>(const simd&);
5160 friend auto& __data<value_type, abi_type>(simd&);
5161 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5162 };
5163
5164// }}}
5165/// @cond undocumented
5166// __data {{{
5167template <typename _Tp, typename _Ap>
5168 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5169 __data(const simd<_Tp, _Ap>& __x)
5170 { return __x._M_data; }
5171
5172template <typename _Tp, typename _Ap>
5173 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5174 __data(simd<_Tp, _Ap>& __x)
5175 { return __x._M_data; }
5176
5177// }}}
5178namespace __float_bitwise_operators { //{{{
5179template <typename _Tp, typename _Ap>
5180 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5181 operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5182 {
5183 return {__private_init,
5184 _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5185 }
5186
5187template <typename _Tp, typename _Ap>
5188 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5189 operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5190 {
5191 return {__private_init,
5192 _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5193 }
5194
5195template <typename _Tp, typename _Ap>
5196 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5197 operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5198 {
5199 return {__private_init,
5200 _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5201 }
5202} // namespace __float_bitwise_operators }}}
5203/// @endcond
5204
5205/// @}
5206_GLIBCXX_SIMD_END_NAMESPACE
5207
5208#endif // __cplusplus >= 201703L
5209#endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5210
5211// vim: foldmethod=marker foldmarker={{{,}}}
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: std/chrono:805
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: std/chrono:819
constexpr bool operator!=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: std/chrono:797
constexpr duration< __common_rep_t< _Rep2, _Rep1 >, _Period > operator*(const _Rep1 &__s, const duration< _Rep2, _Period > &__d)
Definition: std/chrono:700
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: std/chrono:729
constexpr time_point< _Clock, typename common_type< duration< _Rep1, _Period1 >, _Dur2 >::type > operator+(const duration< _Rep1, _Period1 > &__lhs, const time_point< _Clock, _Dur2 > &__rhs)
Adjust a time point forwards by the given duration.
Definition: std/chrono:1016
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: std/chrono:772
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: std/chrono:812
constexpr common_type< duration< _Rep1, _Period1 >, duration< _Rep2, _Period2 > >::type operator-(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
The difference between two durations.
Definition: std/chrono:660
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator/(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: std/chrono:706
basic_istream< _CharT, _Traits > & operator>>(basic_istream< _CharT, _Traits > &__is, complex< _Tp > &__x)
Extraction operator for complex values.
Definition: std/complex:501
basic_ostream< _CharT, _Traits > & operator<<(basic_ostream< _CharT, _Traits > &__os, const complex< _Tp > &__x)
Insertion operator for complex values.
Definition: std/complex:555
constexpr bool operator==(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return true if x is equal to y.
Definition: std/complex:464
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: std/type_traits:83
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: std/type_traits:86
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition: std/tuple:1736
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
void swap(any &__x, any &__y) noexcept
Exchange the states of two any objects.
Definition: std/any:428
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
Definition: valarray:1237
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
Definition: valarray:1215
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
Definition: stl_algo.h:3656
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:254
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
Definition: stl_algo.h:3301
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:230
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: std/numeric:278
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: std/bitset:1435
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition: range_access.h:245
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: std/bitset:1444
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition: range_access.h:290
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: std/bitset:1453