libstdc++
simd_vec.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_VEC_H
26#define _GLIBCXX_SIMD_VEC_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_mask.h"
35#include "simd_flags.h"
36
37#include <bits/utility.h>
38#include <bits/stl_function.h>
39#include <cmath>
40
41// psabi warnings are bogus because the ABI of the internal types never leaks into user code
42#pragma GCC diagnostic push
43#pragma GCC diagnostic ignored "-Wpsabi"
44
45namespace std _GLIBCXX_VISIBILITY(default)
46{
47_GLIBCXX_BEGIN_NAMESPACE_VERSION
48namespace simd
49{
50 // disabled basic_vec
51 template <typename _Tp, typename _Ap>
52 class basic_vec
53 {
54 public:
55 using value_type = _Tp;
56
57 using abi_type = _Ap;
58
59 using mask_type = basic_mask<0, void>; // disabled
60
61#define _GLIBCXX_DELETE_SIMD "This specialization is disabled because of an invalid combination " \
62 "of template arguments to basic_vec."
63
64 basic_vec() = delete(_GLIBCXX_DELETE_SIMD);
65
66 ~basic_vec() = delete(_GLIBCXX_DELETE_SIMD);
67
68 basic_vec(const basic_vec&) = delete(_GLIBCXX_DELETE_SIMD);
69
70 basic_vec& operator=(const basic_vec&) = delete(_GLIBCXX_DELETE_SIMD);
71
72#undef _GLIBCXX_DELETE_SIMD
73 };
74
75 template <typename _Tp, typename _Ap>
76 class _VecBase
77 {
78 using _Vp = basic_vec<_Tp, _Ap>;
79
80 public:
81 using value_type = _Tp;
82
83 using abi_type = _Ap;
84
85 using mask_type = basic_mask<sizeof(_Tp), abi_type>;
86
87 using iterator = __iterator<_Vp>;
88
89 using const_iterator = __iterator<const _Vp>;
90
91 constexpr iterator
92 begin() noexcept
93 { return {static_cast<_Vp&>(*this), 0}; }
94
95 constexpr const_iterator
96 begin() const noexcept
97 { return cbegin(); }
98
99 constexpr const_iterator
100 cbegin() const noexcept
101 { return {static_cast<const _Vp&>(*this), 0}; }
102
103 constexpr default_sentinel_t
104 end() const noexcept
105 { return {}; }
106
107 constexpr default_sentinel_t
108 cend() const noexcept
109 { return {}; }
110
111 static constexpr auto size = __simd_size_c<_Ap::_S_size>;
112
113 _VecBase() = default;
114
115 // LWG issue from 2026-03-04 / P4042R0
116 template <typename _Up, typename _UAbi>
117 requires (_Ap::_S_size != _UAbi::_S_size)
118 _VecBase(const basic_vec<_Up, _UAbi>&) = delete("size mismatch");
119
120 template <typename _Up, typename _UAbi>
121 requires (_Ap::_S_size == _UAbi::_S_size) && (!__explicitly_convertible_to<_Up, _Tp>)
122 explicit
123 _VecBase(const basic_vec<_Up, _UAbi>&)
124 = delete("the value types are not convertible");
125
126 [[__gnu__::__always_inline__]]
127 friend constexpr _Vp
128 operator+(const _Vp& __x, const _Vp& __y) noexcept
129 {
130 _Vp __r = __x;
131 __r += __y;
132 return __r;
133 }
134
135 [[__gnu__::__always_inline__]]
136 friend constexpr _Vp
137 operator-(const _Vp& __x, const _Vp& __y) noexcept
138 {
139 _Vp __r = __x;
140 __r -= __y;
141 return __r;
142 }
143
144 [[__gnu__::__always_inline__]]
145 friend constexpr _Vp
146 operator*(const _Vp& __x, const _Vp& __y) noexcept
147 {
148 _Vp __r = __x;
149 __r *= __y;
150 return __r;
151 }
152
153 [[__gnu__::__always_inline__]]
154 friend constexpr _Vp
155 operator/(const _Vp& __x, const _Vp& __y) noexcept
156 {
157 _Vp __r = __x;
158 __r /= __y;
159 return __r;
160 }
161
162 [[__gnu__::__always_inline__]]
163 friend constexpr _Vp
164 operator%(const _Vp& __x, const _Vp& __y) noexcept
165 requires requires (_Tp __a) { __a % __a; }
166 {
167 _Vp __r = __x;
168 __r %= __y;
169 return __r;
170 }
171
172 [[__gnu__::__always_inline__]]
173 friend constexpr _Vp
174 operator&(const _Vp& __x, const _Vp& __y) noexcept
175 requires requires (_Tp __a) { __a & __a; }
176 {
177 _Vp __r = __x;
178 __r &= __y;
179 return __r;
180 }
181
182 [[__gnu__::__always_inline__]]
183 friend constexpr _Vp
184 operator|(const _Vp& __x, const _Vp& __y) noexcept
185 requires requires (_Tp __a) { __a | __a; }
186 {
187 _Vp __r = __x;
188 __r |= __y;
189 return __r;
190 }
191
192 [[__gnu__::__always_inline__]]
193 friend constexpr _Vp
194 operator^(const _Vp& __x, const _Vp& __y) noexcept
195 requires requires (_Tp __a) { __a ^ __a; }
196 {
197 _Vp __r = __x;
198 __r ^= __y;
199 return __r;
200 }
201
202 [[__gnu__::__always_inline__]]
203 friend constexpr _Vp
204 operator<<(const _Vp& __x, const _Vp& __y) _GLIBCXX_SIMD_NOEXCEPT
205 requires requires (_Tp __a) { __a << __a; }
206 {
207 _Vp __r = __x;
208 __r <<= __y;
209 return __r;
210 }
211
212 [[__gnu__::__always_inline__]]
213 friend constexpr _Vp
214 operator<<(const _Vp& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
215 requires requires (_Tp __a, __simd_size_type __b) { __a << __b; }
216 {
217 _Vp __r = __x;
218 __r <<= __y;
219 return __r;
220 }
221
222 [[__gnu__::__always_inline__]]
223 friend constexpr _Vp
224 operator>>(const _Vp& __x, const _Vp& __y) _GLIBCXX_SIMD_NOEXCEPT
225 requires requires (_Tp __a) { __a >> __a; }
226 {
227 _Vp __r = __x;
228 __r >>= __y;
229 return __r;
230 }
231
232 [[__gnu__::__always_inline__]]
233 friend constexpr _Vp
234 operator>>(const _Vp& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
235 requires requires (_Tp __a, __simd_size_type __b) { __a >> __b; }
236 {
237 _Vp __r = __x;
238 __r >>= __y;
239 return __r;
240 }
241 };
242
243 struct _LoadCtorTag
244 {};
245
246 template <integral _Tp>
247 inline constexpr _Tp __max_shift
248 = (sizeof(_Tp) < sizeof(int) ? sizeof(int) : sizeof(_Tp)) * __CHAR_BIT__;
249
250 template <__vectorizable _Tp, __abi_tag _Ap>
251 requires (_Ap::_S_nreg == 1)
252 class basic_vec<_Tp, _Ap>
253 : public _VecBase<_Tp, _Ap>
254 {
255 template <typename, typename>
256 friend class basic_vec;
257
258 template <size_t, typename>
259 friend class basic_mask;
260
261 static constexpr int _S_size = _Ap::_S_size;
262
263 static constexpr int _S_full_size = __bit_ceil(unsigned(_S_size));
264
265 static constexpr bool _S_is_scalar = _S_size == 1;
266
267 static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask && !_S_is_scalar;
268
269 using _DataType = typename _Ap::template _DataType<_Tp>;
270
271 /** @internal
272 * @brief Underlying vector data storage.
273 *
274 * This member holds the vector object using a GNU vector type or a platform-specific vector
275 * type determined by the ABI tag. For size 1 vectors, this is a single value (_Tp).
276 */
277 _DataType _M_data;
278
279 static constexpr bool _S_is_partial = sizeof(_M_data) > sizeof(_Tp) * _S_size;
280
281 using __canon_value_type = __canonical_vec_type_t<_Tp>;
282
283 public:
284 using value_type = _Tp;
285
286 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
287
288 // internal but public API ----------------------------------------------
289 [[__gnu__::__always_inline__]]
290 static constexpr basic_vec
291 _S_init(_DataType __x)
292 {
293 basic_vec __r;
294 __r._M_data = __x;
295 return __r;
296 }
297
298 [[__gnu__::__always_inline__]]
299 constexpr const _DataType&
300 _M_get() const
301 { return _M_data; }
302
303 [[__gnu__::__always_inline__]]
304 friend constexpr bool
305 __is_const_known(const basic_vec& __x)
306 { return __builtin_constant_p(__x._M_data); }
307
308 [[__gnu__::__always_inline__]]
309 constexpr auto
310 _M_concat_data([[maybe_unused]] bool __do_sanitize = false) const
311 {
312 if constexpr (_S_is_scalar)
313 return __vec_builtin_type<__canon_value_type, 1>{_M_data};
314 else
315 return _M_data;
316 }
317
318 template <int _Size = _S_size, int _Offset = 0, typename _A0, typename _Fp>
319 [[__gnu__::__always_inline__]]
320 static constexpr basic_vec
321 _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&& __idxmap)
322 {
323 using _Xp = basic_vec<value_type, _A0>;
324 basic_vec __r;
325 if constexpr (_S_is_scalar)
326 {
327 constexpr __simd_size_type __j = [&] consteval {
328 if constexpr (__index_permutation_function_sized<_Fp>)
329 return __idxmap(_Offset, _Size);
330 else
331 return __idxmap(_Offset);
332 }();
333 if constexpr (__j == simd::zero_element || __j == simd::uninit_element)
334 return basic_vec();
335 else
336 static_assert(__j >= 0 && __j < _Xp::_S_size);
337 __r._M_data = __x[__j];
338 }
339 else
340 {
341 auto __idxmap2 = [=](auto __i) consteval {
342 if constexpr (int(__i + _Offset) >= _Size) // _S_full_size > _Size
343 return __simd_size_c<simd::uninit_element>;
344 else if constexpr (__index_permutation_function_sized<_Fp>)
345 return __simd_size_c<__idxmap(__i + _Offset, _Size)>;
346 else
347 return __simd_size_c<__idxmap(__i + _Offset)>;
348 };
349 constexpr auto __adj_idx = [](auto __i) {
350 constexpr int __j = __i;
351 if constexpr (__j == simd::zero_element)
352 return __simd_size_c<__bit_ceil(unsigned(_Xp::_S_size))>;
353 else if constexpr (__j == simd::uninit_element)
354 return __simd_size_c<-1>;
355 else
356 {
357 static_assert(__j >= 0 && __j < _Xp::_S_size);
358 return __simd_size_c<__j>;
359 }
360 };
361 constexpr auto [...__is0] = _IotaArray<_S_size>;
362 constexpr bool __needs_zero_element
363 = ((__idxmap2(__simd_size_c<__is0>).value == simd::zero_element) || ...);
364 constexpr auto [...__is_full] = _IotaArray<_S_full_size>;
365 if constexpr (_A0::_S_nreg == 2 && !__needs_zero_element)
366 {
367 __r._M_data = __builtin_shufflevector(
368 __x._M_data0._M_data, __x._M_data1._M_data,
369 __adj_idx(__idxmap2(__simd_size_c<__is_full>)).value...);
370 }
371 else
372 {
373 __r._M_data = __builtin_shufflevector(
374 __x._M_concat_data(), decltype(__x._M_concat_data())(),
375 __adj_idx(__idxmap2(__simd_size_c<__is_full>)).value...);
376 }
377 }
378 return __r;
379 }
380
381 template <typename _Vp>
382 [[__gnu__::__always_inline__]]
383 constexpr auto
384 _M_chunk() const noexcept
385 {
386 constexpr int __n = _S_size / _Vp::_S_size;
387 constexpr int __rem = _S_size % _Vp::_S_size;
388 constexpr auto [...__is] = _IotaArray<__n>;
389 if constexpr (__rem == 0)
390 return array<_Vp, __n> {__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, *this)...};
391 else
392 {
393 using _Rest = resize_t<__rem, _Vp>;
394 return tuple(__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, *this)...,
395 __extract_simd_at<_Rest>(cw<_Vp::_S_size * __n>, *this));
396 }
397 }
398
399 [[__gnu__::__always_inline__]]
400 static constexpr basic_vec
401 _S_concat(const basic_vec& __x0) noexcept
402 { return __x0; }
403
404 template <typename... _As>
405 requires (sizeof...(_As) > 1)
406 [[__gnu__::__always_inline__]]
407 static constexpr basic_vec
408 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
409 {
410 static_assert(_S_size == (_As::_S_size + ...));
411 return __extract_simd_at<basic_vec>(cw<0>, __xs...);
412 }
413
414 /** @internal
415 * Shifts elements to the front by @p _Shift positions (or to the back for negative @p
416 * _Shift).
417 *
418 * This function moves elements towards lower indices (front of the vector).
419 * Elements that would shift beyond the vector bounds are replaced with zero. Negative shift
420 * values shift in the opposite direction.
421 *
422 * @warning The naming can be confusing due to little-endian byte order:
423 * - Despite the name "shifted_to_front", the underlying hardware instruction
424 * shifts bits to the right (psrl...)
425 * - The function name refers to element indices, not bit positions
426 *
427 * @tparam _Shift Number of positions to shift elements towards the front.
428 * Must be -size() < _Shift < size().
429 *
430 * @return A new vector with elements shifted to front or back.
431 *
432 * Example:
433 * @code
434 * __iota<vec<int, 4>>._M_elements_shifted_to_front<2>(); // {2, 3, 0, 0}
435 * __iota<vec<int, 4>>._M_elements_shifted_to_front<-2>(); // {0, 0, 0, 1}
436 * @endcode
437 */
438 template <int _Shift, _ArchTraits _Traits = {}>
439 [[__gnu__::__always_inline__]]
440 constexpr basic_vec
441 _M_elements_shifted_to_front() const
442 {
443 static_assert(_Shift < _S_size && -_Shift < _S_size);
444 if constexpr (_Shift == 0)
445 return *this;
446 else
447 return _S_static_permute(*this, [](int __i) consteval {
448 int __off = __i + _Shift;
449 return __off >= _S_size || __off < 0 ? zero_element : __off;
450 });
451 }
452
453 /** @internal
454 * @brief Set padding elements to @p __id; add more padding elements if necessary.
455 *
456 * @note This function can rearrange the element order since the result is only used for
457 * reductions.
458 */
459 template <typename _Vp, __canon_value_type __id>
460 [[__gnu__::__always_inline__]]
461 constexpr _Vp
462 _M_pad_to_T_with_value() const noexcept
463 {
464 static_assert(!_Vp::_S_is_partial);
465 static_assert(_Ap::_S_nreg == 1);
466 if constexpr (sizeof(_Vp) == 32)
467 { // when we need to reduce from a 512-bit register
468 static_assert(sizeof(_M_data) == 32);
469 constexpr auto __k = _Vp::mask_type::_S_partial_mask_of_n(_S_size);
470 return __select_impl(__k, _Vp::_S_init(_M_data), __id);
471 }
472 else
473 {
474 static_assert(sizeof(_Vp) <= 16); // => max. 7 Bytes need to be zeroed
475 static_assert(sizeof(_M_data) <= sizeof(_Vp));
476 _Vp __v1 = __vec_zero_pad_to<sizeof(_Vp)>(_M_data);
477 if constexpr (__id == 0 && _S_is_partial)
478 // cheapest solution: shift values to the back while shifting in zeros
479 // This is valid because we shift out padding elements and use all elements in a
480 // subsequent reduction.
481 __v1 = __v1.template _M_elements_shifted_to_front<-(_Vp::_S_size - _S_size)>();
482 else if constexpr (_Vp::_S_size - _S_size == 1)
483 // if a single element needs to be changed, use an insert instruction
484 __vec_set(__v1._M_data, _Vp::_S_size - 1, __id);
485 else if constexpr (__has_single_bit(unsigned(_Vp::_S_size - _S_size)))
486 { // if 2^n elements need to be changed, use a single insert instruction
487 constexpr int __n = _Vp::_S_size - _S_size;
488 using _Ip = __integer_from<__n * sizeof(__canon_value_type)>;
489 constexpr auto [...__is] = _IotaArray<__n>;
490 constexpr __canon_value_type __idn[__n] = {((void)__is, __id)...};
491 auto __vn = __vec_bit_cast<_Ip>(__v1._M_data);
492 __vec_set(__vn, _Vp::_S_size / __n - 1, __builtin_bit_cast(_Ip, __idn));
493 __v1._M_data = reinterpret_cast<typename _Vp::_DataType>(__vn);
494 }
495 else if constexpr (__id != 0 && !_S_is_partial)
496 { // if __vec_zero_pad_to added zeros in all the places where we need __id, a
497 // bitwise or is sufficient (needs a vector constant for the __id vector, which
498 // isn't optimal)
499 constexpr _Vp __idn([](int __i) {
500 return __i >= _S_size ? __id : __canon_value_type();
501 });
502 __v1._M_data = __vec_or(__v1._M_data, __idn._M_data);
503 }
504 else if constexpr (__id != 0 || _S_is_partial)
505 { // fallback
506 constexpr auto __k = _Vp::mask_type::_S_partial_mask_of_n(_S_size);
507 __v1 = __select_impl(__k, __v1, __id);
508 }
509 return __v1;
510 }
511 }
512
513 [[__gnu__::__always_inline__]]
514 constexpr auto
515 _M_reduce_to_half(auto __binary_op) const
516 {
517 static_assert(__has_single_bit(unsigned(_S_size)));
518 auto [__a, __b] = chunk<_S_size / 2>(*this);
519 return __binary_op(__a, __b);
520 }
521
522 template <typename _Rest, typename _BinaryOp>
523 [[__gnu__::__always_inline__]]
524 constexpr value_type
525 _M_reduce_tail(const _Rest& __rest, _BinaryOp __binary_op) const
526 {
527 if constexpr (_S_is_scalar)
528 return __binary_op(*this, __rest)._M_data;
529 else if constexpr (_Rest::_S_size == _S_size)
530 return __binary_op(*this, __rest)._M_reduce(__binary_op);
531 else if constexpr (_Rest::_S_size > _S_size)
532 {
533 auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
534 return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
535 }
536 else if constexpr (_Rest::_S_size == 1)
537 return __binary_op(_Rest(_M_reduce(__binary_op)), __rest)[0];
538 else if constexpr (sizeof(_M_data) <= 16
539 && requires { __default_identity_element<__canon_value_type, _BinaryOp>(); })
540 { // extend __rest with identity element for more parallelism
541 constexpr __canon_value_type __id
542 = __default_identity_element<__canon_value_type, _BinaryOp>();
543 return __binary_op(_M_data, __rest.template _M_pad_to_T_with_value<basic_vec, __id>())
544 ._M_reduce(__binary_op);
545 }
546 else
547 return _M_reduce_to_half(__binary_op)._M_reduce_tail(__rest, __binary_op);
548 }
549
550 /** @internal
551 * @brief Reduction over @p __binary_op of all (non-padding) elements.
552 *
553 * @note The implementation assumes it is most efficient to first reduce to one 128-bit SIMD
554 * register and then shuffle elements while sticking to 128-bit registers.
555 */
556 template <typename _BinaryOp, _ArchTraits _Traits = {}>
557 [[__gnu__::__always_inline__]]
558 constexpr value_type
559 _M_reduce(_BinaryOp __binary_op) const
560 {
561 constexpr bool __have_id_elem
562 = requires { __default_identity_element<__canon_value_type, _BinaryOp>(); };
563 if constexpr (_S_size == 1)
564 return operator[](0);
565 else if constexpr (_Traits.template _M_eval_as_f32<value_type>()
566 && (is_same_v<_BinaryOp, plus<>>
567 || is_same_v<_BinaryOp, multiplies<>>))
568 return value_type(rebind_t<float, basic_vec>(*this)._M_reduce(__binary_op));
569#ifdef __SSE2__
570 else if constexpr (is_integral_v<value_type> && sizeof(value_type) == 1
571 && is_same_v<decltype(__binary_op), multiplies<>>)
572 {
573 // convert to unsigned short because of missing 8-bit mul instruction
574 // we don't need to preserve the order of elements
575 //
576 // The left columns under Latency and Throughput show bit-cast to ushort with shift by
577 // 8. The right column uses the alternative in the else branch.
578 // Benchmark on Intel Ultra 7 165U (AVX2)
579 // TYPE Latency Throughput
580 // [cycles/call] [cycles/call]
581 //schar, 2 9.11 7.73 3.17 3.21
582 //schar, 4 31.6 34.9 5.11 6.97
583 //schar, 8 35.7 41.5 7.77 7.17
584 //schar, 16 36.7 44.1 6.66 8.96
585 //schar, 32 42.2 61.1 8.82 10.1
586 if constexpr (!_S_is_partial)
587 { // If all elements participate in the reduction we can take this shortcut
588 using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short, basic_vec>>;
589 auto __a = __builtin_bit_cast(_V16, *this);
590 return __binary_op(__a, __a >> 8)._M_reduce(__binary_op);
591 }
592 else
593 {
594 using _V16 = rebind_t<unsigned short, basic_vec>;
595 return _V16(*this)._M_reduce(__binary_op);
596 }
597 }
598#endif
599 else if constexpr (__has_single_bit(unsigned(_S_size)))
600 {
601 if constexpr (sizeof(_M_data) > 16)
602 return _M_reduce_to_half(__binary_op)._M_reduce(__binary_op);
603 else if constexpr (_S_size == 2)
604 return _M_reduce_to_half(__binary_op)[0];
605 else
606 {
607 static_assert(_S_size <= 16);
608 auto __x = *this;
609#ifdef __SSE2__
610 if constexpr (sizeof(_M_data) <= 16 && is_integral_v<value_type>)
611 {
612 if constexpr (_S_size > 8)
613 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<8>());
614 if constexpr (_S_size > 4)
615 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<4>());
616 if constexpr (_S_size > 2)
617 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<2>());
618 // We could also call __binary_op with vec<T, 1> arguments. However,
619 // micro-benchmarking on Intel Ultra 7 165U showed this to be more efficient:
620 return __binary_op(__x, __x.template _M_elements_shifted_to_front<1>())[0];
621 }
622#endif
623 if constexpr (_S_size > 8)
624 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<8>()));
625 if constexpr (_S_size > 4)
626 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<4>()));
627#ifdef __SSE2__
628 // avoid pshufb by "promoting" to int
629 if constexpr (is_integral_v<value_type> && sizeof(value_type) <= 1)
630 return value_type(resize_t<4, rebind_t<int, basic_vec>>(chunk<4>(__x)[0])
631 ._M_reduce(__binary_op));
632#endif
633 if constexpr (_S_size > 2)
634 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<2>()));
635 if constexpr (is_integral_v<value_type> && sizeof(value_type) == 2)
636 return __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<1>()))[0];
637 else
638 return __binary_op(vec<value_type, 1>(__x[0]), vec<value_type, 1>(__x[1]))[0];
639 }
640 }
641 else if constexpr (sizeof(_M_data) == 32)
642 {
643 const auto [__lo, __hi] = chunk<__bit_floor(unsigned(_S_size))>(*this);
644 return __lo._M_reduce_tail(__hi, __binary_op);
645 }
646 else if constexpr (sizeof(_M_data) == 64)
647 {
648 // e.g. _S_size = 16 + 16 + 15 (vec<char, 47>)
649 // -> 8 + 8 + 7 -> 4 + 4 + 3 -> 2 + 2 + 1 -> 1
650 auto __chunked = chunk<__bit_floor(unsigned(_S_size)) / 2>(*this);
651 using _Cp = decltype(__chunked);
652 if constexpr (tuple_size_v<_Cp> == 4)
653 {
654 const auto& [__a, __b, __c, __rest] = __chunked;
655 constexpr bool __amd_cpu = _Traits._M_have_sse4a();
656 if constexpr (__have_id_elem && __rest._S_size > 1 && __amd_cpu)
657 { // do one 256-bit op -> one 128-bit op
658 // 4 cycles on Zen4/5 until _M_reduce (short, 26, plus<>)
659 // 9 cycles on Skylake-AVX512 until _M_reduce
660 // 9 cycles on Zen4/5 until _M_reduce (short, 27, multiplies<>)
661 // 17 cycles on Skylake-AVX512 until _M_reduce (short, 27, multiplies<>)
662 const auto& [__a, __rest] = chunk<__bit_floor(unsigned(_S_size))>(*this);
663 using _Vp = remove_cvref_t<decltype(__a)>;
664 constexpr __canon_value_type __id
665 = __default_identity_element<__canon_value_type, _BinaryOp>();
666 const _Vp __b = __rest.template _M_pad_to_T_with_value<_Vp, __id>();
667 return __binary_op(__a, __b)._M_reduce(__binary_op);
668 }
669 else if constexpr (__have_id_elem && __rest._S_size > 1)
670 { // do two 128-bit ops -> one 128-bit op
671 // 5 cycles on Zen4/5 until _M_reduce (short, 26, plus<>)
672 // 7 cycles on Skylake-AVX512 until _M_reduce (short, 26, plus<>)
673 // 9 cycles on Zen4/5 until _M_reduce (short, 27, multiplies<>)
674 // 16 cycles on Skylake-AVX512 until _M_reduce (short, 27, multiplies<>)
675 using _Vp = remove_cvref_t<decltype(__a)>;
676 constexpr __canon_value_type __id
677 = __default_identity_element<__canon_value_type, _BinaryOp>();
678 const _Vp __d = __rest.template _M_pad_to_T_with_value<_Vp, __id>();
679 return __binary_op(__binary_op(__a, __b), __binary_op(__c, __d))
680 ._M_reduce(__binary_op);
681 }
682 else
683 return __binary_op(__binary_op(__a, __b), __c)
684 ._M_reduce_tail(__rest, __binary_op);
685 }
686 else if constexpr (tuple_size_v<_Cp> == 3)
687 {
688 const auto& [__a, __b, __rest] = __chunked;
689 return __binary_op(__a, __b)._M_reduce_tail(__rest, __binary_op);
690 }
691 else
692 static_assert(false);
693 }
694 else if constexpr (__have_id_elem)
695 {
696 constexpr __canon_value_type __id
697 = __default_identity_element<__canon_value_type, _BinaryOp>();
698 using _Vp = resize_t<__bit_ceil(unsigned(_S_size)), basic_vec>;
699 return _M_pad_to_T_with_value<_Vp, __id>()._M_reduce(__binary_op);
700 }
701 else
702 {
703 const auto& [__a, __rest] = chunk<__bit_floor(unsigned(_S_size))>(*this);
704 return __a._M_reduce_tail(__rest, __binary_op);
705 }
706 }
707
708 // [simd.math] ----------------------------------------------------------
709 //
710 // ISO/IEC 60559 on the classification operations (5.7.2 General Operations):
711 // "They are never exceptional, even for signaling NaNs."
712 //
713 template <_OptTraits _Traits = {}>
714 [[__gnu__::__always_inline__]]
715 constexpr mask_type
716 _M_isnan() const requires is_floating_point_v<value_type>
717 {
718 if constexpr (_Traits._M_finite_math_only())
719 return mask_type(false);
720 else if constexpr (_S_is_scalar)
721 return mask_type(std::isnan(_M_data));
722 else if constexpr (_S_use_bitmask)
723 return _M_isunordered(*this);
724 else if constexpr (!_Traits._M_support_snan())
725 return !(*this == *this);
726 else if (__is_const_known(_M_data))
727 return mask_type([&](int __i) { return std::isnan(_M_data[__i]); });
728 else
729 {
730 // 60559: NaN is represented as Inf + non-zero mantissa bits
731 using _Ip = __integer_from<sizeof(value_type)>;
732 return __builtin_bit_cast(_Ip, numeric_limits<value_type>::infinity())
733 < __builtin_bit_cast(rebind_t<_Ip, basic_vec>, _M_fabs());
734 }
735 }
736
737 template <_TargetTraits _Traits = {}>
738 [[__gnu__::__always_inline__]]
739 constexpr mask_type
740 _M_isinf() const requires is_floating_point_v<value_type>
741 {
742 if constexpr (_Traits._M_finite_math_only())
743 return mask_type(false);
744 else if constexpr (_S_is_scalar)
745 return mask_type(std::isinf(_M_data));
746 else if (__is_const_known(_M_data))
747 return mask_type([&](int __i) { return std::isinf(_M_data[__i]); });
748#ifdef _GLIBCXX_X86
749 else if constexpr (_S_use_bitmask)
750 return mask_type::_S_init(__x86_bitmask_isinf(_M_data));
751 else if constexpr (_Traits._M_have_avx512dq())
752 return __x86_bit_to_vecmask<typename mask_type::_DataType>(
753 __x86_bitmask_isinf(_M_data));
754#endif
755 else
756 {
757 using _Ip = __integer_from<sizeof(value_type)>;
758 return __vec_bit_cast<_Ip>(_M_fabs()._M_data)
759 == __builtin_bit_cast(_Ip, numeric_limits<value_type>::infinity());
760 }
761 }
762
763 [[__gnu__::__always_inline__]]
764 constexpr basic_vec
765 _M_abs() const requires signed_integral<value_type>
766 { return _M_data < 0 ? -_M_data : _M_data; }
767
768 [[__gnu__::__always_inline__]]
769 constexpr basic_vec
770 _M_fabs() const requires floating_point<value_type>
771 {
772 if constexpr (_S_is_scalar)
773 return std::fabs(_M_data);
774 else
775 return __vec_and(__vec_not(_S_signmask<_DataType>), _M_data);
776 }
777
778 template <_TargetTraits _Traits = {}>
779 [[__gnu__::__always_inline__]]
780 constexpr mask_type
781 _M_isunordered(basic_vec __y) const requires is_floating_point_v<value_type>
782 {
783 if constexpr (_Traits._M_finite_math_only())
784 return mask_type(false);
785 else if constexpr (_S_is_scalar)
786 return mask_type(std::isunordered(_M_data, __y._M_data));
787#ifdef _GLIBCXX_X86
788 else if constexpr (_S_use_bitmask)
789 return _M_bitmask_cmp<_X86Cmp::_Unord>(__y._M_data);
790#endif
791 else
792 return mask_type([&](int __i) {
793 return std::isunordered(_M_data[__i], __y._M_data[__i]);
794 });
795 }
796
797 /** @internal
798 * Implementation of @ref partial_load.
799 *
800 * @param __mem A pointer to an array of @p __n values. Can be complex or real.
801 * @param __n Read no more than @p __n values from memory. However, depending on @p __mem
802 * alignment, out of bounds reads are benign.
803 */
804 template <typename _Up, _ArchTraits _Traits = {}>
805 static inline basic_vec
806 _S_partial_load(const _Up* __mem, size_t __n)
807 {
808 if constexpr (_S_is_scalar)
809 return __n == 0 ? basic_vec() : basic_vec(static_cast<value_type>(*__mem));
810 else if (__is_const_known_equal_to(__n >= size_t(_S_size), true))
811 return basic_vec(_LoadCtorTag(), __mem);
812 else if constexpr (!__converts_trivially<_Up, value_type>)
813 return static_cast<basic_vec>(rebind_t<_Up, basic_vec>::_S_partial_load(__mem, __n));
814 else
815 {
816#if _GLIBCXX_X86
817 if constexpr (_Traits._M_have_avx512f()
818 || (_Traits._M_have_avx() && sizeof(_Up) >= 4))
819 {
820 const auto __k = __n < _S_size ? mask_type::_S_partial_mask_of_n(int(__n))
821 : mask_type(true);
822 return _S_masked_load(__mem, mask_type::_S_partial_mask_of_n(int(__n)));
823 }
824#endif
825 if (__n >= size_t(_S_size)) [[unlikely]]
826 return basic_vec(_LoadCtorTag(), __mem);
827#if _GLIBCXX_X86 // TODO: where else is this "safe"?
828 // allow out-of-bounds read when it cannot lead to a #GP
829 else if (__is_const_known_equal_to(
830 is_sufficiently_aligned<sizeof(_Up) * _S_full_size>(__mem), true))
831 return __select_impl(mask_type::_S_partial_mask_of_n(int(__n)),
832 basic_vec(_LoadCtorTag(), __mem), basic_vec());
833#endif
834 else if constexpr (_S_size > 4)
835 {
836 alignas(_DataType) byte __dst[sizeof(_DataType)] = {};
837 const byte* __src = reinterpret_cast<const byte*>(__mem);
838 __memcpy_chunks<sizeof(_Up), sizeof(_DataType)>(__dst, __src, __n);
839 return __builtin_bit_cast(_DataType, __dst);
840 }
841 else if (__n == 0) [[unlikely]]
842 return basic_vec();
843 else if constexpr (_S_size == 2)
844 return _DataType {static_cast<value_type>(__mem[0]), 0};
845 else
846 {
847 constexpr auto [...__is] = _IotaArray<_S_size - 2>;
848 return _DataType{
849 static_cast<value_type>(__mem[0]),
850 static_cast<value_type>(__is + 1 < __n ? __mem[__is + 1] : 0)...
851 };
852 }
853 }
854 }
855
856 /** @internal
857 * Loads elements from @p __mem according to mask @p __k.
858 *
859 * @param __mem Pointer (in)to array.
860 * @param __k Mask controlling which elements to load. For each bit i in the mask:
861 * - If bit i is 1: copy __mem[i] into result[i]
862 * - If bit i is 0: result[i] is default initialized
863 *
864 * @note This function assumes it's called after determining that no other method
865 * (like full load) is more appropriate. Calling with all mask bits set to 1
866 * is suboptimal for performance but still correct.
867 */
868 template <typename _Up, _ArchTraits _Traits = {}>
869 static inline basic_vec
870 _S_masked_load(const _Up* __mem, mask_type __k)
871 {
872 if constexpr (_S_size == 1)
873 return __k[0] ? static_cast<value_type>(__mem[0]) : value_type();
874#if _GLIBCXX_X86
875 else if constexpr (_Traits._M_have_avx512f())
876 return __x86_masked_load<_DataType>(__mem, __k._M_data);
877 else if constexpr (_Traits._M_have_avx() && (sizeof(_Up) == 4 || sizeof(_Up) == 8))
878 {
879 if constexpr (__converts_trivially<_Up, value_type>)
880 return __x86_masked_load<_DataType>(__mem, __k._M_data);
881 else
882 {
883 using _UV = rebind_t<_Up, basic_vec>;
884 return basic_vec(_UV::_S_masked_load(__mem, typename _UV::mask_type(__k)));
885 }
886 }
887#endif
888 else if (__k._M_none_of()) [[unlikely]]
889 return basic_vec();
890 else if constexpr (_S_is_scalar)
891 return basic_vec(static_cast<value_type>(*__mem));
892 else
893 {
894 // Use at least 4-byte __bits in __bit_foreach for better code-gen
895 _Bitmask<_S_size < 32 ? 32 : _S_size> __bits = __k._M_to_uint();
896 [[assume(__bits != 0)]]; // because of '__k._M_none_of()' branch above
897 if constexpr (__converts_trivially<_Up, value_type>)
898 {
899 _DataType __r = {};
900 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
901 __r[__i] = __mem[__i];
902 });
903 return __r;
904 }
905 else
906 {
907 using _UV = rebind_t<_Up, basic_vec>;
908 alignas(_UV) _Up __tmp[sizeof(_UV) / sizeof(_Up)] = {};
909 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
910 __tmp[__i] = __mem[__i];
911 });
912 return basic_vec(__builtin_bit_cast(_UV, __tmp));
913 }
914 }
915 }
916
917 template <typename _Up>
918 [[__gnu__::__always_inline__]]
919 inline void
920 _M_store(_Up* __mem) const
921 {
922 if constexpr (__converts_trivially<value_type, _Up>)
923 __builtin_memcpy(__mem, &_M_data, sizeof(_Up) * _S_size);
924 else
925 rebind_t<_Up, basic_vec>(*this)._M_store(__mem);
926 }
927
928 /** @internal
929 * Implementation of @ref partial_store.
930 *
931 * @note This is a static function to allow passing @p __v via register in case the function
932 * is not inlined.
933 *
934 * @note The function is not marked @c __always_inline__ since code-gen can become fairly
935 * long.
936 */
937 template <typename _Up, _ArchTraits _Traits = {}>
938 static inline void
939 _S_partial_store(const basic_vec __v, _Up* __mem, size_t __n)
940 {
941 if (__is_const_known_equal_to(__n >= _S_size, true))
942 __v._M_store(__mem);
943#if _GLIBCXX_X86
944 else if constexpr (_Traits._M_have_avx512f() && !_S_is_scalar)
945 {
946 const auto __k = __n < _S_size ? mask_type::_S_partial_mask_of_n(int(__n))
947 : mask_type(true);
948 return _S_masked_store(__v, __mem, __k);
949 }
950#endif
951 else if (__n >= _S_size) [[unlikely]]
952 __v._M_store(__mem);
953 else if (__n == 0) [[unlikely]]
954 return;
955 else if constexpr (__converts_trivially<value_type, _Up>)
956 {
957 byte* __dst = reinterpret_cast<byte*>(__mem);
958 const byte* __src = reinterpret_cast<const byte*>(&__v._M_data);
959 __memcpy_chunks<sizeof(_Up), sizeof(_M_data)>(__dst, __src, __n);
960 }
961 else
962 {
963 using _UV = rebind_t<_Up, basic_vec>;
964 _UV::_S_partial_store(_UV(__v), __mem, __n);
965 }
966 }
967
968 /** @internal
969 * Stores elements of @p __v to @p __mem according to mask @p __k.
970 *
971 * @param __v Values to store to @p __mem.
972 * @param __mem Pointer (in)to array.
973 * @param __k Mask controlling which elements to store. For each bit i in the mask:
974 * - If bit i is 1: store __v[i] to __mem[i]
975 * - If bit i is 0: __mem[i] is left unchanged
976 *
977 * @note This function assumes it's called after determining that no other method
978 * (like full store) is more appropriate. Calling with all mask bits set to 1
979 * is suboptimal for performance but still correct.
980 */
981 template <typename _Up, _ArchTraits _Traits = {}>
982 //[[__gnu__::__always_inline__]]
983 static inline void
984 _S_masked_store(const basic_vec __v, _Up* __mem, const mask_type __k)
985 {
986#if _GLIBCXX_X86
987 if constexpr (_Traits._M_have_avx512f())
988 {
989 __x86_masked_store(__v._M_data, __mem, __k._M_data);
990 return;
991 }
992 else if constexpr (_Traits._M_have_avx() && (sizeof(_Up) == 4 || sizeof(_Up) == 8))
993 {
994 if constexpr (__converts_trivially<value_type, _Up>)
995 __x86_masked_store(__v._M_data, __mem, __k._M_data);
996 else
997 {
998 using _UV = rebind_t<_Up, basic_vec>;
999 _UV::_S_masked_store(_UV(__v), __mem, typename _UV::mask_type(__k));
1000 }
1001 return;
1002 }
1003#endif
1004 if (__k._M_none_of()) [[unlikely]]
1005 return;
1006 else if constexpr (_S_is_scalar)
1007 __mem[0] = __v._M_data;
1008 else
1009 {
1010 // Use at least 4-byte __bits in __bit_foreach for better code-gen
1011 _Bitmask<_S_size < 32 ? 32 : _S_size> __bits = __k._M_to_uint();
1012 [[assume(__bits != 0)]]; // because of '__k._M_none_of()' branch above
1013 if constexpr (__converts_trivially<value_type, _Up>)
1014 {
1015 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
1016 __mem[__i] = __v[__i];
1017 });
1018 }
1019 else
1020 {
1021 const rebind_t<_Up, basic_vec> __cvted(__v);
1022 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
1023 __mem[__i] = __cvted[__i];
1024 });
1025 }
1026 }
1027 }
1028
1029 // [simd.overview] default constructor ----------------------------------
1030 basic_vec() = default;
1031
1032 // [simd.overview] p2 impl-def conversions ------------------------------
1033 using _NativeVecType = decltype([] {
1034 if constexpr (_S_is_scalar)
1035 return __vec_builtin_type<__canon_value_type, 1>();
1036 else
1037 return _DataType();
1038 }());
1039 /**
1040 * @brief Converting constructor from GCC vector builtins.
1041 *
1042 * This constructor enables direct construction from GCC vector builtins
1043 * (`[[gnu::vector_size(N)]]`).
1044 *
1045 * @param __x GCC vector builtin to convert from.
1046 *
1047 * @note This constructor is not available when size() equals 1.
1048 *
1049 * @see operator _NativeVecType() for the reverse conversion.
1050 */
1051 constexpr
1052 basic_vec(_NativeVecType __x)
1053 : _M_data([&] [[__gnu__::__always_inline__]] {
1054 if constexpr (_S_is_scalar)
1055 return __x[0];
1056 else
1057 return __x;
1058 }())
1059 {}
1060
1061 /**
1062 * @brief Conversion operator to GCC vector builtins.
1063 *
1064 * This operator enables implicit conversion from basic_vec to GCC vector builtins.
1065 *
1066 * @note This operator is not available when size() equals 1.
1067 *
1068 * @see basic_vec(_NativeVecType) for the reverse conversion.
1069 */
1070 constexpr
1071 operator _NativeVecType() const
1072 {
1073 if constexpr (_S_is_scalar)
1074 return _NativeVecType{_M_data};
1075 else
1076 return _M_data;
1077 }
1078
1079#if _GLIBCXX_X86
1080 /**
1081 * @brief Converting constructor from Intel Intrinsics (__m128, __m128i, ...).
1082 */
1083 template <__vec_builtin _IV>
1084 requires same_as<__x86_intel_intrin_value_type<value_type>, __vec_value_type<_IV>>
1085 && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
1086 && !is_same_v<_IV, _DataType>)
1087 constexpr
1088 basic_vec(_IV __x)
1089 : _M_data(reinterpret_cast<_DataType>(__x))
1090 {}
1091
1092 /**
1093 * @brief Conversion operator to Intel Intrinsics (__m128, __m128i, ...).
1094 */
1095 template <__vec_builtin _IV>
1096 requires same_as<__x86_intel_intrin_value_type<value_type>, __vec_value_type<_IV>>
1097 && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
1098 && !is_same_v<_IV, _DataType>)
1099 constexpr
1100 operator _IV() const
1101 { return reinterpret_cast<_IV>(_M_data); }
1102#endif
1103
1104 // [simd.ctor] broadcast constructor ------------------------------------
1105 /**
1106 * @brief Broadcast constructor from scalar value.
1107 *
1108 * Constructs a vector where all elements are initialized to the same scalar value.
1109 * The scalar value is converted to the vector's element type.
1110 *
1111 * @param __x Scalar value to broadcast to all vector elements.
1112 * @tparam _Up Type of scalar value (must be explicitly convertible to value_type).
1113 *
1114 * @note The constructor is implicit if the conversion (if any) is value-preserving.
1115 */
1116 template <__broadcast_constructible<value_type> _Up>
1117 [[__gnu__::__always_inline__]]
1118 constexpr
1119 basic_vec(_Up&& __x) noexcept
1120 : _M_data(_DataType() == _DataType() ? static_cast<value_type>(__x) : value_type())
1121 {}
1122
1123 // [simd.ctor] conversion constructor -----------------------------------
1124 template <typename _Up, typename _UAbi, _TargetTraits _Traits = {}>
1125 requires (_S_size == _UAbi::_S_size)
1126 && __explicitly_convertible_to<_Up, value_type>
1127 [[__gnu__::__always_inline__]]
1128 constexpr
1129 explicit(!__value_preserving_convertible_to<_Up, value_type>
1130 || __higher_rank_than<_Up, value_type>)
1131 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
1132 : _M_data([&] [[__gnu__::__always_inline__]] {
1133 if constexpr (_S_is_scalar)
1134 return static_cast<value_type>(__x[0]);
1135 else if constexpr (_UAbi::_S_nreg >= 2)
1136 // __builtin_convertvector (__vec_cast) is inefficient for over-sized inputs.
1137 // Also e.g. vec<float, 12> -> vec<char, 12> (with SSE2) would otherwise emit 4
1138 // vcvttps2dq instructions, where only 3 are needed
1139 return _S_concat(resize_t<__x._N0, basic_vec>(__x._M_data0),
1140 resize_t<__x._N1, basic_vec>(__x._M_data1))._M_data;
1141 else
1142 return __vec_cast<_DataType>(__x._M_concat_data());
1143 }())
1144 {}
1145
1146 using _VecBase<_Tp, _Ap>::_VecBase;
1147
1148 // [simd.ctor] generator constructor ------------------------------------
1149 template <__simd_generator_invokable<value_type, _S_size> _Fp>
1150 [[__gnu__::__always_inline__]]
1151 constexpr explicit
1152 basic_vec(_Fp&& __gen)
1153 : _M_data([&] [[__gnu__::__always_inline__]] {
1154 constexpr auto [...__is] = _IotaArray<_S_size>;
1155 return _DataType{static_cast<value_type>(__gen(__simd_size_c<__is>))...};
1156 }())
1157 {}
1158
1159 // [simd.ctor] load constructor -----------------------------------------
1160 template <typename _Up>
1161 [[__gnu__::__always_inline__]]
1162 constexpr
1163 basic_vec(_LoadCtorTag, const _Up* __ptr)
1164 : _M_data()
1165 {
1166 if constexpr (_S_is_scalar)
1167 _M_data = static_cast<value_type>(__ptr[0]);
1168 else if consteval
1169 {
1170 constexpr auto [...__is] = _IotaArray<_S_size>;
1171 _M_data = _DataType{static_cast<value_type>(__ptr[__is])...};
1172 }
1173 else
1174 {
1175 if constexpr (__converts_trivially<_Up, value_type>)
1176 // This assumes std::floatN_t to be bitwise equal to float/double
1177 __builtin_memcpy(&_M_data, __ptr, sizeof(value_type) * _S_size);
1178 else
1179 {
1180 __vec_builtin_type<_Up, _S_full_size> __tmp = {};
1181 __builtin_memcpy(&__tmp, __ptr, sizeof(_Up) * _S_size);
1182 _M_data = __vec_cast<_DataType>(__tmp);
1183 }
1184 }
1185 }
1186
1187 template <ranges::contiguous_range _Rg, typename... _Flags>
1188 requires __static_sized_range<_Rg, _S_size>
1189 && __vectorizable<ranges::range_value_t<_Rg>>
1190 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
1191 [[__gnu__::__always_inline__]]
1192 constexpr
1193 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
1194 : basic_vec(_LoadCtorTag(), __flags.template _S_adjust_pointer<basic_vec>(
1195 ranges::data(__range)))
1196 {
1197 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
1198 _Flags...>);
1199 }
1200
1201 // [simd.subscr] --------------------------------------------------------
1202 /**
1203 * @brief Return the value of the element at index @p __i.
1204 *
1205 * @pre __i >= 0 && __i < size().
1206 */
1207 [[__gnu__::__always_inline__]]
1208 constexpr value_type
1209 operator[](__simd_size_type __i) const
1210 {
1211 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
1212 if constexpr (_S_is_scalar)
1213 return _M_data;
1214 else
1215 return _M_data[__i];
1216 }
1217
1218 // [simd.unary] unary operators -----------------------------------------
1219 // increment and decrement are implemented in terms of operator+=/-= which avoids UB on
1220 // padding elements while not breaking UBsan
1221 [[__gnu__::__always_inline__]]
1222 constexpr basic_vec&
1223 operator++() noexcept requires requires(value_type __a) { ++__a; }
1224 { return *this += value_type(1); }
1225
1226 [[__gnu__::__always_inline__]]
1227 constexpr basic_vec
1228 operator++(int) noexcept requires requires(value_type __a) { __a++; }
1229 {
1230 basic_vec __r = *this;
1231 *this += value_type(1);
1232 return __r;
1233 }
1234
1235 [[__gnu__::__always_inline__]]
1236 constexpr basic_vec&
1237 operator--() noexcept requires requires(value_type __a) { --__a; }
1238 { return *this -= value_type(1); }
1239
1240 [[__gnu__::__always_inline__]]
1241 constexpr basic_vec
1242 operator--(int) noexcept requires requires(value_type __a) { __a--; }
1243 {
1244 basic_vec __r = *this;
1245 *this -= value_type(1);
1246 return __r;
1247 }
1248
1249 [[__gnu__::__always_inline__]]
1250 constexpr mask_type
1251 operator!() const noexcept requires requires(value_type __a) { !__a; }
1252 { return *this == value_type(); }
1253
1254 /**
1255 * @brief Unary plus operator (no-op).
1256 *
1257 * Returns an unchanged copy of the object.
1258 */
1259 [[__gnu__::__always_inline__]]
1260 constexpr basic_vec
1261 operator+() const noexcept requires requires(value_type __a) { +__a; }
1262 { return *this; }
1263
1264 /**
1265 * @brief Unary negation operator.
1266 *
1267 * Returns a new SIMD vector after element-wise negation.
1268 */
1269 [[__gnu__::__always_inline__]]
1270 constexpr basic_vec
1271 operator-() const noexcept requires requires(value_type __a) { -__a; }
1272 { return _S_init(-_M_data); }
1273
1274 /**
1275 * @brief Bitwise NOT / complement operator.
1276 *
1277 * Returns a new SIMD vector after element-wise complement.
1278 */
1279 [[__gnu__::__always_inline__]]
1280 constexpr basic_vec
1281 operator~() const noexcept requires requires(value_type __a) { ~__a; }
1282 { return _S_init(~_M_data); }
1283
1284 // [simd.cassign] binary operators
1285 /**
1286 * @brief Bitwise AND operator.
1287 *
1288 * Returns a new SIMD vector after element-wise AND.
1289 */
1290 [[__gnu__::__always_inline__]]
1291 friend constexpr basic_vec&
1292 operator&=(basic_vec& __x, const basic_vec& __y) noexcept
1293 requires requires(value_type __a) { __a & __a; }
1294 {
1295 __x._M_data &= __y._M_data;
1296 return __x;
1297 }
1298
1299 /**
1300 * @brief Bitwise OR operator.
1301 *
1302 * Returns a new SIMD vector after element-wise OR.
1303 */
1304 [[__gnu__::__always_inline__]]
1305 friend constexpr basic_vec&
1306 operator|=(basic_vec& __x, const basic_vec& __y) noexcept
1307 requires requires(value_type __a) { __a | __a; }
1308 {
1309 __x._M_data |= __y._M_data;
1310 return __x;
1311 }
1312
1313 /**
1314 * @brief Bitwise XOR operator.
1315 *
1316 * Returns a new SIMD vector after element-wise XOR.
1317 */
1318 [[__gnu__::__always_inline__]]
1319 friend constexpr basic_vec&
1320 operator^=(basic_vec& __x, const basic_vec& __y) noexcept
1321 requires requires(value_type __a) { __a ^ __a; }
1322 {
1323 __x._M_data ^= __y._M_data;
1324 return __x;
1325 }
1326
1327 /**
1328 * @brief Applies the compound assignment operator element-wise.
1329 *
1330 * @pre If @c value_type is a signed integral type, the result is representable by @c
1331 * value_type. (This does not apply to padding elements the implementation might add for
1332 * non-power-of-2 widths.) UBsan will only see a call to @c unreachable() on overflow.
1333 *
1334 * @note The overflow detection code is discarded unless UBsan is active.
1335 */
1336 [[__gnu__::__always_inline__]]
1337 friend constexpr basic_vec&
1338 operator+=(basic_vec& __x, const basic_vec& __y) noexcept
1339 requires requires(value_type __a) { __a + __a; }
1340 {
1341 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1342 { // avoid spurious UB on signed integer overflow of the padding element(s). But don't
1343 // remove UB of the active elements (so that UBsan can still do its job).
1344 //
1345 // This check is essentially free (at runtime) because DCE removes everything except
1346 // the final change to _M_data. The overflow check is only emitted if UBsan is active.
1347 //
1348 // The alternative would be to always zero padding elements after operations that can
1349 // produce non-zero values. However, right now:
1350 // - auto f(simd::mask<int, 3> k) { return +k; } is a single VPABSD and would have to
1351 // sanitize
1352 // - bit_cast to basic_vec with non-zero padding elements is fine
1353 // - conversion from intrinsics can create non-zero padding elements
1354 // - shuffles are allowed to put whatever they want into padding elements for
1355 // optimization purposes (e.g. for better instruction selection)
1356 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1357 const _DataType __result
1358 = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1359 + reinterpret_cast<_UV>(__y._M_data));
1360 const auto __positive = __y > value_type();
1361 const auto __overflow = __positive != (__result > __x);
1362 if (__overflow._M_any_of())
1363 __builtin_unreachable(); // trigger UBsan
1364 __x._M_data = __result;
1365 }
1366 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1367 __x = basic_vec(rebind_t<float, basic_vec>(__x) + __y);
1368 else
1369 __x._M_data += __y._M_data;
1370 return __x;
1371 }
1372
1373 /** @copydoc operator+=
1374 */
1375 [[__gnu__::__always_inline__]]
1376 friend constexpr basic_vec&
1377 operator-=(basic_vec& __x, const basic_vec& __y) noexcept
1378 requires requires(value_type __a) { __a - __a; }
1379 {
1380 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1381 { // see comment on operator+=
1382 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1383 const _DataType __result
1384 = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1385 - reinterpret_cast<_UV>(__y._M_data));
1386 const auto __positive = __y > value_type();
1387 const auto __overflow = __positive != (__result < __x);
1388 if (__overflow._M_any_of())
1389 __builtin_unreachable(); // trigger UBsan
1390 __x._M_data = __result;
1391 }
1392 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1393 __x = basic_vec(rebind_t<float, basic_vec>(__x) - __y);
1394 else
1395 __x._M_data -= __y._M_data;
1396 return __x;
1397 }
1398
1399 /** @copydoc operator+=
1400 */
1401 [[__gnu__::__always_inline__]]
1402 friend constexpr basic_vec&
1403 operator*=(basic_vec& __x, const basic_vec& __y) noexcept
1404 requires requires(value_type __a) { __a * __a; }
1405 {
1406 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1407 { // see comment on operator+=
1408 for (int __i = 0; __i < _S_size; ++__i)
1409 {
1410 if (__builtin_mul_overflow_p(__x._M_data[__i], __y._M_data[__i], value_type()))
1411 __builtin_unreachable();
1412 }
1413 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1414 __x._M_data = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1415 * reinterpret_cast<_UV>(__y._M_data));
1416 }
1417
1418 // 'uint16 * uint16' promotes to int and can therefore lead to UB. The standard does not
1419 // require to avoid the undefined behavior. It's unnecessary and easy to avoid. It's also
1420 // unexpected because there's no UB on the vector types (which don't promote).
1421 else if constexpr (_S_is_scalar && is_unsigned_v<value_type>
1422 && is_signed_v<decltype(value_type() * value_type())>)
1423 __x._M_data = unsigned(__x._M_data) * unsigned(__y._M_data);
1424
1425 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1426 __x = basic_vec(rebind_t<float, basic_vec>(__x) * __y);
1427
1428 else
1429 __x._M_data *= __y._M_data;
1430 return __x;
1431 }
1432
1433 template <_TargetTraits _Traits = {}>
1434 [[__gnu__::__always_inline__]]
1435 friend constexpr basic_vec&
1436 operator/=(basic_vec& __x, const basic_vec& __y) noexcept
1437 requires requires(value_type __a) { __a / __a; }
1438 {
1439 const basic_vec __result([&](int __i) -> value_type { return __x[__i] / __y[__i]; });
1440 if (__is_const_known(__result))
1441 // the optimizer already knows the values of the result
1442 return __x = __result;
1443
1444#ifdef __SSE2__
1445 // x86 doesn't have integral SIMD division instructions
1446 // While division is faster, the required conversions are still a problem:
1447 // see PR121274, PR121284, and PR121296 for missed optimizations wrt. conversions
1448 //
1449 // With only 1 or 2 divisions, the conversion to and from fp is too expensive.
1450 if constexpr (is_integral_v<value_type> && _S_size > 2
1451 && __value_preserving_convertible_to<value_type, double>)
1452 {
1453 // If the denominator (y) is known to the optimizer, don't convert to fp because the
1454 // integral division can be translated into shifts/multiplications.
1455 if (!__is_const_known(__y))
1456 {
1457 // With AVX512FP16 use vdivph for 8-bit integers
1458 if constexpr (_Traits._M_have_avx512fp16()
1459 && __value_preserving_convertible_to<value_type, _Float16>)
1460 return __x = basic_vec(rebind_t<_Float16, basic_vec>(__x) / __y);
1461 else if constexpr (__value_preserving_convertible_to<value_type, float>)
1462 return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
1463 else
1464 return __x = basic_vec(rebind_t<double, basic_vec>(__x) / __y);
1465 }
1466 }
1467#endif
1468 if constexpr (_Traits._M_eval_as_f32<value_type>())
1469 return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
1470
1471 basic_vec __y1 = __y;
1472 if constexpr (_S_is_partial)
1473 {
1474 if constexpr (is_integral_v<value_type>)
1475 {
1476 // Assume integral division doesn't have SIMD instructions and must be done per
1477 // element anyway. Partial vectors should skip their padding elements.
1478 for (int __i = 0; __i < _S_size; ++__i)
1479 __x._M_data[__i] /= __y._M_data[__i];
1480 return __x;
1481 }
1482 else
1483 __y1 = __select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
1484 __y, basic_vec(value_type(1)));
1485 }
1486 __x._M_data /= __y1._M_data;
1487 return __x;
1488 }
1489
1490 [[__gnu__::__always_inline__]]
1491 friend constexpr basic_vec&
1492 operator%=(basic_vec& __x, const basic_vec& __y) noexcept
1493 requires requires(value_type __a) { __a % __a; }
1494 {
1495 static_assert(is_integral_v<value_type>);
1496 if constexpr (_S_is_partial)
1497 {
1498 const basic_vec __y1 = __select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
1499 __y, basic_vec(value_type(1)));
1500 if (__is_const_known(__y1))
1501 __x._M_data %= __y1._M_data;
1502 else
1503 {
1504 // Assume integral division doesn't have SIMD instructions and must be done per
1505 // element anyway. Partial vectors should skip their padding elements.
1506 for (int __i = 0; __i < _S_size; ++__i)
1507 __x._M_data[__i] %= __y._M_data[__i];
1508 }
1509 }
1510 else
1511 __x._M_data %= __y._M_data;
1512 return __x;
1513 }
1514
1515 [[__gnu__::__always_inline__]]
1516 friend constexpr basic_vec&
1517 operator<<=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT
1518 requires requires(value_type __a) { __a << __a; }
1519 {
1520 __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y >= value_type()),
1521 "negative shift is undefined behavior");
1522 __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
1523 "too large shift invokes undefined behavior");
1524 __x._M_data <<= __y._M_data;
1525 return __x;
1526 }
1527
1528 [[__gnu__::__always_inline__]]
1529 friend constexpr basic_vec&
1530 operator>>=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT
1531 requires requires(value_type __a) { __a >> __a; }
1532 {
1533 __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y >= value_type()),
1534 "negative shift is undefined behavior");
1535 __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
1536 "too large shift invokes undefined behavior");
1537 __x._M_data >>= __y._M_data;
1538 return __x;
1539 }
1540
1541 [[__gnu__::__always_inline__]]
1542 friend constexpr basic_vec&
1543 operator<<=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
1544 requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
1545 {
1546 __glibcxx_simd_precondition(__y >= 0, "negative shift is undefined behavior");
1547 __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
1548 "too large shift invokes undefined behavior");
1549 __x._M_data <<= __y;
1550 return __x;
1551 }
1552
1553 [[__gnu__::__always_inline__]]
1554 friend constexpr basic_vec&
1555 operator>>=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
1556 requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
1557 {
1558 __glibcxx_simd_precondition(__y >= 0, "negative shift is undefined behavior");
1559 __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
1560 "too large shift invokes undefined behavior");
1561 __x._M_data >>= __y;
1562 return __x;
1563 }
1564
1565 // [simd.comparison] ----------------------------------------------------
1566#if _GLIBCXX_X86
1567 template <_X86Cmp _Cmp>
1568 [[__gnu__::__always_inline__]]
1569 constexpr mask_type
1570 _M_bitmask_cmp(_DataType __y) const
1571 {
1572 static_assert(_S_use_bitmask);
1573 if (__is_const_known(_M_data, __y))
1574 {
1575 constexpr auto [...__is] = _IotaArray<_S_size>;
1576 constexpr auto __cmp_op = [] [[__gnu__::__always_inline__]]
1577 (value_type __a, value_type __b) {
1578 if constexpr (_Cmp == _X86Cmp::_Eq)
1579 return __a == __b;
1580 else if constexpr (_Cmp == _X86Cmp::_Lt)
1581 return __a < __b;
1582 else if constexpr (_Cmp == _X86Cmp::_Le)
1583 return __a <= __b;
1584 else if constexpr (_Cmp == _X86Cmp::_Unord)
1585 return std::isunordered(__a, __b);
1586 else if constexpr (_Cmp == _X86Cmp::_Neq)
1587 return __a != __b;
1588 else if constexpr (_Cmp == _X86Cmp::_Nlt)
1589 return !(__a < __b);
1590 else if constexpr (_Cmp == _X86Cmp::_Nle)
1591 return !(__a <= __b);
1592 else
1593 static_assert(false);
1594 };
1595 const _Bitmask<_S_size> __bits
1596 = ((__cmp_op(__vec_get(_M_data, __is), __vec_get(__y, __is))
1597 ? (1ULL << __is) : 0) | ...);
1598 return mask_type::_S_init(__bits);
1599 }
1600 else
1601 return mask_type::_S_init(__x86_bitmask_cmp<_Cmp>(_M_data, __y));
1602 }
1603#endif
1604
1605 [[__gnu__::__always_inline__]]
1606 friend constexpr mask_type
1607 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
1608 {
1609#if _GLIBCXX_X86
1610 if constexpr (_S_use_bitmask)
1611 return __x._M_bitmask_cmp<_X86Cmp::_Eq>(__y._M_data);
1612 else
1613#endif
1614 return mask_type::_S_init(__x._M_data == __y._M_data);
1615 }
1616
1617 [[__gnu__::__always_inline__]]
1618 friend constexpr mask_type
1619 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
1620 {
1621#if _GLIBCXX_X86
1622 if constexpr (_S_use_bitmask)
1623 return __x._M_bitmask_cmp<_X86Cmp::_Neq>(__y._M_data);
1624 else
1625#endif
1626 return mask_type::_S_init(__x._M_data != __y._M_data);
1627 }
1628
1629 [[__gnu__::__always_inline__]]
1630 friend constexpr mask_type
1631 operator<(const basic_vec& __x, const basic_vec& __y) noexcept
1632 {
1633#if _GLIBCXX_X86
1634 if constexpr (_S_use_bitmask)
1635 return __x._M_bitmask_cmp<_X86Cmp::_Lt>(__y._M_data);
1636 else
1637#endif
1638 return mask_type::_S_init(__x._M_data < __y._M_data);
1639 }
1640
1641 [[__gnu__::__always_inline__]]
1642 friend constexpr mask_type
1643 operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
1644 {
1645#if _GLIBCXX_X86
1646 if constexpr (_S_use_bitmask)
1647 return __x._M_bitmask_cmp<_X86Cmp::_Le>(__y._M_data);
1648 else
1649#endif
1650 return mask_type::_S_init(__x._M_data <= __y._M_data);
1651 }
1652
1653 [[__gnu__::__always_inline__]]
1654 friend constexpr mask_type
1655 operator>(const basic_vec& __x, const basic_vec& __y) noexcept
1656 { return __y < __x; }
1657
1658 [[__gnu__::__always_inline__]]
1659 friend constexpr mask_type
1660 operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
1661 { return __y <= __x; }
1662
1663 // [simd.cond] ---------------------------------------------------------
1664 template <_TargetTraits _Traits = {}>
1665 [[__gnu__::__always_inline__]]
1666 friend constexpr basic_vec
1667 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
1668 {
1669 if constexpr (_S_size == 1)
1670 return __k[0] ? __t : __f;
1671 else if constexpr (_S_use_bitmask)
1672 {
1673#if _GLIBCXX_X86
1674 if (__is_const_known(__k, __t, __f))
1675 return basic_vec([&](int __i) { return __k[__i] ? __t[__i] : __f[__i]; });
1676 else
1677 return __x86_bitmask_blend(__k._M_data, __t._M_data, __f._M_data);
1678#else
1679 static_assert(false, "TODO");
1680#endif
1681 }
1682 else if consteval
1683 {
1684 return __k._M_data ? __t._M_data : __f._M_data;
1685 }
1686 else
1687 {
1688 constexpr bool __uses_simd_register = sizeof(_M_data) >= 8;
1689 using _VO = _VecOps<_DataType>;
1690 if (_VO::_S_is_const_known_equal_to(__f._M_data, 0))
1691 {
1692 if (is_integral_v<value_type> && __uses_simd_register
1693 && _VO::_S_is_const_known_equal_to(__t._M_data, 1))
1694 // This is equivalent to converting the mask into a vec of 0s and 1s. So +__k.
1695 // However, basic_mask::operator+ arrives here; returning +__k would be
1696 // recursive. Instead we use -__k (which is a no-op for vector-masks) and then
1697 // flip all -1 elements to +1 by taking the absolute value.
1698 return basic_vec((-__k)._M_abs());
1699 else
1700 return __vec_and(reinterpret_cast<_DataType>(__k._M_data), __t._M_data);
1701 }
1702 else if (_VecOps<_DataType>::_S_is_const_known_equal_to(__t._M_data, 0))
1703 {
1704 if (is_integral_v<value_type> && __uses_simd_register
1705 && _VO::_S_is_const_known_equal_to(__f._M_data, 1))
1706 return value_type(1) + basic_vec(-__k);
1707 else
1708 return __vec_and(reinterpret_cast<_DataType>(__vec_not(__k._M_data)), __f._M_data);
1709 }
1710 else
1711 {
1712#if _GLIBCXX_X86
1713 // this works around bad code-gen when the compiler can't see that __k is a vector-mask.
1714 // This pattern, is recognized to match the x86 blend instructions, which only consider
1715 // the sign bit of the mask register. Also, without SSE4, if the compiler knows that __k
1716 // is a vector-mask, then the '< 0' is elided.
1717 return __k._M_data < 0 ? __t._M_data : __f._M_data;
1718#endif
1719 return __k._M_data ? __t._M_data : __f._M_data;
1720 }
1721 }
1722 }
1723 };
1724
1725 template <__vectorizable _Tp, __abi_tag _Ap>
1726 requires (_Ap::_S_nreg > 1)
1727 class basic_vec<_Tp, _Ap>
1728 : public _VecBase<_Tp, _Ap>
1729 {
1730 template <typename, typename>
1731 friend class basic_vec;
1732
1733 template <size_t, typename>
1734 friend class basic_mask;
1735
1736 static constexpr int _S_size = _Ap::_S_size;
1737
1738 static constexpr int _N0 = __bit_ceil(unsigned(_S_size)) / 2;
1739
1740 static constexpr int _N1 = _S_size - _N0;
1741
1742 using _DataType0 = __similar_vec<_Tp, _N0, _Ap>;
1743
1744 // the implementation (and users) depend on elements being contiguous in memory
1745 static_assert(_N0 * sizeof(_Tp) == sizeof(_DataType0));
1746
1747 using _DataType1 = __similar_vec<_Tp, _N1, _Ap>;
1748
1749 static_assert(_DataType0::abi_type::_S_nreg + _DataType1::abi_type::_S_nreg == _Ap::_S_nreg);
1750
1751 static constexpr bool _S_is_scalar = _DataType0::_S_is_scalar;
1752
1753 _DataType0 _M_data0;
1754
1755 _DataType1 _M_data1;
1756
1757 static constexpr bool _S_use_bitmask = _DataType0::_S_use_bitmask;
1758
1759 static constexpr bool _S_is_partial = _DataType1::_S_is_partial;
1760
1761 public:
1762 using value_type = _Tp;
1763
1764 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
1765
1766 [[__gnu__::__always_inline__]]
1767 static constexpr basic_vec
1768 _S_init(const _DataType0& __x, const _DataType1& __y)
1769 {
1770 basic_vec __r;
1771 __r._M_data0 = __x;
1772 __r._M_data1 = __y;
1773 return __r;
1774 }
1775
1776 [[__gnu__::__always_inline__]]
1777 constexpr const _DataType0&
1778 _M_get_low() const
1779 { return _M_data0; }
1780
1781 [[__gnu__::__always_inline__]]
1782 constexpr const _DataType1&
1783 _M_get_high() const
1784 { return _M_data1; }
1785
1786 [[__gnu__::__always_inline__]]
1787 friend constexpr bool
1788 __is_const_known(const basic_vec& __x)
1789 { return __is_const_known(__x._M_data0) && __is_const_known(__x._M_data1); }
1790
1791 [[__gnu__::__always_inline__]]
1792 constexpr auto
1793 _M_concat_data([[maybe_unused]] bool __do_sanitize = false) const
1794 {
1795 return __vec_concat(_M_data0._M_concat_data(false),
1796 __vec_zero_pad_to<sizeof(_M_data0)>(
1797 _M_data1._M_concat_data(__do_sanitize)));
1798 }
1799
1800 template <int _Size = _S_size, int _Offset = 0, typename _A0, typename _Fp>
1801 [[__gnu__::__always_inline__]]
1802 static constexpr basic_vec
1803 _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&& __idxmap)
1804 {
1805 return _S_init(
1806 _DataType0::template _S_static_permute<_Size, _Offset>(__x, __idxmap),
1807 _DataType1::template _S_static_permute<_Size, _Offset + _N0>(__x, __idxmap));
1808 }
1809
1810 template <typename _Vp>
1811 [[__gnu__::__always_inline__]]
1812 constexpr auto
1813 _M_chunk() const noexcept
1814 {
1815 constexpr int __n = _S_size / _Vp::_S_size;
1816 constexpr int __rem = _S_size % _Vp::_S_size;
1817 constexpr auto [...__is] = _IotaArray<__n>;
1818 if constexpr (__rem == 0)
1819 return array<_Vp, __n>{__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>,
1820 _M_data0, _M_data1)...};
1821 else
1822 {
1823 using _Rest = resize_t<__rem, _Vp>;
1824 return tuple(__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, _M_data0, _M_data1)...,
1825 __extract_simd_at<_Rest>(cw<_Vp::_S_size * __n>, _M_data0, _M_data1));
1826 }
1827 }
1828
1829 [[__gnu__::__always_inline__]]
1830 static constexpr const basic_vec&
1831 _S_concat(const basic_vec& __x0) noexcept
1832 { return __x0; }
1833
1834 template <typename... _As>
1835 requires (sizeof...(_As) >= 2)
1836 [[__gnu__::__always_inline__]]
1837 static constexpr basic_vec
1838 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
1839 {
1840 static_assert(_S_size == (_As::_S_size + ...));
1841 return _S_init(__extract_simd_at<_DataType0>(cw<0>, __xs...),
1842 __extract_simd_at<_DataType1>(cw<_N0>, __xs...));
1843 }
1844
1845 [[__gnu__::__always_inline__]]
1846 constexpr auto
1847 _M_reduce_to_half(auto __binary_op) const requires (_N0 == _N1)
1848 { return __binary_op(_M_data0, _M_data1); }
1849
1850 [[__gnu__::__always_inline__]]
1851 constexpr value_type
1852 _M_reduce_tail(const auto& __rest, auto __binary_op) const
1853 {
1854 if constexpr (__rest.size() > _S_size)
1855 {
1856 auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
1857 return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
1858 }
1859 else if constexpr (__rest.size() == _S_size)
1860 return __binary_op(*this, __rest)._M_reduce(__binary_op);
1861 else
1862 return _M_reduce_to_half(__binary_op)._M_reduce_tail(__rest, __binary_op);
1863 }
1864
1865 template <typename _BinaryOp, _TargetTraits _Traits = {}>
1866 [[__gnu__::__always_inline__]]
1867 constexpr value_type
1868 _M_reduce(_BinaryOp __binary_op) const
1869 {
1870 if constexpr (_Traits.template _M_eval_as_f32<value_type>()
1871 && (is_same_v<_BinaryOp, plus<>>
1872 || is_same_v<_BinaryOp, multiplies<>>))
1873 return value_type(rebind_t<float, basic_vec>(*this)._M_reduce(__binary_op));
1874#ifdef __SSE2__
1875 else if constexpr (is_integral_v<value_type> && sizeof(value_type) == 1
1876 && is_same_v<decltype(__binary_op), multiplies<>>)
1877 {
1878 // convert to unsigned short because of missing 8-bit mul instruction
1879 // we don't need to preserve the order of elements
1880 //
1881 // The left columns under Latency and Throughput show bit-cast to ushort with shift by
1882 // 8. The right column uses the alternative in the else branch.
1883 // Benchmark on Intel Ultra 7 165U (AVX2)
1884 // TYPE Latency Throughput
1885 // [cycles/call] [cycles/call]
1886 //schar, 64 59.9 70.7 10.5 13.3
1887 //schar, 128 81.4 97.2 12.2 21
1888 //schar, 256 92.4 129 17.2 35.2
1889 if constexpr (_DataType1::_S_is_scalar)
1890 return __binary_op(_DataType1(_M_data0._M_reduce(__binary_op)), _M_data1)[0];
1891 // TODO: optimize trailing scalar (e.g. (8+8)+(8+1))
1892 else if constexpr (_S_size % 2 == 0)
1893 { // If all elements participate in the reduction we can take this shortcut
1894 using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short, basic_vec>>;
1895 auto __a = __builtin_bit_cast(_V16, *this);
1896 return __binary_op(__a, __a >> __CHAR_BIT__)._M_reduce(__binary_op);
1897 }
1898 else
1899 {
1900 using _V16 = rebind_t<unsigned short, basic_vec>;
1901 return _V16(*this)._M_reduce(__binary_op);
1902 }
1903 }
1904#endif
1905 else
1906 return _M_data0._M_reduce_tail(_M_data1, __binary_op);
1907 }
1908
1909 [[__gnu__::__always_inline__]]
1910 constexpr mask_type
1911 _M_isnan() const requires is_floating_point_v<value_type>
1912 { return mask_type::_S_init(_M_data0._M_isnan(), _M_data1._M_isnan()); }
1913
1914 [[__gnu__::__always_inline__]]
1915 constexpr mask_type
1916 _M_isinf() const requires is_floating_point_v<value_type>
1917 { return mask_type::_S_init(_M_data0._M_isinf(), _M_data1._M_isinf()); }
1918
1919 [[__gnu__::__always_inline__]]
1920 constexpr mask_type
1921 _M_isunordered(basic_vec __y) const requires is_floating_point_v<value_type>
1922 {
1923 return mask_type::_S_init(_M_data0._M_isunordered(__y._M_data0),
1924 _M_data1._M_isunordered(__y._M_data1));
1925 }
1926
1927 [[__gnu__::__always_inline__]]
1928 constexpr basic_vec
1929 _M_abs() const requires signed_integral<value_type>
1930 { return _S_init(_M_data0._M_abs(), _M_data1._M_abs()); }
1931
1932 [[__gnu__::__always_inline__]]
1933 constexpr basic_vec
1934 _M_fabs() const requires floating_point<value_type>
1935 { return _S_init(_M_data0._M_fabs(), _M_data1._M_fabs()); }
1936
1937 template <typename _Up>
1938 [[__gnu__::__always_inline__]]
1939 static inline basic_vec
1940 _S_partial_load(const _Up* __mem, size_t __n)
1941 {
1942 if (__n >= _N0)
1943 return _S_init(_DataType0(_LoadCtorTag(), __mem),
1944 _DataType1::_S_partial_load(__mem + _N0, __n - _N0));
1945 else
1946 return _S_init(_DataType0::_S_partial_load(__mem, __n),
1947 _DataType1());
1948 }
1949
1950 template <typename _Up, _ArchTraits _Traits = {}>
1951 static inline basic_vec
1952 _S_masked_load(const _Up* __mem, mask_type __k)
1953 {
1954 return _S_init(_DataType0::_S_masked_load(__mem, __k._M_data0),
1955 _DataType1::_S_masked_load(__mem + _N0, __k._M_data1));
1956 }
1957
1958 template <typename _Up>
1959 [[__gnu__::__always_inline__]]
1960 inline void
1961 _M_store(_Up* __mem) const
1962 {
1963 _M_data0._M_store(__mem);
1964 _M_data1._M_store(__mem + _N0);
1965 }
1966
1967 template <typename _Up>
1968 [[__gnu__::__always_inline__]]
1969 static inline void
1970 _S_partial_store(const basic_vec& __v, _Up* __mem, size_t __n)
1971 {
1972 if (__n >= _N0)
1973 {
1974 __v._M_data0._M_store(__mem);
1975 _DataType1::_S_partial_store(__v._M_data1, __mem + _N0, __n - _N0);
1976 }
1977 else
1978 {
1979 _DataType0::_S_partial_store(__v._M_data0, __mem, __n);
1980 }
1981 }
1982
1983 template <typename _Up>
1984 [[__gnu__::__always_inline__]]
1985 static inline void
1986 _S_masked_store(const basic_vec& __v, _Up* __mem, const mask_type& __k)
1987 {
1988 _DataType0::_S_masked_store(__v._M_data0, __mem, __k._M_data0);
1989 _DataType1::_S_masked_store(__v._M_data1, __mem + _N0, __k._M_data1);
1990 }
1991
1992 basic_vec() = default;
1993
1994 // [simd.overview] p2 impl-def conversions ------------------------------
1995 using _NativeVecType = __vec_builtin_type<value_type, __bit_ceil(unsigned(_S_size))>;
1996
1997 [[__gnu__::__always_inline__]]
1998 constexpr
1999 basic_vec(const _NativeVecType& __x)
2000 : _M_data0(_VecOps<__vec_builtin_type<value_type, _N0>>::_S_extract(__x)),
2001 _M_data1(_VecOps<__vec_builtin_type<value_type, __bit_ceil(unsigned(_N1))>>
2002 ::_S_extract(__x, integral_constant<int, _N0>()))
2003 {}
2004
2005 [[__gnu__::__always_inline__]]
2006 constexpr
2007 operator _NativeVecType() const
2008 { return _M_concat_data(); }
2009
2010 // [simd.ctor] broadcast constructor ------------------------------------
2011 template <__broadcast_constructible<value_type> _Up>
2012 [[__gnu__::__always_inline__]]
2013 constexpr
2014 basic_vec(_Up&& __x) noexcept
2015 : _M_data0(static_cast<value_type>(__x)), _M_data1(static_cast<value_type>(__x))
2016 {}
2017
2018 // [simd.ctor] conversion constructor -----------------------------------
2019 template <typename _Up, typename _UAbi>
2020 requires (_S_size == _UAbi::_S_size)
2021 && __explicitly_convertible_to<_Up, value_type>
2022 [[__gnu__::__always_inline__]]
2023 constexpr
2024 explicit(!__value_preserving_convertible_to<_Up, value_type>
2025 || __higher_rank_than<_Up, value_type>)
2026 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
2027 : _M_data0(get<0>(chunk<_N0>(__x))),
2028 _M_data1(get<1>(chunk<_N0>(__x)))
2029 {}
2030
2031 using _VecBase<_Tp, _Ap>::_VecBase;
2032
2033 // [simd.ctor] generator constructor ------------------------------------
2034 template <__simd_generator_invokable<value_type, _S_size> _Fp>
2035 [[__gnu__::__always_inline__]]
2036 constexpr explicit
2037 basic_vec(_Fp&& __gen)
2038 : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]] (auto __i) {
2039 return __gen(__simd_size_c<__i + _N0>);
2040 })
2041 {}
2042
2043 // [simd.ctor] load constructor -----------------------------------------
2044 template <typename _Up>
2045 [[__gnu__::__always_inline__]]
2046 constexpr
2047 basic_vec(_LoadCtorTag, const _Up* __ptr)
2048 : _M_data0(_LoadCtorTag(), __ptr),
2049 _M_data1(_LoadCtorTag(), __ptr + _N0)
2050 {}
2051
2052 template <ranges::contiguous_range _Rg, typename... _Flags>
2053 requires __static_sized_range<_Rg, _S_size>
2054 && __vectorizable<ranges::range_value_t<_Rg>>
2055 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
2056 constexpr
2057 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
2058 : basic_vec(_LoadCtorTag(),
2059 __flags.template _S_adjust_pointer<basic_vec>(ranges::data(__range)))
2060 {
2061 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
2062 _Flags...>);
2063 }
2064
2065 // [simd.subscr] --------------------------------------------------------
2066 [[__gnu__::__always_inline__]]
2067 constexpr value_type
2068 operator[](__simd_size_type __i) const
2069 {
2070 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
2071 if (__is_const_known(__i))
2072 return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
2073 else
2074 {
2075 using _AliasingT [[__gnu__::__may_alias__]] = value_type;
2076 return reinterpret_cast<const _AliasingT*>(this)[__i];
2077 }
2078 }
2079
2080 // [simd.unary] unary operators -----------------------------------------
2081 [[__gnu__::__always_inline__]]
2082 constexpr basic_vec&
2083 operator++() noexcept requires requires(value_type __a) { ++__a; }
2084 {
2085 ++_M_data0;
2086 ++_M_data1;
2087 return *this;
2088 }
2089
2090 [[__gnu__::__always_inline__]]
2091 constexpr basic_vec
2092 operator++(int) noexcept requires requires(value_type __a) { __a++; }
2093 {
2094 basic_vec __r = *this;
2095 ++_M_data0;
2096 ++_M_data1;
2097 return __r;
2098 }
2099
2100 [[__gnu__::__always_inline__]]
2101 constexpr basic_vec&
2102 operator--() noexcept requires requires(value_type __a) { --__a; }
2103 {
2104 --_M_data0;
2105 --_M_data1;
2106 return *this;
2107 }
2108
2109 [[__gnu__::__always_inline__]]
2110 constexpr basic_vec
2111 operator--(int) noexcept requires requires(value_type __a) { __a--; }
2112 {
2113 basic_vec __r = *this;
2114 --_M_data0;
2115 --_M_data1;
2116 return __r;
2117 }
2118
2119 [[__gnu__::__always_inline__]]
2120 constexpr mask_type
2121 operator!() const noexcept requires requires(value_type __a) { !__a; }
2122 { return mask_type::_S_init(!_M_data0, !_M_data1); }
2123
2124 [[__gnu__::__always_inline__]]
2125 constexpr basic_vec
2126 operator+() const noexcept requires requires(value_type __a) { +__a; }
2127 { return *this; }
2128
2129 [[__gnu__::__always_inline__]]
2130 constexpr basic_vec
2131 operator-() const noexcept requires requires(value_type __a) { -__a; }
2132 { return _S_init(-_M_data0, -_M_data1); }
2133
2134 [[__gnu__::__always_inline__]]
2135 constexpr basic_vec
2136 operator~() const noexcept requires requires(value_type __a) { ~__a; }
2137 { return _S_init(~_M_data0, ~_M_data1); }
2138
2139 // [simd.cassign] -------------------------------------------------------
2140#define _GLIBCXX_SIMD_DEFINE_OP(sym) \
2141 [[__gnu__::__always_inline__]] \
2142 friend constexpr basic_vec& \
2143 operator sym##=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT \
2144 { \
2145 __x._M_data0 sym##= __y._M_data0; \
2146 __x._M_data1 sym##= __y._M_data1; \
2147 return __x; \
2148 }
2149
2150 _GLIBCXX_SIMD_DEFINE_OP(+)
2151 _GLIBCXX_SIMD_DEFINE_OP(-)
2152 _GLIBCXX_SIMD_DEFINE_OP(*)
2153 _GLIBCXX_SIMD_DEFINE_OP(/)
2154 _GLIBCXX_SIMD_DEFINE_OP(%)
2155 _GLIBCXX_SIMD_DEFINE_OP(&)
2156 _GLIBCXX_SIMD_DEFINE_OP(|)
2157 _GLIBCXX_SIMD_DEFINE_OP(^)
2158 _GLIBCXX_SIMD_DEFINE_OP(<<)
2159 _GLIBCXX_SIMD_DEFINE_OP(>>)
2160
2161#undef _GLIBCXX_SIMD_DEFINE_OP
2162
2163 [[__gnu__::__always_inline__]]
2164 friend constexpr basic_vec&
2165 operator<<=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
2166 requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
2167 {
2168 __x._M_data0 <<= __y;
2169 __x._M_data1 <<= __y;
2170 return __x;
2171 }
2172
2173 [[__gnu__::__always_inline__]]
2174 friend constexpr basic_vec&
2175 operator>>=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
2176 requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
2177 {
2178 __x._M_data0 >>= __y;
2179 __x._M_data1 >>= __y;
2180 return __x;
2181 }
2182
2183 // [simd.comparison] ----------------------------------------------------
2184 [[__gnu__::__always_inline__]]
2185 friend constexpr mask_type
2186 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
2187 { return mask_type::_S_init(__x._M_data0 == __y._M_data0, __x._M_data1 == __y._M_data1); }
2188
2189 [[__gnu__::__always_inline__]]
2190 friend constexpr mask_type
2191 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
2192 { return mask_type::_S_init(__x._M_data0 != __y._M_data0, __x._M_data1 != __y._M_data1); }
2193
2194 [[__gnu__::__always_inline__]]
2195 friend constexpr mask_type
2196 operator<(const basic_vec& __x, const basic_vec& __y) noexcept
2197 { return mask_type::_S_init(__x._M_data0 < __y._M_data0, __x._M_data1 < __y._M_data1); }
2198
2199 [[__gnu__::__always_inline__]]
2200 friend constexpr mask_type
2201 operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
2202 { return mask_type::_S_init(__x._M_data0 <= __y._M_data0, __x._M_data1 <= __y._M_data1); }
2203
2204 [[__gnu__::__always_inline__]]
2205 friend constexpr mask_type
2206 operator>(const basic_vec& __x, const basic_vec& __y) noexcept
2207 { return mask_type::_S_init(__x._M_data0 > __y._M_data0, __x._M_data1 > __y._M_data1); }
2208
2209 [[__gnu__::__always_inline__]]
2210 friend constexpr mask_type
2211 operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
2212 { return mask_type::_S_init(__x._M_data0 >= __y._M_data0, __x._M_data1 >= __y._M_data1); }
2213
2214 // [simd.cond] ---------------------------------------------------------
2215 [[__gnu__::__always_inline__]]
2216 friend constexpr basic_vec
2217 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
2218 {
2219 return _S_init(__select_impl(__k._M_data0, __t._M_data0, __f._M_data0),
2220 __select_impl(__k._M_data1, __t._M_data1, __f._M_data1));
2221 }
2222 };
2223
2224 // [simd.overview] deduction guide ------------------------------------------
2225 template <ranges::contiguous_range _Rg, typename... _Ts>
2226 requires __static_sized_range<_Rg>
2227 basic_vec(_Rg&& __r, _Ts...)
2228 -> basic_vec<ranges::range_value_t<_Rg>,
2229 __deduce_abi_t<ranges::range_value_t<_Rg>,
2230#if 0 // PR117849
2231 static_cast<__simd_size_type>(ranges::size(__r))>>;
2232#else
2233 static_cast<__simd_size_type>(decltype(std::span(__r))::extent)>>;
2234#endif
2235
2236 template <size_t _Bytes, typename _Ap>
2237 basic_vec(basic_mask<_Bytes, _Ap>)
2238 -> basic_vec<__integer_from<_Bytes>,
2239 decltype(__abi_rebind<__integer_from<_Bytes>, basic_mask<_Bytes, _Ap>::size.value,
2240 _Ap>())>;
2241
2242 // [P3319R5] ----------------------------------------------------------------
2243 template <__vectorizable _Tp>
2244 requires is_arithmetic_v<_Tp>
2245 inline constexpr _Tp
2246 __iota<_Tp> = _Tp();
2247
2248 template <typename _Tp, typename _Ap>
2249 inline constexpr basic_vec<_Tp, _Ap>
2250 __iota<basic_vec<_Tp, _Ap>> = basic_vec<_Tp, _Ap>([](_Tp __i) -> _Tp {
2251 static_assert(_Ap::_S_size - 1 <= numeric_limits<_Tp>::max(),
2252 "iota object would overflow");
2253 return __i;
2254 });
2255} // namespace simd
2256_GLIBCXX_END_NAMESPACE_VERSION
2257} // namespace std
2258
2259#pragma GCC diagnostic pop
2260#endif // C++26
2261#endif // _GLIBCXX_SIMD_VEC_H
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:859
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:873
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:826
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:866
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition complex:374
std::is_sufficiently_aligned
bool is_sufficiently_aligned(_Tp *__ptr)
Is __ptr aligned to an _Align byte boundary?
Definition align.h:118
std
ISO C++ entities toplevel namespace is std.
std::fabs
_Tp fabs(const std::complex< _Tp > &__z)
fabs(__z) TR1 8.1.8 [tr.c99.cmplx.fabs]
Definition complex:2525
std::data
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition range_access.h:324
std::numeric_limits::max
static constexpr _Tp max() noexcept
Definition limits:328
std::numeric_limits::infinity
static constexpr _Tp infinity() noexcept
Definition limits:348