13 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MMX__) 17 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE__) 21 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE2__) 25 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE3__) 29 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSSE3__) 33 #if !defined(_MSC_VER) || __has_feature(modules) || \ 34 (defined(__SSE4_2__) || defined(__SSE4_1__)) 38 #if !defined(_MSC_VER) || __has_feature(modules) || \ 39 (defined(__AES__) || defined(__PCLMUL__)) 43 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLFLUSHOPT__) 47 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLWB__) 51 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX__) 55 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX2__) 59 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__F16C__) 63 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VPCLMULQDQ__) 67 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__) 71 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__) 75 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__LZCNT__) 79 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__POPCNT__) 83 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FMA__) 87 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512F__) 91 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VL__) 95 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BW__) 99 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BITALG__) 103 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512CD__) 107 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VPOPCNTDQ__) 111 #if !defined(_MSC_VER) || __has_feature(modules) || \ 112 (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 116 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VNNI__) 120 #if !defined(_MSC_VER) || __has_feature(modules) || \ 121 (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 125 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512DQ__) 129 #if !defined(_MSC_VER) || __has_feature(modules) || \ 130 (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 134 #if !defined(_MSC_VER) || __has_feature(modules) || \ 135 (defined(__AVX512VL__) && defined(__AVX512BW__)) 139 #if !defined(_MSC_VER) || __has_feature(modules) || \ 140 (defined(__AVX512VL__) && defined(__AVX512CD__)) 144 #if !defined(_MSC_VER) || __has_feature(modules) || \ 145 (defined(__AVX512VL__) && defined(__AVX512DQ__)) 149 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512ER__) 153 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512IFMA__) 157 #if !defined(_MSC_VER) || __has_feature(modules) || \ 158 (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 162 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI__) 166 #if !defined(_MSC_VER) || __has_feature(modules) || \ 167 (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 171 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI2__) 175 #if !defined(_MSC_VER) || __has_feature(modules) || \ 176 (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 180 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512PF__) 184 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BF16__) 188 #if !defined(_MSC_VER) || __has_feature(modules) || \ 189 (defined(__AVX512VL__) && defined(__AVX512BF16__)) 193 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__) 197 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VAES__) 201 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__GFNI__) 205 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDPID__) 211 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"rdpid")))
213 return __builtin_ia32_rdpid();
217 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDRND__) 218 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
219 _rdrand16_step(
unsigned short *
__p)
221 return __builtin_ia32_rdrand16_step(__p);
224 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
225 _rdrand32_step(
unsigned int *
__p)
227 return __builtin_ia32_rdrand32_step(__p);
231 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
232 _rdrand64_step(
unsigned long long *__p)
234 return __builtin_ia32_rdrand64_step(__p);
239 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__) 241 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
242 _readfsbase_u32(
void)
244 return __builtin_ia32_rdfsbase32();
247 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
248 _readfsbase_u64(
void)
250 return __builtin_ia32_rdfsbase64();
253 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
254 _readgsbase_u32(
void)
256 return __builtin_ia32_rdgsbase32();
259 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
260 _readgsbase_u64(
void)
262 return __builtin_ia32_rdgsbase64();
265 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
266 _writefsbase_u32(
unsigned int __V)
268 __builtin_ia32_wrfsbase32(__V);
271 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
272 _writefsbase_u64(
unsigned long long __V)
274 __builtin_ia32_wrfsbase64(__V);
277 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
278 _writegsbase_u32(
unsigned int __V)
280 __builtin_ia32_wrgsbase32(__V);
283 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
284 _writegsbase_u64(
unsigned long long __V)
286 __builtin_ia32_wrgsbase64(__V);
292 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MOVBE__) 300 static __inline__
short __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
301 _loadbe_i16(
void const *
__P) {
305 return __builtin_bswap16(((
struct __loadu_i16*)__P)->
__v);
308 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
310 struct __storeu_i16 {
313 ((
struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D);
316 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
317 _loadbe_i32(
void const *
__P) {
321 return __builtin_bswap32(((
struct __loadu_i32*)__P)->
__v);
324 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
325 _storebe_i32(
void * __P,
int __D) {
326 struct __storeu_i32 {
329 ((
struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D);
333 static __inline__
long long __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
334 _loadbe_i64(
void const * __P) {
338 return __builtin_bswap64(((
struct __loadu_i64*)__P)->
__v);
341 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
342 _storebe_i64(
void * __P,
long long __D) {
343 struct __storeu_i64 {
346 ((
struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D);
351 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__) 356 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHA__) 360 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FXSR__) 367 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__) 371 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEC__) 375 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVES__) 379 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHSTK__) 387 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDSEED__) 391 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WBNOINVD__) 395 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLDEMOTE__) 399 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WAITPKG__) 403 #if !defined(_MSC_VER) || __has_feature(modules) || \ 404 defined(__MOVDIRI__) || defined(__MOVDIR64B__) 408 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PCONFIG__) 412 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SGX__) 416 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PTWRITE__) 420 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__INVPCID__) 424 #if !defined(_MSC_VER) || __has_feature(modules) || \ 425 defined(__AVX512VP2INTERSECT__) 429 #if !defined(_MSC_VER) || __has_feature(modules) || \ 430 (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__)) 434 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__ENQCMD__) 438 #if defined(_MSC_VER) && __has_extension(gnu_asm) 440 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 447 #if defined(__i386__) || defined(__x86_64__) 449 _InterlockedExchange_HLEAcquire(
long volatile *_Target,
long _Value) {
450 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1" 451 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
455 _InterlockedExchange_HLERelease(
long volatile *_Target,
long _Value) {
456 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1" 457 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
461 #if defined(__x86_64__) 463 _InterlockedExchange64_HLEAcquire(__int64
volatile *_Target, __int64 _Value) {
464 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1" 465 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
469 _InterlockedExchange64_HLERelease(__int64
volatile *_Target, __int64 _Value) {
470 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1" 471 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
478 #if defined(__i386__) || defined(__x86_64__) 480 _InterlockedCompareExchange_HLEAcquire(
long volatile *_Destination,
481 long _Exchange,
long _Comparand) {
482 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1" 483 :
"+a" (_Comparand),
"+m" (*_Destination)
484 :
"r" (_Exchange) :
"memory");
488 _InterlockedCompareExchange_HLERelease(
long volatile *_Destination,
489 long _Exchange,
long _Comparand) {
490 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1" 491 :
"+a" (_Comparand),
"+m" (*_Destination)
492 :
"r" (_Exchange) :
"memory");
496 #if defined(__x86_64__) 498 _InterlockedCompareExchange64_HLEAcquire(__int64
volatile *_Destination,
499 __int64 _Exchange, __int64 _Comparand) {
500 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1" 501 :
"+a" (_Comparand),
"+m" (*_Destination)
502 :
"r" (_Exchange) :
"memory");
506 _InterlockedCompareExchange64_HLERelease(__int64
volatile *_Destination,
507 __int64 _Exchange, __int64 _Comparand) {
508 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1" 509 :
"+a" (_Comparand),
"+m" (*_Destination)
510 :
"r" (_Exchange) :
"memory");
518 #undef __DEFAULT_FN_ATTRS
struct __storeu_i16 *__P __v
static __inline__ void short __D
__inline __m128 __m64 const * __P
__asm__("vmuleuw %0,%1,%2" :"=v"(result) :"v"(__A), "v"(__B) :)
#define __DEFAULT_FN_ATTRS
static __inline unsigned char unsigned int unsigned int unsigned int * __p
static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) _rdpid_u32(void)
Returns the value of the IA32_TSC_AUX MSR (0xc0000103).