27 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MMX__) 31 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE__) 35 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE2__) 39 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE3__) 43 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSSE3__) 47 #if !defined(_MSC_VER) || __has_feature(modules) || \ 48 (defined(__SSE4_2__) || defined(__SSE4_1__)) 52 #if !defined(_MSC_VER) || __has_feature(modules) || \ 53 (defined(__AES__) || defined(__PCLMUL__)) 57 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLFLUSHOPT__) 61 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLWB__) 65 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX__) 69 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX2__) 73 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__F16C__) 77 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VPCLMULQDQ__) 81 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__) 85 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__) 89 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__LZCNT__) 93 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__POPCNT__) 97 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FMA__) 101 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512F__) 105 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VL__) 109 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BW__) 113 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BITALG__) 117 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512CD__) 121 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VPOPCNTDQ__) 125 #if !defined(_MSC_VER) || __has_feature(modules) || \ 126 (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 130 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VNNI__) 134 #if !defined(_MSC_VER) || __has_feature(modules) || \ 135 (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 139 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512DQ__) 143 #if !defined(_MSC_VER) || __has_feature(modules) || \ 144 (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 148 #if !defined(_MSC_VER) || __has_feature(modules) || \ 149 (defined(__AVX512VL__) && defined(__AVX512BW__)) 153 #if !defined(_MSC_VER) || __has_feature(modules) || \ 154 (defined(__AVX512VL__) && defined(__AVX512CD__)) 158 #if !defined(_MSC_VER) || __has_feature(modules) || \ 159 (defined(__AVX512VL__) && defined(__AVX512DQ__)) 163 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512ER__) 167 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512IFMA__) 171 #if !defined(_MSC_VER) || __has_feature(modules) || \ 172 (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 176 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI__) 180 #if !defined(_MSC_VER) || __has_feature(modules) || \ 181 (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 185 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI2__) 189 #if !defined(_MSC_VER) || __has_feature(modules) || \ 190 (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 194 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512PF__) 198 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__) 202 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VAES__) 206 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__GFNI__) 210 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDPID__) 216 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"rdpid")))
218 return __builtin_ia32_rdpid();
222 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDRND__) 223 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
224 _rdrand16_step(
unsigned short *
__p)
226 return __builtin_ia32_rdrand16_step(__p);
229 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
230 _rdrand32_step(
unsigned int *
__p)
232 return __builtin_ia32_rdrand32_step(__p);
236 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
237 _rdrand64_step(
unsigned long long *__p)
239 return __builtin_ia32_rdrand64_step(__p);
246 _bit_scan_forward(
int __A) {
247 return __builtin_ctz(__A);
251 static __inline__
int __attribute__((__always_inline__, __nodebug__))
252 _bit_scan_reverse(
int __A) {
253 return 31 - __builtin_clz(__A);
256 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__) 258 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
259 _readfsbase_u32(
void)
261 return __builtin_ia32_rdfsbase32();
264 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
265 _readfsbase_u64(
void)
267 return __builtin_ia32_rdfsbase64();
270 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
271 _readgsbase_u32(
void)
273 return __builtin_ia32_rdgsbase32();
276 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
277 _readgsbase_u64(
void)
279 return __builtin_ia32_rdgsbase64();
282 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
283 _writefsbase_u32(
unsigned int __V)
285 __builtin_ia32_wrfsbase32(__V);
288 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
289 _writefsbase_u64(
unsigned long long __V)
291 __builtin_ia32_wrfsbase64(__V);
294 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
295 _writegsbase_u32(
unsigned int __V)
297 __builtin_ia32_wrgsbase32(__V);
300 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
301 _writegsbase_u64(
unsigned long long __V)
303 __builtin_ia32_wrgsbase64(__V);
309 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__) 314 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHA__) 318 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FXSR__) 322 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVE__) 326 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__) 330 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEC__) 334 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVES__) 338 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHSTK__) 346 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDSEED__) 350 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WBNOINVD__) 354 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLDEMOTE__) 358 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WAITPKG__) 362 #if !defined(_MSC_VER) || __has_feature(modules) || \ 363 defined(__MOVDIRI__) || defined(__MOVDIR64B__) 367 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PCONFIG__) 371 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SGX__) 375 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PTWRITE__) 379 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__INVPCID__) 385 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 392 #if defined(__i386__) || defined(__x86_64__) 394 _InterlockedExchange_HLEAcquire(
long volatile *_Target,
long _Value) {
395 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1" 396 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
400 _InterlockedExchange_HLERelease(
long volatile *_Target,
long _Value) {
401 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1" 402 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
406 #if defined(__x86_64__) 408 _InterlockedExchange64_HLEAcquire(__int64
volatile *_Target, __int64 _Value) {
409 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1" 410 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
414 _InterlockedExchange64_HLERelease(__int64
volatile *_Target, __int64 _Value) {
415 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1" 416 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
423 #if defined(__i386__) || defined(__x86_64__) 425 _InterlockedCompareExchange_HLEAcquire(
long volatile *_Destination,
426 long _Exchange,
long _Comparand) {
427 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1" 428 :
"+a" (_Comparand),
"+m" (*_Destination)
429 :
"r" (_Exchange) :
"memory");
433 _InterlockedCompareExchange_HLERelease(
long volatile *_Destination,
434 long _Exchange,
long _Comparand) {
435 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1" 436 :
"+a" (_Comparand),
"+m" (*_Destination)
437 :
"r" (_Exchange) :
"memory");
441 #if defined(__x86_64__) 443 _InterlockedCompareExchange64_HLEAcquire(__int64
volatile *_Destination,
444 __int64 _Exchange, __int64 _Comparand) {
445 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1" 446 :
"+a" (_Comparand),
"+m" (*_Destination)
447 :
"r" (_Exchange) :
"memory");
451 _InterlockedCompareExchange64_HLERelease(__int64
volatile *_Destination,
452 __int64 _Exchange, __int64 _Comparand) {
453 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1" 454 :
"+a" (_Comparand),
"+m" (*_Destination)
455 :
"r" (_Exchange) :
"memory");
463 #undef __DEFAULT_FN_ATTRS
static __inline unsigned char unsigned int unsigned int unsigned int * __p
#define __DEFAULT_FN_ATTRS
static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) _rdpid_u32(void)
Returns the value of the IA32_TSC_AUX MSR (0xc0000103).