13 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
18 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
23 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
28 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
33 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
38 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
39 (defined(__SSE4_2__) || defined(__SSE4_1__))
43 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
44 (defined(__AES__) || defined(__PCLMUL__))
48 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
49 defined(__CLFLUSHOPT__)
53 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
58 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
63 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
68 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
73 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
74 defined(__VPCLMULQDQ__)
81 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
86 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
91 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
96 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
101 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
106 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
107 defined(__AVX512VL__)
111 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
112 defined(__AVX512BW__)
116 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
117 defined(__AVX512BITALG__)
121 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
122 defined(__AVX512CD__)
126 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
127 defined(__AVX512VPOPCNTDQ__)
131 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
132 (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
136 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
137 defined(__AVX512VNNI__)
141 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
142 (defined(__AVX512VL__) && defined(__AVX512VNNI__))
146 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
147 defined(__AVX512DQ__)
151 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
152 (defined(__AVX512VL__) && defined(__AVX512BITALG__))
156 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
157 (defined(__AVX512VL__) && defined(__AVX512BW__))
161 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
162 (defined(__AVX512VL__) && defined(__AVX512CD__))
166 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
167 (defined(__AVX512VL__) && defined(__AVX512DQ__))
171 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
172 defined(__AVX512ER__)
176 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
177 defined(__AVX512IFMA__)
181 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
182 (defined(__AVX512IFMA__) && defined(__AVX512VL__))
186 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
187 defined(__AVX512VBMI__)
191 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
192 (defined(__AVX512VBMI__) && defined(__AVX512VL__))
196 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
197 defined(__AVX512VBMI2__)
201 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
202 (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
206 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
207 defined(__AVX512PF__)
211 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
212 defined(__AVX512BF16__)
216 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
217 (defined(__AVX512VL__) && defined(__AVX512BF16__))
221 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
226 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
231 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
236 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
243 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"rdpid")))
245 return __builtin_ia32_rdpid();
249 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
251 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
252 _rdrand16_step(
unsigned short *
__p)
254 return __builtin_ia32_rdrand16_step(
__p);
257 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
258 _rdrand32_step(
unsigned int *
__p)
260 return __builtin_ia32_rdrand32_step(
__p);
264 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
265 _rdrand64_step(
unsigned long long *
__p)
267 return __builtin_ia32_rdrand64_step(
__p);
272 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
273 defined(__FSGSBASE__)
275 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
276 _readfsbase_u32(
void)
278 return __builtin_ia32_rdfsbase32();
281 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
282 _readfsbase_u64(
void)
284 return __builtin_ia32_rdfsbase64();
287 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
288 _readgsbase_u32(
void)
290 return __builtin_ia32_rdgsbase32();
293 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
294 _readgsbase_u64(
void)
296 return __builtin_ia32_rdgsbase64();
299 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
300 _writefsbase_u32(
unsigned int __V)
302 __builtin_ia32_wrfsbase32(__V);
305 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
306 _writefsbase_u64(
unsigned long long __V)
308 __builtin_ia32_wrfsbase64(__V);
311 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
312 _writegsbase_u32(
unsigned int __V)
314 __builtin_ia32_wrgsbase32(__V);
317 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
318 _writegsbase_u64(
unsigned long long __V)
320 __builtin_ia32_wrgsbase64(__V);
326 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
335 static __inline__
short __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
336 _loadbe_i16(
void const * __P) {
340 return __builtin_bswap16(((
const struct __loadu_i16*)__P)->
__v);
343 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
344 _storebe_i16(
void * __P,
short __D) {
345 struct __storeu_i16 {
348 ((
struct __storeu_i16*)__P)->__v = __builtin_bswap16(
__D);
351 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
352 _loadbe_i32(
void const * __P) {
356 return __builtin_bswap32(((
const struct __loadu_i32*)__P)->
__v);
359 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
360 _storebe_i32(
void * __P,
int __D) {
361 struct __storeu_i32 {
364 ((
struct __storeu_i32*)__P)->__v = __builtin_bswap32(
__D);
368 static __inline__
long long __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
369 _loadbe_i64(
void const * __P) {
373 return __builtin_bswap64(((
const struct __loadu_i64*)__P)->
__v);
376 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
377 _storebe_i64(
void * __P,
long long __D) {
378 struct __storeu_i64 {
381 ((
struct __storeu_i64*)__P)->__v = __builtin_bswap64(
__D);
386 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
392 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
397 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
405 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
406 defined(__XSAVEOPT__)
410 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
415 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
420 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
429 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
434 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
435 defined(__WBNOINVD__)
439 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
440 defined(__CLDEMOTE__)
444 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
449 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
450 defined(__MOVDIRI__) || defined(__MOVDIR64B__)
454 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
459 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
464 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
469 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
474 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
475 defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
479 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
480 defined(__AVX512VP2INTERSECT__)
484 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
485 (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
489 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
494 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
495 defined(__SERIALIZE__)
499 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \
500 defined(__TSXLDTRK__)
504 #if defined(_MSC_VER) && __has_extension(gnu_asm)
506 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
513 #if defined(__i386__) || defined(__x86_64__)
515 _InterlockedExchange_HLEAcquire(
long volatile *_Target,
long _Value) {
516 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1"
517 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
521 _InterlockedExchange_HLERelease(
long volatile *_Target,
long _Value) {
522 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1"
523 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
527 #if defined(__x86_64__)
529 _InterlockedExchange64_HLEAcquire(__int64
volatile *_Target, __int64 _Value) {
530 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1"
531 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
535 _InterlockedExchange64_HLERelease(__int64
volatile *_Target, __int64 _Value) {
536 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1"
537 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
544 #if defined(__i386__) || defined(__x86_64__)
546 _InterlockedCompareExchange_HLEAcquire(
long volatile *_Destination,
547 long _Exchange,
long _Comparand) {
548 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1"
549 :
"+a" (_Comparand),
"+m" (*_Destination)
550 :
"r" (_Exchange) :
"memory");
554 _InterlockedCompareExchange_HLERelease(
long volatile *_Destination,
555 long _Exchange,
long _Comparand) {
556 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1"
557 :
"+a" (_Comparand),
"+m" (*_Destination)
558 :
"r" (_Exchange) :
"memory");
562 #if defined(__x86_64__)
564 _InterlockedCompareExchange64_HLEAcquire(__int64
volatile *_Destination,
565 __int64 _Exchange, __int64 _Comparand) {
566 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1"
567 :
"+a" (_Comparand),
"+m" (*_Destination)
568 :
"r" (_Exchange) :
"memory");
572 _InterlockedCompareExchange64_HLERelease(__int64
volatile *_Destination,
573 __int64 _Exchange, __int64 _Comparand) {
574 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1"
575 :
"+a" (_Comparand),
"+m" (*_Destination)
576 :
"r" (_Exchange) :
"memory");
584 #undef __DEFAULT_FN_ATTRS
#define __DEFAULT_FN_ATTRS
static __inline unsigned char unsigned int unsigned int unsigned int * __p
static __inline__ void short __D
struct __storeu_i16 *__P __v
static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) _rdpid_u32(void)
Returns the value of the IA32_TSC_AUX MSR (0xc0000103).