37#include "llvm/IR/IntrinsicsAArch64.h"
38#include "llvm/IR/IntrinsicsAMDGPU.h"
39#include "llvm/IR/IntrinsicsARM.h"
40#include "llvm/IR/IntrinsicsNVPTX.h"
41#include "llvm/IR/IntrinsicsRISCV.h"
42#include "llvm/IR/IntrinsicsWebAssembly.h"
43#include "llvm/IR/IntrinsicsX86.h"
66#define DEBUG_TYPE "auto-upgrade"
85 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
100 Type *LastArgType =
F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
116 if (
F->getReturnType()->isVectorTy())
129 Type *Arg1Type =
F->getFunctionType()->getParamType(1);
130 Type *Arg2Type =
F->getFunctionType()->getParamType(2);
147 Type *Arg1Type =
F->getFunctionType()->getParamType(1);
148 Type *Arg2Type =
F->getFunctionType()->getParamType(2);
162 if (
F->getReturnType()->getScalarType()->isBFloatTy())
172 if (
F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
186 if (Name.consume_front(
"avx."))
187 return (Name.starts_with(
"blend.p") ||
188 Name ==
"cvt.ps2.pd.256" ||
189 Name ==
"cvtdq2.pd.256" ||
190 Name ==
"cvtdq2.ps.256" ||
191 Name.starts_with(
"movnt.") ||
192 Name.starts_with(
"sqrt.p") ||
193 Name.starts_with(
"storeu.") ||
194 Name.starts_with(
"vbroadcast.s") ||
195 Name.starts_with(
"vbroadcastf128") ||
196 Name.starts_with(
"vextractf128.") ||
197 Name.starts_with(
"vinsertf128.") ||
198 Name.starts_with(
"vperm2f128.") ||
199 Name.starts_with(
"vpermil."));
201 if (Name.consume_front(
"avx2."))
202 return (Name ==
"movntdqa" ||
203 Name.starts_with(
"pabs.") ||
204 Name.starts_with(
"padds.") ||
205 Name.starts_with(
"paddus.") ||
206 Name.starts_with(
"pblendd.") ||
208 Name.starts_with(
"pbroadcast") ||
209 Name.starts_with(
"pcmpeq.") ||
210 Name.starts_with(
"pcmpgt.") ||
211 Name.starts_with(
"pmax") ||
212 Name.starts_with(
"pmin") ||
213 Name.starts_with(
"pmovsx") ||
214 Name.starts_with(
"pmovzx") ||
215 Name.starts_with(
"pmulh.w") ||
216 Name.starts_with(
"pmulhu.w") ||
218 Name ==
"pmulu.dq" ||
219 Name.starts_with(
"psll.dq") ||
220 Name.starts_with(
"psrl.dq") ||
221 Name.starts_with(
"psubs.") ||
222 Name.starts_with(
"psubus.") ||
223 Name.starts_with(
"vbroadcast") ||
224 Name ==
"vbroadcasti128" ||
225 Name ==
"vextracti128" ||
226 Name ==
"vinserti128" ||
227 Name ==
"vperm2i128");
229 if (Name.consume_front(
"avx512.")) {
230 if (Name.consume_front(
"mask."))
232 return (Name.starts_with(
"add.p") ||
233 Name.starts_with(
"and.") ||
234 Name.starts_with(
"andn.") ||
235 Name.starts_with(
"broadcast.s") ||
236 Name.starts_with(
"broadcastf32x4.") ||
237 Name.starts_with(
"broadcastf32x8.") ||
238 Name.starts_with(
"broadcastf64x2.") ||
239 Name.starts_with(
"broadcastf64x4.") ||
240 Name.starts_with(
"broadcasti32x4.") ||
241 Name.starts_with(
"broadcasti32x8.") ||
242 Name.starts_with(
"broadcasti64x2.") ||
243 Name.starts_with(
"broadcasti64x4.") ||
244 Name.starts_with(
"cmp.b") ||
245 Name.starts_with(
"cmp.d") ||
246 Name.starts_with(
"cmp.q") ||
247 Name.starts_with(
"cmp.w") ||
248 Name.starts_with(
"compress.b") ||
249 Name.starts_with(
"compress.d") ||
250 Name.starts_with(
"compress.p") ||
251 Name.starts_with(
"compress.q") ||
252 Name.starts_with(
"compress.store.") ||
253 Name.starts_with(
"compress.w") ||
254 Name.starts_with(
"conflict.") ||
255 Name.starts_with(
"cvtdq2pd.") ||
256 Name.starts_with(
"cvtdq2ps.") ||
257 Name ==
"cvtpd2dq.256" ||
258 Name ==
"cvtpd2ps.256" ||
259 Name ==
"cvtps2pd.128" ||
260 Name ==
"cvtps2pd.256" ||
261 Name.starts_with(
"cvtqq2pd.") ||
262 Name ==
"cvtqq2ps.256" ||
263 Name ==
"cvtqq2ps.512" ||
264 Name ==
"cvttpd2dq.256" ||
265 Name ==
"cvttps2dq.128" ||
266 Name ==
"cvttps2dq.256" ||
267 Name.starts_with(
"cvtudq2pd.") ||
268 Name.starts_with(
"cvtudq2ps.") ||
269 Name.starts_with(
"cvtuqq2pd.") ||
270 Name ==
"cvtuqq2ps.256" ||
271 Name ==
"cvtuqq2ps.512" ||
272 Name.starts_with(
"dbpsadbw.") ||
273 Name.starts_with(
"div.p") ||
274 Name.starts_with(
"expand.b") ||
275 Name.starts_with(
"expand.d") ||
276 Name.starts_with(
"expand.load.") ||
277 Name.starts_with(
"expand.p") ||
278 Name.starts_with(
"expand.q") ||
279 Name.starts_with(
"expand.w") ||
280 Name.starts_with(
"fpclass.p") ||
281 Name.starts_with(
"insert") ||
282 Name.starts_with(
"load.") ||
283 Name.starts_with(
"loadu.") ||
284 Name.starts_with(
"lzcnt.") ||
285 Name.starts_with(
"max.p") ||
286 Name.starts_with(
"min.p") ||
287 Name.starts_with(
"movddup") ||
288 Name.starts_with(
"move.s") ||
289 Name.starts_with(
"movshdup") ||
290 Name.starts_with(
"movsldup") ||
291 Name.starts_with(
"mul.p") ||
292 Name.starts_with(
"or.") ||
293 Name.starts_with(
"pabs.") ||
294 Name.starts_with(
"packssdw.") ||
295 Name.starts_with(
"packsswb.") ||
296 Name.starts_with(
"packusdw.") ||
297 Name.starts_with(
"packuswb.") ||
298 Name.starts_with(
"padd.") ||
299 Name.starts_with(
"padds.") ||
300 Name.starts_with(
"paddus.") ||
301 Name.starts_with(
"palignr.") ||
302 Name.starts_with(
"pand.") ||
303 Name.starts_with(
"pandn.") ||
304 Name.starts_with(
"pavg") ||
305 Name.starts_with(
"pbroadcast") ||
306 Name.starts_with(
"pcmpeq.") ||
307 Name.starts_with(
"pcmpgt.") ||
308 Name.starts_with(
"perm.df.") ||
309 Name.starts_with(
"perm.di.") ||
310 Name.starts_with(
"permvar.") ||
311 Name.starts_with(
"pmaddubs.w.") ||
312 Name.starts_with(
"pmaddw.d.") ||
313 Name.starts_with(
"pmax") ||
314 Name.starts_with(
"pmin") ||
315 Name ==
"pmov.qd.256" ||
316 Name ==
"pmov.qd.512" ||
317 Name ==
"pmov.wb.256" ||
318 Name ==
"pmov.wb.512" ||
319 Name.starts_with(
"pmovsx") ||
320 Name.starts_with(
"pmovzx") ||
321 Name.starts_with(
"pmul.dq.") ||
322 Name.starts_with(
"pmul.hr.sw.") ||
323 Name.starts_with(
"pmulh.w.") ||
324 Name.starts_with(
"pmulhu.w.") ||
325 Name.starts_with(
"pmull.") ||
326 Name.starts_with(
"pmultishift.qb.") ||
327 Name.starts_with(
"pmulu.dq.") ||
328 Name.starts_with(
"por.") ||
329 Name.starts_with(
"prol.") ||
330 Name.starts_with(
"prolv.") ||
331 Name.starts_with(
"pror.") ||
332 Name.starts_with(
"prorv.") ||
333 Name.starts_with(
"pshuf.b.") ||
334 Name.starts_with(
"pshuf.d.") ||
335 Name.starts_with(
"pshufh.w.") ||
336 Name.starts_with(
"pshufl.w.") ||
337 Name.starts_with(
"psll.d") ||
338 Name.starts_with(
"psll.q") ||
339 Name.starts_with(
"psll.w") ||
340 Name.starts_with(
"pslli") ||
341 Name.starts_with(
"psllv") ||
342 Name.starts_with(
"psra.d") ||
343 Name.starts_with(
"psra.q") ||
344 Name.starts_with(
"psra.w") ||
345 Name.starts_with(
"psrai") ||
346 Name.starts_with(
"psrav") ||
347 Name.starts_with(
"psrl.d") ||
348 Name.starts_with(
"psrl.q") ||
349 Name.starts_with(
"psrl.w") ||
350 Name.starts_with(
"psrli") ||
351 Name.starts_with(
"psrlv") ||
352 Name.starts_with(
"psub.") ||
353 Name.starts_with(
"psubs.") ||
354 Name.starts_with(
"psubus.") ||
355 Name.starts_with(
"pternlog.") ||
356 Name.starts_with(
"punpckh") ||
357 Name.starts_with(
"punpckl") ||
358 Name.starts_with(
"pxor.") ||
359 Name.starts_with(
"shuf.f") ||
360 Name.starts_with(
"shuf.i") ||
361 Name.starts_with(
"shuf.p") ||
362 Name.starts_with(
"sqrt.p") ||
363 Name.starts_with(
"store.b.") ||
364 Name.starts_with(
"store.d.") ||
365 Name.starts_with(
"store.p") ||
366 Name.starts_with(
"store.q.") ||
367 Name.starts_with(
"store.w.") ||
368 Name ==
"store.ss" ||
369 Name.starts_with(
"storeu.") ||
370 Name.starts_with(
"sub.p") ||
371 Name.starts_with(
"ucmp.") ||
372 Name.starts_with(
"unpckh.") ||
373 Name.starts_with(
"unpckl.") ||
374 Name.starts_with(
"valign.") ||
375 Name ==
"vcvtph2ps.128" ||
376 Name ==
"vcvtph2ps.256" ||
377 Name.starts_with(
"vextract") ||
378 Name.starts_with(
"vfmadd.") ||
379 Name.starts_with(
"vfmaddsub.") ||
380 Name.starts_with(
"vfnmadd.") ||
381 Name.starts_with(
"vfnmsub.") ||
382 Name.starts_with(
"vpdpbusd.") ||
383 Name.starts_with(
"vpdpbusds.") ||
384 Name.starts_with(
"vpdpwssd.") ||
385 Name.starts_with(
"vpdpwssds.") ||
386 Name.starts_with(
"vpermi2var.") ||
387 Name.starts_with(
"vpermil.p") ||
388 Name.starts_with(
"vpermilvar.") ||
389 Name.starts_with(
"vpermt2var.") ||
390 Name.starts_with(
"vpmadd52") ||
391 Name.starts_with(
"vpshld.") ||
392 Name.starts_with(
"vpshldv.") ||
393 Name.starts_with(
"vpshrd.") ||
394 Name.starts_with(
"vpshrdv.") ||
395 Name.starts_with(
"vpshufbitqmb.") ||
396 Name.starts_with(
"xor."));
398 if (Name.consume_front(
"mask3."))
400 return (Name.starts_with(
"vfmadd.") ||
401 Name.starts_with(
"vfmaddsub.") ||
402 Name.starts_with(
"vfmsub.") ||
403 Name.starts_with(
"vfmsubadd.") ||
404 Name.starts_with(
"vfnmsub."));
406 if (Name.consume_front(
"maskz."))
408 return (Name.starts_with(
"pternlog.") ||
409 Name.starts_with(
"vfmadd.") ||
410 Name.starts_with(
"vfmaddsub.") ||
411 Name.starts_with(
"vpdpbusd.") ||
412 Name.starts_with(
"vpdpbusds.") ||
413 Name.starts_with(
"vpdpwssd.") ||
414 Name.starts_with(
"vpdpwssds.") ||
415 Name.starts_with(
"vpermt2var.") ||
416 Name.starts_with(
"vpmadd52") ||
417 Name.starts_with(
"vpshldv.") ||
418 Name.starts_with(
"vpshrdv."));
421 return (Name ==
"movntdqa" ||
422 Name ==
"pmul.dq.512" ||
423 Name ==
"pmulu.dq.512" ||
424 Name.starts_with(
"broadcastm") ||
425 Name.starts_with(
"cmp.p") ||
426 Name.starts_with(
"cvtb2mask.") ||
427 Name.starts_with(
"cvtd2mask.") ||
428 Name.starts_with(
"cvtmask2") ||
429 Name.starts_with(
"cvtq2mask.") ||
430 Name ==
"cvtusi2sd" ||
431 Name.starts_with(
"cvtw2mask.") ||
436 Name ==
"kortestc.w" ||
437 Name ==
"kortestz.w" ||
438 Name.starts_with(
"kunpck") ||
441 Name.starts_with(
"padds.") ||
442 Name.starts_with(
"pbroadcast") ||
443 Name.starts_with(
"pmulh.w") ||
444 Name.starts_with(
"pmulhu.w") ||
445 Name.starts_with(
"prol") ||
446 Name.starts_with(
"pror") ||
447 Name.starts_with(
"psll.dq") ||
448 Name.starts_with(
"psrl.dq") ||
449 Name.starts_with(
"psubs.") ||
450 Name.starts_with(
"ptestm") ||
451 Name.starts_with(
"ptestnm") ||
452 Name.starts_with(
"storent.") ||
453 Name.starts_with(
"vbroadcast.s") ||
454 Name.starts_with(
"vpshld.") ||
455 Name.starts_with(
"vpshrd."));
458 if (Name.consume_front(
"fma."))
459 return (Name.starts_with(
"vfmadd.") ||
460 Name.starts_with(
"vfmsub.") ||
461 Name.starts_with(
"vfmsubadd.") ||
462 Name.starts_with(
"vfnmadd.") ||
463 Name.starts_with(
"vfnmsub."));
465 if (Name.consume_front(
"fma4."))
466 return Name.starts_with(
"vfmadd.s");
468 if (Name.consume_front(
"sse."))
469 return (Name ==
"add.ss" ||
470 Name ==
"cvtsi2ss" ||
471 Name ==
"cvtsi642ss" ||
474 Name.starts_with(
"sqrt.p") ||
476 Name.starts_with(
"storeu.") ||
479 if (Name.consume_front(
"sse2."))
480 return (Name ==
"add.sd" ||
481 Name ==
"cvtdq2pd" ||
482 Name ==
"cvtdq2ps" ||
483 Name ==
"cvtps2pd" ||
484 Name ==
"cvtsi2sd" ||
485 Name ==
"cvtsi642sd" ||
486 Name ==
"cvtss2sd" ||
489 Name.starts_with(
"padds.") ||
490 Name.starts_with(
"paddus.") ||
491 Name.starts_with(
"pcmpeq.") ||
492 Name.starts_with(
"pcmpgt.") ||
498 Name ==
"pmulhu.w" ||
499 Name ==
"pmulu.dq" ||
500 Name.starts_with(
"pshuf") ||
501 Name.starts_with(
"psll.dq") ||
502 Name.starts_with(
"psrl.dq") ||
503 Name.starts_with(
"psubs.") ||
504 Name.starts_with(
"psubus.") ||
505 Name.starts_with(
"sqrt.p") ||
507 Name ==
"storel.dq" ||
508 Name.starts_with(
"storeu.") ||
511 if (Name.consume_front(
"sse41."))
512 return (Name.starts_with(
"blendp") ||
513 Name ==
"movntdqa" ||
523 Name.starts_with(
"pmovsx") ||
524 Name.starts_with(
"pmovzx") ||
527 if (Name.consume_front(
"sse42."))
528 return Name ==
"crc32.64.8";
530 if (Name.consume_front(
"sse4a."))
531 return Name.starts_with(
"movnt.");
533 if (Name.consume_front(
"ssse3."))
534 return (Name ==
"pabs.b.128" ||
535 Name ==
"pabs.d.128" ||
536 Name ==
"pabs.w.128");
538 if (Name.consume_front(
"xop."))
539 return (Name ==
"vpcmov" ||
540 Name ==
"vpcmov.256" ||
541 Name.starts_with(
"vpcom") ||
542 Name.starts_with(
"vprot"));
544 if (Name.consume_front(
"bmi."))
545 return (Name.starts_with(
"pdep.") ||
546 Name.starts_with(
"pext."));
548 return (Name ==
"addcarry.u32" ||
549 Name ==
"addcarry.u64" ||
550 Name ==
"addcarryx.u32" ||
551 Name ==
"addcarryx.u64" ||
552 Name ==
"subborrow.u32" ||
553 Name ==
"subborrow.u64" ||
554 Name.starts_with(
"vcvtph2ps."));
560 if (!Name.consume_front(
"x86."))
568 if (Name ==
"rdtscp") {
570 if (
F->getFunctionType()->getNumParams() == 0)
575 Intrinsic::x86_rdtscp);
582 if (Name.consume_front(
"sse41.ptest")) {
584 .
Case(
"c", Intrinsic::x86_sse41_ptestc)
585 .
Case(
"z", Intrinsic::x86_sse41_ptestz)
586 .
Case(
"nzc", Intrinsic::x86_sse41_ptestnzc)
599 .
Case(
"sse41.insertps", Intrinsic::x86_sse41_insertps)
600 .
Case(
"sse41.dppd", Intrinsic::x86_sse41_dppd)
601 .
Case(
"sse41.dpps", Intrinsic::x86_sse41_dpps)
602 .
Case(
"sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
603 .
Case(
"avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
604 .
Case(
"avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
609 if (Name.consume_front(
"avx512.")) {
610 if (Name.consume_front(
"mask.cmp.")) {
613 .
Case(
"pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
614 .
Case(
"pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
615 .
Case(
"pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
616 .
Case(
"ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
617 .
Case(
"ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
618 .
Case(
"ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
622 }
else if (Name.starts_with(
"vpdpbusd.") ||
623 Name.starts_with(
"vpdpbusds.")) {
626 .
Case(
"vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
627 .
Case(
"vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
628 .
Case(
"vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
629 .
Case(
"vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
630 .
Case(
"vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
631 .
Case(
"vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
635 }
else if (Name.starts_with(
"vpdpwssd.") ||
636 Name.starts_with(
"vpdpwssds.")) {
639 .
Case(
"vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
640 .
Case(
"vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
641 .
Case(
"vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
642 .
Case(
"vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
643 .
Case(
"vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
644 .
Case(
"vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
652 if (Name.consume_front(
"avx2.")) {
653 if (Name.consume_front(
"vpdpb")) {
656 .
Case(
"ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
657 .
Case(
"ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
658 .
Case(
"ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
659 .
Case(
"ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
660 .
Case(
"sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
661 .
Case(
"sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
662 .
Case(
"suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
663 .
Case(
"suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
664 .
Case(
"uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
665 .
Case(
"uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
666 .
Case(
"uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
667 .
Case(
"uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
671 }
else if (Name.consume_front(
"vpdpw")) {
674 .
Case(
"sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
675 .
Case(
"sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
676 .
Case(
"suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
677 .
Case(
"suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
678 .
Case(
"usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
679 .
Case(
"usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
680 .
Case(
"usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
681 .
Case(
"usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
682 .
Case(
"uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
683 .
Case(
"uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
684 .
Case(
"uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
685 .
Case(
"uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
693 if (Name.consume_front(
"avx10.")) {
694 if (Name.consume_front(
"vpdpb")) {
697 .
Case(
"ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
698 .
Case(
"ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
699 .
Case(
"sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
700 .
Case(
"suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
701 .
Case(
"uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
702 .
Case(
"uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
706 }
else if (Name.consume_front(
"vpdpw")) {
708 .
Case(
"sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
709 .
Case(
"suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
710 .
Case(
"usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
711 .
Case(
"usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
712 .
Case(
"uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
713 .
Case(
"uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
721 if (Name.consume_front(
"avx512bf16.")) {
724 .
Case(
"cvtne2ps2bf16.128",
725 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
726 .
Case(
"cvtne2ps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
728 .
Case(
"cvtne2ps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
730 .
Case(
"mask.cvtneps2bf16.128",
731 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
732 .
Case(
"cvtneps2bf16.256",
733 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
734 .
Case(
"cvtneps2bf16.512",
735 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
742 .
Case(
"dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
743 .
Case(
"dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
744 .
Case(
"dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
751 if (Name.consume_front(
"xop.")) {
753 if (Name.starts_with(
"vpermil2")) {
756 auto Idx =
F->getFunctionType()->getParamType(2);
757 if (Idx->isFPOrFPVectorTy()) {
758 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
759 unsigned EltSize = Idx->getScalarSizeInBits();
760 if (EltSize == 64 && IdxSize == 128)
761 ID = Intrinsic::x86_xop_vpermil2pd;
762 else if (EltSize == 32 && IdxSize == 128)
763 ID = Intrinsic::x86_xop_vpermil2ps;
764 else if (EltSize == 64 && IdxSize == 256)
765 ID = Intrinsic::x86_xop_vpermil2pd_256;
767 ID = Intrinsic::x86_xop_vpermil2ps_256;
769 }
else if (
F->arg_size() == 2)
772 .
Case(
"vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
773 .
Case(
"vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
784 if (Name ==
"seh.recoverfp") {
786 Intrinsic::eh_recoverfp);
798 if (Name.starts_with(
"rbit")) {
801 F->getParent(), Intrinsic::bitreverse,
F->arg_begin()->getType());
805 if (Name ==
"thread.pointer") {
808 F->getParent(), Intrinsic::thread_pointer,
F->getReturnType());
812 bool Neon = Name.consume_front(
"neon.");
817 if (Name.consume_front(
"bfdot.")) {
821 .
Cases({
"v2f32.v8i8",
"v4f32.v16i8"},
826 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
827 assert((OperandWidth == 64 || OperandWidth == 128) &&
828 "Unexpected operand width");
830 std::array<Type *, 2> Tys{
841 if (Name.consume_front(
"bfm")) {
843 if (Name.consume_back(
".v4f32.v16i8")) {
889 F->arg_begin()->getType());
893 if (Name.consume_front(
"vst")) {
895 static const Regex vstRegex(
"^([1234]|[234]lane)\\.v[a-z0-9]*$");
899 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
900 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
903 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
904 Intrinsic::arm_neon_vst4lane};
906 auto fArgs =
F->getFunctionType()->params();
907 Type *Tys[] = {fArgs[0], fArgs[1]};
910 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
913 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
922 if (Name.consume_front(
"mve.")) {
924 if (Name ==
"vctp64") {
934 if (Name.starts_with(
"vrintn.v")) {
936 F->getParent(), Intrinsic::roundeven,
F->arg_begin()->getType());
941 if (Name.consume_back(
".v4i1")) {
943 if (Name.consume_back(
".predicated.v2i64.v4i32"))
945 return Name ==
"mull.int" || Name ==
"vqdmull";
947 if (Name.consume_back(
".v2i64")) {
949 bool IsGather = Name.consume_front(
"vldr.gather.");
950 if (IsGather || Name.consume_front(
"vstr.scatter.")) {
951 if (Name.consume_front(
"base.")) {
953 Name.consume_front(
"wb.");
956 return Name ==
"predicated.v2i64";
959 if (Name.consume_front(
"offset.predicated."))
960 return Name == (IsGather ?
"v2i64.p0i64" :
"p0i64.v2i64") ||
961 Name == (IsGather ?
"v2i64.p0" :
"p0.v2i64");
974 if (Name.consume_front(
"cde.vcx")) {
976 if (Name.consume_back(
".predicated.v2i64.v4i1"))
978 return Name ==
"1q" || Name ==
"1qa" || Name ==
"2q" || Name ==
"2qa" ||
979 Name ==
"3q" || Name ==
"3qa";
993 F->arg_begin()->getType());
999 .
Case(
"smax", Intrinsic::smax)
1000 .
Case(
"smin", Intrinsic::smin)
1001 .
Case(
"umax", Intrinsic::umax)
1002 .
Case(
"umin", Intrinsic::umin)
1005 if (
F->arg_size() != 2 || !
F->getReturnType()->isIntOrIntVectorTy())
1008 F->getReturnType());
1012 if (Name.starts_with(
"addp")) {
1014 if (
F->arg_size() != 2)
1017 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1019 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1025 if (Name.starts_with(
"bfcvt")) {
1031 if (Name ==
"vcvtfp2hf" || Name ==
"vcvthf2fp") {
1038 if (Name.consume_front(
"sve.")) {
1040 if (Name.consume_front(
"bf")) {
1041 if (Name ==
"mmla") {
1042 Type *Tys[] = {
F->getReturnType(),
1043 std::next(
F->arg_begin())->getType()};
1045 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1048 if (Name.consume_back(
".lane")) {
1052 .
Case(
"dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1053 .
Case(
"mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1054 .
Case(
"mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1066 if (Name ==
"fcvt.bf16f32" || Name ==
"fcvtnt.bf16f32") {
1071 if (Name.consume_front(
"convert.from.svbool")) {
1074 if (!TTy || TTy->getName() !=
"aarch64.svcount")
1077 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1082 if (Name.consume_front(
"convert.to.svbool")) {
1085 if (!TTy || TTy->getName() !=
"aarch64.svcount")
1088 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1093 if (Name.consume_front(
"addqv")) {
1095 if (!
F->getReturnType()->isFPOrFPVectorTy())
1098 auto Args =
F->getFunctionType()->params();
1099 Type *Tys[] = {
F->getReturnType(), Args[1]};
1101 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1105 if (Name.consume_front(
"ld")) {
1107 static const Regex LdRegex(
"^[234](.nxv[a-z0-9]+|$)");
1108 if (LdRegex.
match(Name)) {
1114 "Expected 2 arguments for ld* intrinsic.");
1115 Type *PtrTy =
F->getArg(1)->getType();
1118 Intrinsic::aarch64_sve_ld2_sret,
1119 Intrinsic::aarch64_sve_ld3_sret,
1120 Intrinsic::aarch64_sve_ld4_sret,
1123 F->getParent(), LoadIDs[Name[0] -
'2'], {Ty, PtrTy});
1129 if (Name.consume_front(
"tuple.")) {
1131 if (Name.starts_with(
"get")) {
1133 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
1135 F->getParent(), Intrinsic::vector_extract, Tys);
1139 if (Name.starts_with(
"set")) {
1141 auto Args =
F->getFunctionType()->params();
1142 Type *Tys[] = {Args[0], Args[2], Args[1]};
1144 F->getParent(), Intrinsic::vector_insert, Tys);
1148 static const Regex CreateTupleRegex(
"^create[234](.nxv[a-z0-9]+|$)");
1149 if (CreateTupleRegex.
match(Name)) {
1151 auto Args =
F->getFunctionType()->params();
1152 Type *Tys[] = {
F->getReturnType(), Args[1]};
1154 F->getParent(), Intrinsic::vector_insert, Tys);
1160 if (Name.starts_with(
"rev.nxv")) {
1163 F->getParent(), Intrinsic::vector_reverse,
F->getReturnType());
1169 if (Name.consume_front(
"sme.")) {
1171 if (Name.consume_front(
"ftmopa.")) {
1176 .
Case(
"za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1177 .
Case(
"za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1195#define NVVM_TMA_G2S_MODES(M) \
1196 M(tile_1d, "tile.1d") \
1197 M(tile_2d, "tile.2d") \
1198 M(tile_3d, "tile.3d") \
1199 M(tile_4d, "tile.4d") \
1200 M(tile_5d, "tile.5d") \
1201 M(tile_gather4_2d, "tile.gather4.2d") \
1202 M(im2col_3d, "im2col.3d") \
1203 M(im2col_4d, "im2col.4d") \
1204 M(im2col_5d, "im2col.5d") \
1205 M(im2col_w_3d, "im2col.w.3d") \
1206 M(im2col_w_4d, "im2col.w.4d") \
1207 M(im2col_w_5d, "im2col.w.5d") \
1208 M(im2col_w_128_3d, "im2col.w.128.3d") \
1209 M(im2col_w_128_4d, "im2col.w.128.4d") \
1210 M(im2col_w_128_5d, "im2col.w.128.5d")
1222 if (!Name.consume_front(
"cp.async.bulk.tensor.g2s."))
1225#define G2S_ID(ID_SUFFIX, NAME) \
1226 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1236 size_t NumParams =
F->getFunctionType()->getNumParams();
1240 if (!
F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1247 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1248 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1249 "expected the i64 cache-hint after the multicast mask");
1250 Type *MaskTy = Params[MaskIdx];
1265 if (!Name.consume_front(
"cp.async.bulk.tensor.g2s.cta."))
1268#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1269 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1281 if (!
F->getFunctionType()
1282 ->getParamType(
F->getFunctionType()->getNumParams() - 1)
1298 if (!Name.consume_front(
"cp.async.bulk.global.to.shared.cluster"))
1303 size_t NumParams =
F->getFunctionType()->getNumParams();
1304 if (!
F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1308 Type *MaskTy =
F->getFunctionType()->getParamType(NumParams - 4);
1313 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1326 if (!Name.consume_front(
"cp.async.bulk.global.to.shared.cta"))
1331 if (!
F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1334 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1354 if (!Name.consume_front(
"cp.async.bulk.tensor.reduce."))
1357 auto [RedOpName, ShapeName] = Name.split(
'.');
1362 .
Case(
"tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1363 .
Case(
"tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1364 .
Case(
"tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1365 .
Case(
"tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1366 .
Case(
"tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1367 .
Case(
"im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1368 .
Case(
"im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1369 .
Case(
"im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1375 if (!Name.consume_front(
"mbarrier.") ||
1376 !Name.consume_back(
".space.cluster") ||
F->arg_size() != 2)
1389 if (Name.consume_front(
"mapa.shared.cluster"))
1390 if (
F->getReturnType()->getPointerAddressSpace() ==
1392 return Intrinsic::nvvm_mapa_shared_cluster;
1394 if (Name.consume_front(
"cp.async.bulk.")) {
1397 .
Case(
"shared.cta.to.cluster",
1398 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1402 if (
F->getArg(0)->getType()->getPointerAddressSpace() ==
1412 if (!Name.consume_front(
"tcgen05.commit."))
1415 if (Name.consume_front(
"shared."))
1417 .
Case(
"cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1418 .
Case(
"cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1421 if (Name.consume_front(
"mc.shared.")) {
1423 if (!
F->getArg(1)->getType()->isIntegerTy(16))
1427 .
Case(
"cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1428 .
Case(
"cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1437 if (
F->arg_size() != 2)
1441 .
Case(
"tcgen05.alloc.shared.cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1442 .
Case(
"tcgen05.alloc.shared.cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1447 if (Name.consume_front(
"fmax."))
1449 .
Case(
"bf16", Intrinsic::nvvm_fmax_bf16)
1450 .
Case(
"bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1451 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1452 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1453 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1454 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1455 .
Case(
"ftz.nan.xorsign.abs.bf16",
1456 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1457 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
1458 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1459 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1460 .
Case(
"ftz.xorsign.abs.bf16x2",
1461 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1462 .
Case(
"nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1463 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1464 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1465 .
Case(
"nan.xorsign.abs.bf16x2",
1466 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1467 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1468 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1471 if (Name.consume_front(
"fmin."))
1473 .
Case(
"bf16", Intrinsic::nvvm_fmin_bf16)
1474 .
Case(
"bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1475 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1476 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1477 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1478 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1479 .
Case(
"ftz.nan.xorsign.abs.bf16",
1480 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1481 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
1482 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1483 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1484 .
Case(
"ftz.xorsign.abs.bf16x2",
1485 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1486 .
Case(
"nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1487 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1488 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1489 .
Case(
"nan.xorsign.abs.bf16x2",
1490 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1491 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1492 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1495 if (Name.consume_front(
"neg."))
1497 .
Case(
"bf16", Intrinsic::nvvm_neg_bf16)
1498 .
Case(
"bf16x2", Intrinsic::nvvm_neg_bf16x2)
1507 auto IsOldBF16StorageTy = [](
Type *OldTy,
Type *NewTy) {
1512 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1515 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1518 for (
unsigned I = 0,
E = OldFnTy->getNumParams();
I !=
E; ++
I)
1519 if (!IsOldBF16StorageTy(OldFnTy->getParamType(
I), NewFnTy->getParamType(
I)))
1529 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
1533 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
1536 {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
1537 {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
1538 Intrinsic::nvvm_fma_ftz_relu}};
1544 bool IsFTZ,
bool IsSat,
1549 .
Case(
"f", !IsRelu && !(IsSat && IsFMul))
1550 .
Case(
"d", !IsFTZ && !IsSat && !IsRelu)
1551 .
Case(
"f16", IsRN && (IsFMA || IsSat))
1552 .
Case(
"v2f16", IsRN && !IsFMA && IsSat)
1553 .
Case(
"f16x2", IsRN && IsFMA)
1554 .
Cases({
"bf16",
"bf16x2"}, IsRN && IsFMA && !IsFTZ && !IsSat)
1558static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1560 auto [Modifiers,
Type] = Name.rsplit(
'.');
1570 return std::nullopt;
1572 StringRef Rest = Modifiers.drop_front(2);
1577 return std::nullopt;
1579 const Intrinsic::ID IID = IIDs[IsFTZ][IsSat ? 1 : IsRelu ? 2 : 0];
1581 return std::nullopt;
1587 return std::nullopt;
1593 if (Name !=
"mbarrier.init" && Name !=
"mbarrier.init.shared")
1596 return Intrinsic::nvvm_mbarrier_init;
1600 return Name.consume_front(
"local") || Name.consume_front(
"shared") ||
1601 Name.consume_front(
"global") || Name.consume_front(
"constant") ||
1602 Name.consume_front(
"param");
1606 if (!Name.consume_front(
"vp."))
1635 .
StartsWith(
"ptrtoint", Instruction::PtrToInt)
1636 .
StartsWith(
"inttoptr", Instruction::IntToPtr)
1643 if (!Name.consume_front(
"vp."))
1663 .
StartsWith(
"nearbyint", Intrinsic::nearbyint)
1664 .
StartsWith(
"roundeven", Intrinsic::roundeven)
1669 .
StartsWith(
"bitreverse", Intrinsic::bitreverse)
1681 .
StartsWith(
"is.fpclass", Intrinsic::is_fpclass)
1692 if (Name.starts_with(
"to.fp16")) {
1696 FuncTy->getReturnType());
1699 if (Name.starts_with(
"from.fp16")) {
1703 FuncTy->getReturnType());
1713 if (Defaults.empty())
1716 unsigned FullArgCount = FirstDefault + Defaults.size();
1719 if (
F->arg_size() < FirstDefault ||
F->arg_size() >= FullArgCount)
1722 unsigned NumMissingTrailingParams = FullArgCount -
F->arg_size();
1724 NumMissingTrailingParams))
1727 return FullArgCount;
1734 unsigned FullArgCount =
1736 if (FullArgCount == 0)
1742 "total number of default args does not match intrinsic signature");
1747 bool CanUpgradeDebugIntrinsicsToRecords) {
1748 assert(
F &&
"Illegal to upgrade a non-existent Function.");
1753 if (!Name.consume_front(
"llvm.") || Name.empty())
1759 bool IsArm = Name.consume_front(
"arm.");
1760 if (IsArm || Name.consume_front(
"aarch64.")) {
1766 if (Name.consume_front(
"amdgcn.")) {
1767 if (Name ==
"alignbit") {
1770 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1774 if (Name.consume_front(
"atomic.")) {
1775 if (Name.starts_with(
"inc") || Name.starts_with(
"dec") ||
1776 Name.starts_with(
"cond.sub") || Name.starts_with(
"csub")) {
1785 if (Name.starts_with(
"addrspacecast.nonnull")) {
1792 switch (
F->getIntrinsicID()) {
1796 case Intrinsic::amdgcn_asyncmark:
1797 case Intrinsic::amdgcn_wait_asyncmark: {
1799 if (
F->arg_size() == (ID == Intrinsic::amdgcn_asyncmark ? 0u : 1u)) {
1807 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1808 if (
F->arg_size() == 7) {
1813 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1814 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1815 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1816 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1817 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1818 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1819 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1820 if (
F->arg_size() == 8) {
1827 if (Name.consume_front(
"ds.") || Name.consume_front(
"global.atomic.") ||
1828 Name.consume_front(
"flat.atomic.")) {
1829 if (Name.starts_with(
"fadd") ||
1831 (Name.starts_with(
"fmin") && !Name.starts_with(
"fmin.num")) ||
1832 (Name.starts_with(
"fmax") && !Name.starts_with(
"fmax.num"))) {
1840 if (Name.starts_with(
"fcmp.") || Name.starts_with(
"icmp.")) {
1845 if (Name.starts_with(
"ldexp.")) {
1848 F->getParent(), Intrinsic::ldexp,
1849 {F->getReturnType(), F->getArg(1)->getType()});
1858 if (
F->arg_size() == 1) {
1859 if (Name.consume_front(
"convert.")) {
1873 F->arg_begin()->getType());
1879 if (Name ==
"coro.end" &&
1880 (
F->arg_size() == 2 ||
F->getReturnType()->isIntegerTy(1)))
1881 CoroEndID = Intrinsic::coro_end;
1882 else if (Name ==
"coro.end.async" &&
F->getReturnType()->isIntegerTy(1))
1883 CoroEndID = Intrinsic::coro_end_async;
1894 if (Name.consume_front(
"dbg.")) {
1896 if (CanUpgradeDebugIntrinsicsToRecords) {
1897 if (Name ==
"addr" || Name ==
"value" || Name ==
"assign" ||
1898 Name ==
"declare" || Name ==
"label") {
1907 if (Name ==
"addr" || (Name ==
"value" &&
F->arg_size() == 4)) {
1910 Intrinsic::dbg_value);
1917 if (Name.consume_front(
"experimental.vector.")) {
1923 .
StartsWith(
"extract.", Intrinsic::vector_extract)
1924 .
StartsWith(
"insert.", Intrinsic::vector_insert)
1925 .
StartsWith(
"reverse.", Intrinsic::vector_reverse)
1926 .
StartsWith(
"interleave2.", Intrinsic::vector_interleave2)
1927 .
StartsWith(
"deinterleave2.", Intrinsic::vector_deinterleave2)
1929 Intrinsic::vector_partial_reduce_add)
1932 const auto *FT =
F->getFunctionType();
1934 if (ID == Intrinsic::vector_extract ||
1935 ID == Intrinsic::vector_interleave2)
1938 if (ID != Intrinsic::vector_interleave2)
1940 if (ID == Intrinsic::vector_insert ||
1941 ID == Intrinsic::vector_partial_reduce_add)
1949 if (Name.consume_front(
"reduce.")) {
1951 static const Regex R(
"^([a-z]+)\\.[a-z][0-9]+");
1952 if (R.match(Name, &
Groups))
1954 .
Case(
"add", Intrinsic::vector_reduce_add)
1955 .
Case(
"mul", Intrinsic::vector_reduce_mul)
1956 .
Case(
"and", Intrinsic::vector_reduce_and)
1957 .
Case(
"or", Intrinsic::vector_reduce_or)
1958 .
Case(
"xor", Intrinsic::vector_reduce_xor)
1959 .
Case(
"smax", Intrinsic::vector_reduce_smax)
1960 .
Case(
"smin", Intrinsic::vector_reduce_smin)
1961 .
Case(
"umax", Intrinsic::vector_reduce_umax)
1962 .
Case(
"umin", Intrinsic::vector_reduce_umin)
1963 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
1964 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
1969 static const Regex R2(
"^v2\\.([a-z]+)\\.[fi][0-9]+");
1974 .
Case(
"fadd", Intrinsic::vector_reduce_fadd)
1975 .
Case(
"fmul", Intrinsic::vector_reduce_fmul)
1980 auto Args =
F->getFunctionType()->params();
1982 {Args[V2 ? 1 : 0]});
1988 if (Name.consume_front(
"splice"))
1992 if (Name.consume_front(
"experimental.stepvector.")) {
1996 F->getParent(), ID,
F->getFunctionType()->getReturnType());
2001 if (Name.starts_with(
"flt.rounds")) {
2004 Intrinsic::get_rounding);
2009 if (Name.starts_with(
"invariant.group.barrier")) {
2011 auto Args =
F->getFunctionType()->params();
2012 Type* ObjectPtr[1] = {Args[0]};
2015 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
2020 bool IsLifetimeStart = Name.consume_front(
"lifetime.start");
2021 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front(
"lifetime.end");
2022 if (IsLifetimeStart || IsLifetimeEnd) {
2023 if (
F->arg_size() == 2) {
2024 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
2025 : Intrinsic::lifetime_end;
2030 F->getArg(1)->getType());
2032 }
else if (
F->arg_size() == 1 && Name ==
".i64") {
2052 .StartsWith(
"memcpy.", Intrinsic::memcpy)
2053 .StartsWith(
"memmove.", Intrinsic::memmove)
2055 if (
F->arg_size() == 5) {
2059 F->getFunctionType()->params().slice(0, 3);
2065 if (Name.starts_with(
"memset.") &&
F->arg_size() == 5) {
2068 const auto *FT =
F->getFunctionType();
2069 Type *ParamTypes[2] = {
2070 FT->getParamType(0),
2074 Intrinsic::memset, ParamTypes);
2080 .
StartsWith(
"masked.load", Intrinsic::masked_load)
2081 .
StartsWith(
"masked.gather", Intrinsic::masked_gather)
2082 .
StartsWith(
"masked.store", Intrinsic::masked_store)
2083 .
StartsWith(
"masked.scatter", Intrinsic::masked_scatter)
2085 if (MaskedID &&
F->arg_size() == 4) {
2087 if (MaskedID == Intrinsic::masked_load ||
2088 MaskedID == Intrinsic::masked_gather) {
2090 F->getParent(), MaskedID,
2091 {F->getReturnType(), F->getArg(0)->getType()});
2095 F->getParent(), MaskedID,
2096 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2102 if (Name.consume_front(
"nvvm.")) {
2104 if (
F->arg_size() == 1) {
2107 .
Cases({
"brev32",
"brev64"}, Intrinsic::bitreverse)
2108 .Case(
"clz.i", Intrinsic::ctlz)
2109 .
Case(
"popc.i", Intrinsic::ctpop)
2113 {F->getReturnType()});
2116 }
else if (
F->arg_size() == 2) {
2119 .
Cases({
"max.s",
"max.i",
"max.ll"}, Intrinsic::smax)
2120 .Cases({
"min.s",
"min.i",
"min.ll"}, Intrinsic::smin)
2121 .Cases({
"max.us",
"max.ui",
"max.ull"}, Intrinsic::umax)
2122 .Cases({
"min.us",
"min.ui",
"min.ull"}, Intrinsic::umin)
2123 .Cases({
"mulhi.s",
"mulhi.i",
"mulhi.ll"}, Intrinsic::smulh)
2124 .Cases({
"mulhi.us",
"mulhi.ui",
"mulhi.ull"}, Intrinsic::umulh)
2128 {F->getReturnType()});
2174 F->getParent(), IID,
F->getReturnType(),
2175 F->getFunctionType()->params());
2184 {F->getArg(0)->getType()});
2230 F->getArg(0)->getType());
2238 bool Expand =
false;
2239 if (Name.consume_front(
"abs."))
2242 Name ==
"i" || Name ==
"ll" || Name ==
"bf16" || Name ==
"bf16x2";
2243 else if (Name.consume_front(
"fabs."))
2245 Expand = Name ==
"f" || Name ==
"ftz.f" || Name ==
"d";
2246 else if (Name.consume_front(
"add."))
2249 else if (Name.consume_front(
"mul."))
2252 else if (Name.consume_front(
"ex2.approx."))
2255 Name ==
"f" || Name ==
"ftz.f" || Name ==
"d" || Name ==
"f16x2";
2256 else if (Name.consume_front(
"fma."))
2259 else if (Name.consume_front(
"atomic.load."))
2268 else if (Name.consume_front(
"atomic."))
2283 else if (Name.consume_front(
"bitcast."))
2286 Name ==
"f2i" || Name ==
"i2f" || Name ==
"ll2d" || Name ==
"d2ll";
2287 else if (Name.consume_front(
"rotate."))
2289 Expand = Name ==
"b32" || Name ==
"b64" || Name ==
"right.b64";
2290 else if (Name.consume_front(
"ptr.gen.to."))
2293 else if (Name.consume_front(
"ptr."))
2296 else if (Name.consume_front(
"ldg.global."))
2298 Expand = (Name.starts_with(
"i.") || Name.starts_with(
"f.") ||
2299 Name.starts_with(
"p."));
2302 .
Case(
"barrier0",
true)
2303 .
Case(
"barrier.n",
true)
2304 .
Case(
"barrier.sync.cnt",
true)
2305 .
Case(
"barrier.sync",
true)
2306 .
Case(
"barrier",
true)
2307 .
Case(
"bar.sync",
true)
2308 .
Case(
"barrier0.popc",
true)
2309 .
Case(
"barrier0.and",
true)
2310 .
Case(
"barrier0.or",
true)
2311 .
Case(
"clz.ll",
true)
2312 .
Case(
"popc.ll",
true)
2314 .
Case(
"swap.lo.hi.b64",
true)
2315 .
Case(
"tanh.approx.f32",
true)
2327 if (Name.starts_with(
"objectsize.")) {
2328 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
2329 if (
F->arg_size() == 2 ||
F->arg_size() == 3) {
2332 Intrinsic::objectsize, Tys);
2339 if (Name.starts_with(
"ptr.annotation.") &&
F->arg_size() == 4) {
2342 F->getParent(), Intrinsic::ptr_annotation,
2343 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2349 if (Name.consume_front(
"riscv.")) {
2352 .
Case(
"aes32dsi", Intrinsic::riscv_aes32dsi)
2353 .
Case(
"aes32dsmi", Intrinsic::riscv_aes32dsmi)
2354 .
Case(
"aes32esi", Intrinsic::riscv_aes32esi)
2355 .
Case(
"aes32esmi", Intrinsic::riscv_aes32esmi)
2358 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2371 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2372 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2381 .
StartsWith(
"sha256sig0", Intrinsic::riscv_sha256sig0)
2382 .
StartsWith(
"sha256sig1", Intrinsic::riscv_sha256sig1)
2383 .
StartsWith(
"sha256sum0", Intrinsic::riscv_sha256sum0)
2384 .
StartsWith(
"sha256sum1", Intrinsic::riscv_sha256sum1)
2389 if (
F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2398 if (Name ==
"clmul.i32" || Name ==
"clmul.i64") {
2400 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2409 if (Name ==
"stackprotectorcheck") {
2413 if (Name.starts_with(
"strip.invariant.group")) {
2418 F->getParent(), Intrinsic::launder_invariant_group,
2419 F->getReturnType());
2425 if (Name ==
"thread.pointer") {
2427 F->getParent(), Intrinsic::thread_pointer,
F->getReturnType());
2433 if (Name ==
"var.annotation" &&
F->arg_size() == 4) {
2436 F->getParent(), Intrinsic::var_annotation,
2437 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2440 if (Name.consume_front(
"vector.splice")) {
2441 if (Name.starts_with(
".left") || Name.starts_with(
".right"))
2451 if (Name.consume_front(
"wasm.")) {
2454 .
StartsWith(
"fma.", Intrinsic::wasm_relaxed_madd)
2455 .
StartsWith(
"fms.", Intrinsic::wasm_relaxed_nmadd)
2456 .
StartsWith(
"laneselect.", Intrinsic::wasm_relaxed_laneselect)
2461 F->getReturnType());
2465 if (Name.consume_front(
"dot.i8x16.i7x16.")) {
2467 .
Case(
"signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2469 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2488 if (ST && (!
ST->isLiteral() ||
ST->isPacked()) &&
2498 std::string
Name =
F->getName().str();
2501 Name,
F->getParent());
2512 if (Result != std::nullopt) {
2529 bool CanUpgradeDebugIntrinsicsToRecords) {
2549 GV->
getName() ==
"llvm.global_dtors")) ||
2563 unsigned N =
Init->getNumOperands();
2564 std::vector<Constant *> NewCtors(
N);
2565 for (
unsigned i = 0; i !=
N; ++i) {
2568 Ctor->getAggregateElement(1),
2582 unsigned NumElts = ResultTy->getNumElements() * 8;
2586 Op = Builder.CreateBitCast(
Op, VecTy,
"cast");
2596 for (
unsigned l = 0; l != NumElts; l += 16)
2597 for (
unsigned i = 0; i != 16; ++i) {
2598 unsigned Idx = NumElts + i - Shift;
2600 Idx -= NumElts - 16;
2601 Idxs[l + i] = Idx + l;
2604 Res = Builder.CreateShuffleVector(Res,
Op,
ArrayRef(Idxs, NumElts));
2608 return Builder.CreateBitCast(Res, ResultTy,
"cast");
2616 unsigned NumElts = ResultTy->getNumElements() * 8;
2620 Op = Builder.CreateBitCast(
Op, VecTy,
"cast");
2630 for (
unsigned l = 0; l != NumElts; l += 16)
2631 for (
unsigned i = 0; i != 16; ++i) {
2632 unsigned Idx = i + Shift;
2634 Idx += NumElts - 16;
2635 Idxs[l + i] = Idx + l;
2638 Res = Builder.CreateShuffleVector(
Op, Res,
ArrayRef(Idxs, NumElts));
2642 return Builder.CreateBitCast(Res, ResultTy,
"cast");
2650 Mask = Builder.CreateBitCast(Mask, MaskTy);
2656 for (
unsigned i = 0; i != NumElts; ++i)
2658 Mask = Builder.CreateShuffleVector(Mask, Mask,
ArrayRef(Indices, NumElts),
2669 if (
C->isAllOnesValue())
2674 return Builder.CreateSelect(Mask, Op0, Op1);
2681 if (
C->isAllOnesValue())
2685 Mask->getType()->getIntegerBitWidth());
2686 Mask = Builder.CreateBitCast(Mask, MaskTy);
2687 Mask = Builder.CreateExtractElement(Mask, (
uint64_t)0);
2688 return Builder.CreateSelect(Mask, Op0, Op1);
2701 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
2702 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
2707 ShiftVal &= (NumElts - 1);
2716 if (ShiftVal > 16) {
2724 for (
unsigned l = 0; l < NumElts; l += 16) {
2725 for (
unsigned i = 0; i != 16; ++i) {
2726 unsigned Idx = ShiftVal + i;
2727 if (!IsVALIGN && Idx >= 16)
2728 Idx += NumElts - 16;
2729 Indices[l + i] = Idx + l;
2734 Op1, Op0,
ArrayRef(Indices, NumElts),
"palignr");
2740 bool ZeroMask,
bool IndexForm) {
2743 unsigned EltWidth = Ty->getScalarSizeInBits();
2744 bool IsFloat = Ty->isFPOrFPVectorTy();
2746 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2747 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2748 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2749 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2750 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2751 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2752 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2753 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2754 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2755 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2756 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2757 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2758 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2759 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2760 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2761 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2762 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2763 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2764 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2765 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2766 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2767 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2768 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2769 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2770 else if (VecWidth == 128 && EltWidth == 16)
2771 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2772 else if (VecWidth == 256 && EltWidth == 16)
2773 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2774 else if (VecWidth == 512 && EltWidth == 16)
2775 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2776 else if (VecWidth == 128 && EltWidth == 8)
2777 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2778 else if (VecWidth == 256 && EltWidth == 8)
2779 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2780 else if (VecWidth == 512 && EltWidth == 8)
2781 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2792 Value *
V = Builder.CreateIntrinsic(IID, Args);
2804 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2815 bool IsRotateRight) {
2825 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
2826 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2829 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2830 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2875 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2880 bool IsShiftRight,
bool ZeroMask) {
2894 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
2895 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2898 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2899 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2914 const Align Alignment =
2916 ?
Align(
Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2921 if (
C->isAllOnesValue())
2922 return Builder.CreateAlignedStore(
Data, Ptr, Alignment);
2927 return Builder.CreateMaskedStore(
Data, Ptr, Alignment, Mask);
2933 const Align Alignment =
2942 if (
C->isAllOnesValue())
2943 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2948 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2954 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2955 {Op0, Builder.getInt1(
false)});
2970 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2971 LHS = Builder.CreateShl(
LHS, ShiftAmt);
2972 LHS = Builder.CreateAShr(
LHS, ShiftAmt);
2973 RHS = Builder.CreateShl(
RHS, ShiftAmt);
2974 RHS = Builder.CreateAShr(
RHS, ShiftAmt);
2977 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2978 LHS = Builder.CreateAnd(
LHS, Mask);
2979 RHS = Builder.CreateAnd(
RHS, Mask);
2996 if (!
C || !
C->isAllOnesValue())
2997 Vec = Builder.CreateAnd(Vec,
getX86MaskVec(Builder, Mask, NumElts));
3002 for (
unsigned i = 0; i != NumElts; ++i)
3004 for (
unsigned i = NumElts; i != 8; ++i)
3005 Indices[i] = NumElts + i % NumElts;
3006 Vec = Builder.CreateShuffleVector(Vec,
3010 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
3022 }
else if (
CC == 7) {
3058 Value* AndNode = Builder.CreateAnd(Mask,
APInt(8, 1));
3059 Value* Cmp = Builder.CreateIsNotNull(AndNode);
3061 Value* Extract2 = Builder.CreateExtractElement(Src, (
uint64_t)0);
3062 Value*
Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
3071 return Builder.CreateSExt(Mask, ReturnOp,
"vpmovm2");
3077 Name = Name.substr(12);
3082 if (Name.starts_with(
"max.p")) {
3083 if (VecWidth == 128 && EltWidth == 32)
3084 IID = Intrinsic::x86_sse_max_ps;
3085 else if (VecWidth == 128 && EltWidth == 64)
3086 IID = Intrinsic::x86_sse2_max_pd;
3087 else if (VecWidth == 256 && EltWidth == 32)
3088 IID = Intrinsic::x86_avx_max_ps_256;
3089 else if (VecWidth == 256 && EltWidth == 64)
3090 IID = Intrinsic::x86_avx_max_pd_256;
3093 }
else if (Name.starts_with(
"min.p")) {
3094 if (VecWidth == 128 && EltWidth == 32)
3095 IID = Intrinsic::x86_sse_min_ps;
3096 else if (VecWidth == 128 && EltWidth == 64)
3097 IID = Intrinsic::x86_sse2_min_pd;
3098 else if (VecWidth == 256 && EltWidth == 32)
3099 IID = Intrinsic::x86_avx_min_ps_256;
3100 else if (VecWidth == 256 && EltWidth == 64)
3101 IID = Intrinsic::x86_avx_min_pd_256;
3104 }
else if (Name.starts_with(
"pshuf.b.")) {
3105 if (VecWidth == 128)
3106 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3107 else if (VecWidth == 256)
3108 IID = Intrinsic::x86_avx2_pshuf_b;
3109 else if (VecWidth == 512)
3110 IID = Intrinsic::x86_avx512_pshuf_b_512;
3113 }
else if (Name.starts_with(
"pmul.hr.sw.")) {
3114 if (VecWidth == 128)
3115 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3116 else if (VecWidth == 256)
3117 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3118 else if (VecWidth == 512)
3119 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3122 }
else if (Name.starts_with(
"pmulh.w")) {
3123 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3124 "Unexpected intrinsic");
3127 }
else if (Name.starts_with(
"pmulhu.w")) {
3128 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3129 "Unexpected intrinsic");
3132 }
else if (Name.starts_with(
"pmaddw.d.")) {
3133 if (VecWidth == 128)
3134 IID = Intrinsic::x86_sse2_pmadd_wd;
3135 else if (VecWidth == 256)
3136 IID = Intrinsic::x86_avx2_pmadd_wd;
3137 else if (VecWidth == 512)
3138 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3141 }
else if (Name.starts_with(
"pmaddubs.w.")) {
3142 if (VecWidth == 128)
3143 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3144 else if (VecWidth == 256)
3145 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3146 else if (VecWidth == 512)
3147 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3150 }
else if (Name.starts_with(
"packsswb.")) {
3151 if (VecWidth == 128)
3152 IID = Intrinsic::x86_sse2_packsswb_128;
3153 else if (VecWidth == 256)
3154 IID = Intrinsic::x86_avx2_packsswb;
3155 else if (VecWidth == 512)
3156 IID = Intrinsic::x86_avx512_packsswb_512;
3159 }
else if (Name.starts_with(
"packssdw.")) {
3160 if (VecWidth == 128)
3161 IID = Intrinsic::x86_sse2_packssdw_128;
3162 else if (VecWidth == 256)
3163 IID = Intrinsic::x86_avx2_packssdw;
3164 else if (VecWidth == 512)
3165 IID = Intrinsic::x86_avx512_packssdw_512;
3168 }
else if (Name.starts_with(
"packuswb.")) {
3169 if (VecWidth == 128)
3170 IID = Intrinsic::x86_sse2_packuswb_128;
3171 else if (VecWidth == 256)
3172 IID = Intrinsic::x86_avx2_packuswb;
3173 else if (VecWidth == 512)
3174 IID = Intrinsic::x86_avx512_packuswb_512;
3177 }
else if (Name.starts_with(
"packusdw.")) {
3178 if (VecWidth == 128)
3179 IID = Intrinsic::x86_sse41_packusdw;
3180 else if (VecWidth == 256)
3181 IID = Intrinsic::x86_avx2_packusdw;
3182 else if (VecWidth == 512)
3183 IID = Intrinsic::x86_avx512_packusdw_512;
3186 }
else if (Name.starts_with(
"vpermilvar.")) {
3187 if (VecWidth == 128 && EltWidth == 32)
3188 IID = Intrinsic::x86_avx_vpermilvar_ps;
3189 else if (VecWidth == 128 && EltWidth == 64)
3190 IID = Intrinsic::x86_avx_vpermilvar_pd;
3191 else if (VecWidth == 256 && EltWidth == 32)
3192 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3193 else if (VecWidth == 256 && EltWidth == 64)
3194 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3195 else if (VecWidth == 512 && EltWidth == 32)
3196 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3197 else if (VecWidth == 512 && EltWidth == 64)
3198 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3201 }
else if (Name ==
"cvtpd2dq.256") {
3202 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3203 }
else if (Name ==
"cvtpd2ps.256") {
3204 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3205 }
else if (Name ==
"cvttpd2dq.256") {
3206 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3207 }
else if (Name ==
"cvttps2dq.128") {
3208 IID = Intrinsic::x86_sse2_cvttps2dq;
3209 }
else if (Name ==
"cvttps2dq.256") {
3210 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3211 }
else if (Name.starts_with(
"permvar.")) {
3213 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3214 IID = Intrinsic::x86_avx2_permps;
3215 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3216 IID = Intrinsic::x86_avx2_permd;
3217 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3218 IID = Intrinsic::x86_avx512_permvar_df_256;
3219 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3220 IID = Intrinsic::x86_avx512_permvar_di_256;
3221 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3222 IID = Intrinsic::x86_avx512_permvar_sf_512;
3223 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3224 IID = Intrinsic::x86_avx512_permvar_si_512;
3225 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3226 IID = Intrinsic::x86_avx512_permvar_df_512;
3227 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3228 IID = Intrinsic::x86_avx512_permvar_di_512;
3229 else if (VecWidth == 128 && EltWidth == 16)
3230 IID = Intrinsic::x86_avx512_permvar_hi_128;
3231 else if (VecWidth == 256 && EltWidth == 16)
3232 IID = Intrinsic::x86_avx512_permvar_hi_256;
3233 else if (VecWidth == 512 && EltWidth == 16)
3234 IID = Intrinsic::x86_avx512_permvar_hi_512;
3235 else if (VecWidth == 128 && EltWidth == 8)
3236 IID = Intrinsic::x86_avx512_permvar_qi_128;
3237 else if (VecWidth == 256 && EltWidth == 8)
3238 IID = Intrinsic::x86_avx512_permvar_qi_256;
3239 else if (VecWidth == 512 && EltWidth == 8)
3240 IID = Intrinsic::x86_avx512_permvar_qi_512;
3243 }
else if (Name.starts_with(
"dbpsadbw.")) {
3244 if (VecWidth == 128)
3245 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3246 else if (VecWidth == 256)
3247 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3248 else if (VecWidth == 512)
3249 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3252 }
else if (Name.starts_with(
"pmultishift.qb.")) {
3253 if (VecWidth == 128)
3254 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3255 else if (VecWidth == 256)
3256 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3257 else if (VecWidth == 512)
3258 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3261 }
else if (Name.starts_with(
"conflict.")) {
3262 if (Name[9] ==
'd' && VecWidth == 128)
3263 IID = Intrinsic::x86_avx512_conflict_d_128;
3264 else if (Name[9] ==
'd' && VecWidth == 256)
3265 IID = Intrinsic::x86_avx512_conflict_d_256;
3266 else if (Name[9] ==
'd' && VecWidth == 512)
3267 IID = Intrinsic::x86_avx512_conflict_d_512;
3268 else if (Name[9] ==
'q' && VecWidth == 128)
3269 IID = Intrinsic::x86_avx512_conflict_q_128;
3270 else if (Name[9] ==
'q' && VecWidth == 256)
3271 IID = Intrinsic::x86_avx512_conflict_q_256;
3272 else if (Name[9] ==
'q' && VecWidth == 512)
3273 IID = Intrinsic::x86_avx512_conflict_q_512;
3276 }
else if (Name.starts_with(
"pavg.")) {
3277 if (Name[5] ==
'b' && VecWidth == 128)
3278 IID = Intrinsic::x86_sse2_pavg_b;
3279 else if (Name[5] ==
'b' && VecWidth == 256)
3280 IID = Intrinsic::x86_avx2_pavg_b;
3281 else if (Name[5] ==
'b' && VecWidth == 512)
3282 IID = Intrinsic::x86_avx512_pavg_b_512;
3283 else if (Name[5] ==
'w' && VecWidth == 128)
3284 IID = Intrinsic::x86_sse2_pavg_w;
3285 else if (Name[5] ==
'w' && VecWidth == 256)
3286 IID = Intrinsic::x86_avx2_pavg_w;
3287 else if (Name[5] ==
'w' && VecWidth == 512)
3288 IID = Intrinsic::x86_avx512_pavg_w_512;
3297 Rep = Builder.CreateIntrinsic(IID, Args);
3308 if (AsmStr->find(
"mov\tfp") == 0 &&
3309 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
3310 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
3311 AsmStr->replace(Pos, 1,
";");
3319 assert(Result &&
"unsupported nvvm.{add,mul,fma}.* intrinsic");
3327 if (IsLegacyBF16Storage)
3328 Ty = Name.ends_with(
"bf16x2")
3330 :
static_cast<Type *
>(Builder.getBFloatTy());
3334 Args.push_back(IsLegacyBF16Storage ? Builder.CreateBitCast(Arg, Ty) : Arg);
3335 Args.push_back(Builder.getInt32(
static_cast<int>(
RoundingMode)));
3337 Value *Rep = Builder.CreateIntrinsic(Ty, IID, Args);
3338 return IsLegacyBF16Storage ? Builder.CreateBitCast(Rep, CI->
getType()) : Rep;
3343 Value *Rep =
nullptr;
3345 if (Name ==
"abs.i" || Name ==
"abs.ll") {
3347 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->
getType()},
3348 {Arg, Builder.getTrue()},
3350 }
else if (Name ==
"abs.bf16" || Name ==
"abs.bf16x2") {
3351 Type *Ty = (Name ==
"abs.bf16")
3355 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3356 Rep = Builder.CreateBitCast(Abs, CI->
getType());
3357 }
else if (Name ==
"fabs.f" || Name ==
"fabs.ftz.f" || Name ==
"fabs.d") {
3358 Intrinsic::ID IID = (Name ==
"fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3359 : Intrinsic::nvvm_fabs;
3360 Rep = Builder.CreateUnaryIntrinsic(IID, CI->
getArgOperand(0));
3361 }
else if (Name.consume_front(
"add.")) {
3364 }
else if (Name.consume_front(
"mul.")) {
3367 }
else if (Name.consume_front(
"ex2.approx.")) {
3369 Intrinsic::ID IID = Name.starts_with(
"ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3370 : Intrinsic::nvvm_ex2_approx;
3371 Rep = Builder.CreateUnaryIntrinsic(IID, CI->
getArgOperand(0));
3372 }
else if (Name.consume_front(
"fma.")) {
3375 }
else if (Name.starts_with(
"atomic.load.add.f32.p") ||
3376 Name.starts_with(
"atomic.load.add.f64.p")) {
3379 Rep = Builder.CreateAtomicRMW(
3383 LLVMContext::MD_atomic_ignore_denormal_mode,
3388 }
else if (Name.starts_with(
"atomic.load.inc.32.p") ||
3389 Name.starts_with(
"atomic.load.dec.32.p")) {
3394 Rep = Builder.CreateAtomicRMW(
3398 }
else if (Name.starts_with(
"atomic.") && Name.contains(
".gen.")) {
3404 Op.contains(
".cta.") ?
"block" :
"");
3405 if (
Op.starts_with(
"cas.")) {
3407 Value *Pair = Builder.CreateAtomicCmpXchg(
3410 Rep = Builder.CreateExtractValue(Pair, 0);
3428 "unexpected nvvm scoped atomic intrinsic");
3429 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val,
MaybeAlign(),
3433 LLVMContext::MD_atomic_ignore_denormal_mode,
3437 }
else if (Name ==
"clz.ll") {
3440 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->
getType()},
3441 {Arg, Builder.getFalse()},
3443 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(),
"ctlz.trunc");
3444 }
else if (Name ==
"popc.ll") {
3448 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->
getType()},
3449 Arg,
nullptr,
"ctpop");
3450 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(),
"ctpop.trunc");
3451 }
else if (Name ==
"h2f") {
3453 Builder.CreateBitCast(CI->
getArgOperand(0), Builder.getHalfTy());
3454 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3455 }
else if (Name.consume_front(
"bitcast.") &&
3456 (Name ==
"f2i" || Name ==
"i2f" || Name ==
"ll2d" ||
3459 }
else if (Name ==
"rotate.b32") {
3462 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3463 {Arg, Arg, ShiftAmt});
3464 }
else if (Name ==
"rotate.b64") {
3468 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3469 {Arg, Arg, ZExtShiftAmt});
3470 }
else if (Name ==
"rotate.right.b64") {
3474 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3475 {Arg, Arg, ZExtShiftAmt});
3476 }
else if (Name ==
"swap.lo.hi.b64") {
3479 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3480 {Arg, Arg, Builder.getInt64(32)});
3481 }
else if ((Name.consume_front(
"ptr.gen.to.") &&
3484 Name.starts_with(
".to.gen"))) {
3486 }
else if (Name.consume_front(
"ldg.global")) {
3490 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3493 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3495 }
else if (Name ==
"tanh.approx.f32") {
3499 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->
getArgOperand(0),
3501 }
else if (Name ==
"barrier0" || Name ==
"barrier.n" || Name ==
"bar.sync") {
3503 Name.ends_with(
'0') ? Builder.getInt32(0) : CI->
getArgOperand(0);
3504 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3506 }
else if (Name ==
"barrier") {
3507 Rep = Builder.CreateIntrinsic(
3508 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3510 }
else if (Name ==
"barrier.sync") {
3511 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3513 }
else if (Name ==
"barrier.sync.cnt") {
3514 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3516 }
else if (Name ==
"barrier0.popc" || Name ==
"barrier0.and" ||
3517 Name ==
"barrier0.or") {
3519 C = Builder.CreateICmpNE(
C, Builder.getInt32(0));
3523 .
Case(
"barrier0.popc",
3524 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3525 .
Case(
"barrier0.and",
3526 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3527 .
Case(
"barrier0.or",
3528 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3529 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0),
C});
3530 Rep = Builder.CreateZExt(Bar, CI->
getType());
3544 ? Builder.CreateBitCast(Arg, NewType)
3547 Rep = Builder.CreateCall(NewFn, Args);
3548 if (
F->getReturnType()->isIntegerTy())
3549 Rep = Builder.CreateBitCast(Rep,
F->getReturnType());
3559 Value *Rep =
nullptr;
3561 if (Name.starts_with(
"sse4a.movnt.")) {
3573 Builder.CreateExtractElement(Arg1, (
uint64_t)0,
"extractelement");
3576 SI->setMetadata(LLVMContext::MD_nontemporal,
Node);
3577 }
else if (Name.starts_with(
"avx.movnt.") ||
3578 Name.starts_with(
"avx512.storent.")) {
3590 SI->setMetadata(LLVMContext::MD_nontemporal,
Node);
3591 }
else if (Name ==
"sse2.storel.dq") {
3596 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy,
"cast");
3597 Value *Elt = Builder.CreateExtractElement(BC0, (
uint64_t)0);
3598 Builder.CreateAlignedStore(Elt, Arg0,
Align(1));
3599 }
else if (Name.starts_with(
"sse.storeu.") ||
3600 Name.starts_with(
"sse2.storeu.") ||
3601 Name.starts_with(
"avx.storeu.")) {
3604 Builder.CreateAlignedStore(Arg1, Arg0,
Align(1));
3605 }
else if (Name ==
"avx512.mask.store.ss") {
3609 }
else if (Name.starts_with(
"avx512.mask.store")) {
3611 bool Aligned = Name[17] !=
'u';
3614 }
else if (Name.starts_with(
"sse2.pcmp") || Name.starts_with(
"avx2.pcmp")) {
3617 bool CmpEq = Name[9] ==
'e';
3620 Rep = Builder.CreateSExt(Rep, CI->
getType(),
"");
3621 }
else if (Name.starts_with(
"avx512.broadcastm")) {
3628 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3629 }
else if (Name ==
"sse.sqrt.ss" || Name ==
"sse2.sqrt.sd") {
3631 Value *Elt0 = Builder.CreateExtractElement(Vec, (
uint64_t)0);
3632 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->
getType(), Elt0);
3633 Rep = Builder.CreateInsertElement(Vec, Elt0, (
uint64_t)0);
3634 }
else if (Name.starts_with(
"avx.sqrt.p") ||
3635 Name.starts_with(
"sse2.sqrt.p") ||
3636 Name.starts_with(
"sse.sqrt.p")) {
3637 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->
getType(),
3638 {CI->getArgOperand(0)});
3639 }
else if (Name.starts_with(
"avx512.mask.sqrt.p")) {
3643 Intrinsic::ID IID = Name[18] ==
's' ? Intrinsic::x86_avx512_sqrt_ps_512
3644 : Intrinsic::x86_avx512_sqrt_pd_512;
3647 Rep = Builder.CreateIntrinsic(IID, Args);
3649 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->
getType(),
3650 {CI->getArgOperand(0)});
3654 }
else if (Name.starts_with(
"avx512.ptestm") ||
3655 Name.starts_with(
"avx512.ptestnm")) {
3659 Rep = Builder.CreateAnd(Op0, Op1);
3665 Rep = Builder.CreateICmp(
Pred, Rep,
Zero);
3667 }
else if (Name.starts_with(
"avx512.mask.pbroadcast")) {
3670 Rep = Builder.CreateVectorSplat(NumElts, CI->
getArgOperand(0));
3673 }
else if (Name.starts_with(
"avx512.kunpck")) {
3678 for (
unsigned i = 0; i != NumElts; ++i)
3687 Rep = Builder.CreateShuffleVector(
RHS,
LHS,
ArrayRef(Indices, NumElts));
3688 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3689 }
else if (Name ==
"avx512.kand.w") {
3692 Rep = Builder.CreateAnd(
LHS,
RHS);
3693 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3694 }
else if (Name ==
"avx512.kandn.w") {
3697 LHS = Builder.CreateNot(
LHS);
3698 Rep = Builder.CreateAnd(
LHS,
RHS);
3699 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3700 }
else if (Name ==
"avx512.kor.w") {
3703 Rep = Builder.CreateOr(
LHS,
RHS);
3704 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3705 }
else if (Name ==
"avx512.kxor.w") {
3708 Rep = Builder.CreateXor(
LHS,
RHS);
3709 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3710 }
else if (Name ==
"avx512.kxnor.w") {
3713 LHS = Builder.CreateNot(
LHS);
3714 Rep = Builder.CreateXor(
LHS,
RHS);
3715 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3716 }
else if (Name ==
"avx512.knot.w") {
3718 Rep = Builder.CreateNot(Rep);
3719 Rep = Builder.CreateBitCast(Rep, CI->
getType());
3720 }
else if (Name ==
"avx512.kortestz.w" || Name ==
"avx512.kortestc.w") {
3723 Rep = Builder.CreateOr(
LHS,
RHS);
3724 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3726 if (Name[14] ==
'c')
3730 Rep = Builder.CreateICmpEQ(Rep,
C);
3731 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3732 }
else if (Name ==
"sse.add.ss" || Name ==
"sse2.add.sd" ||
3733 Name ==
"sse.sub.ss" || Name ==
"sse2.sub.sd" ||
3734 Name ==
"sse.mul.ss" || Name ==
"sse2.mul.sd" ||
3735 Name ==
"sse.div.ss" || Name ==
"sse2.div.sd") {
3738 ConstantInt::get(I32Ty, 0));
3740 ConstantInt::get(I32Ty, 0));
3742 if (Name.contains(
".add."))
3743 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3744 else if (Name.contains(
".sub."))
3745 EltOp = Builder.CreateFSub(Elt0, Elt1);
3746 else if (Name.contains(
".mul."))
3747 EltOp = Builder.CreateFMul(Elt0, Elt1);
3749 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3750 Rep = Builder.CreateInsertElement(CI->
getArgOperand(0), EltOp,
3751 ConstantInt::get(I32Ty, 0));
3752 }
else if (Name.starts_with(
"avx512.mask.pcmp")) {
3754 bool CmpEq = Name[16] ==
'e';
3756 }
else if (Name.starts_with(
"avx512.mask.vpshufbitqmb.")) {
3758 unsigned VecWidth =
OpTy->getPrimitiveSizeInBits();
3765 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3768 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3771 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3778 }
else if (Name.starts_with(
"avx512.mask.fpclass.p")) {
3780 unsigned VecWidth =
OpTy->getPrimitiveSizeInBits();
3781 unsigned EltWidth =
OpTy->getScalarSizeInBits();
3783 if (VecWidth == 128 && EltWidth == 32)
3784 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3785 else if (VecWidth == 256 && EltWidth == 32)
3786 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3787 else if (VecWidth == 512 && EltWidth == 32)
3788 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3789 else if (VecWidth == 128 && EltWidth == 64)
3790 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3791 else if (VecWidth == 256 && EltWidth == 64)
3792 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3793 else if (VecWidth == 512 && EltWidth == 64)
3794 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3801 }
else if (Name.starts_with(
"avx512.cmp.p")) {
3804 unsigned VecWidth =
OpTy->getPrimitiveSizeInBits();
3805 unsigned EltWidth =
OpTy->getScalarSizeInBits();
3807 if (VecWidth == 128 && EltWidth == 32)
3808 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3809 else if (VecWidth == 256 && EltWidth == 32)
3810 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3811 else if (VecWidth == 512 && EltWidth == 32)
3812 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3813 else if (VecWidth == 128 && EltWidth == 64)
3814 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3815 else if (VecWidth == 256 && EltWidth == 64)
3816 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3817 else if (VecWidth == 512 && EltWidth == 64)
3818 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3823 if (VecWidth == 512)
3825 Args.push_back(Mask);
3827 Rep = Builder.CreateIntrinsic(IID, Args);
3828 }
else if (Name.starts_with(
"avx512.mask.cmp.")) {
3832 }
else if (Name.starts_with(
"avx512.mask.ucmp.")) {
3835 }
else if (Name.starts_with(
"avx512.cvtb2mask.") ||
3836 Name.starts_with(
"avx512.cvtw2mask.") ||
3837 Name.starts_with(
"avx512.cvtd2mask.") ||
3838 Name.starts_with(
"avx512.cvtq2mask.")) {
3843 }
else if (Name ==
"ssse3.pabs.b.128" || Name ==
"ssse3.pabs.w.128" ||
3844 Name ==
"ssse3.pabs.d.128" || Name.starts_with(
"avx2.pabs") ||
3845 Name.starts_with(
"avx512.mask.pabs")) {
3847 }
else if (Name ==
"sse41.pmaxsb" || Name ==
"sse2.pmaxs.w" ||
3848 Name ==
"sse41.pmaxsd" || Name.starts_with(
"avx2.pmaxs") ||
3849 Name.starts_with(
"avx512.mask.pmaxs")) {
3851 }
else if (Name ==
"sse2.pmaxu.b" || Name ==
"sse41.pmaxuw" ||
3852 Name ==
"sse41.pmaxud" || Name.starts_with(
"avx2.pmaxu") ||
3853 Name.starts_with(
"avx512.mask.pmaxu")) {
3855 }
else if (Name ==
"sse41.pminsb" || Name ==
"sse2.pmins.w" ||
3856 Name ==
"sse41.pminsd" || Name.starts_with(
"avx2.pmins") ||
3857 Name.starts_with(
"avx512.mask.pmins")) {
3859 }
else if (Name ==
"sse2.pminu.b" || Name ==
"sse41.pminuw" ||
3860 Name ==
"sse41.pminud" || Name.starts_with(
"avx2.pminu") ||
3861 Name.starts_with(
"avx512.mask.pminu")) {
3863 }
else if (Name ==
"sse2.pmulh.w" || Name.starts_with(
"avx2.pmulh.w") ||
3864 Name.starts_with(
"avx512.pmulh.w")) {
3866 }
else if (Name ==
"sse2.pmulhu.w" || Name.starts_with(
"avx2.pmulhu.w") ||
3867 Name.starts_with(
"avx512.pmulhu.w")) {
3869 }
else if (Name ==
"sse2.pmulu.dq" || Name ==
"avx2.pmulu.dq" ||
3870 Name ==
"avx512.pmulu.dq.512" ||
3871 Name.starts_with(
"avx512.mask.pmulu.dq.")) {
3873 }
else if (Name ==
"sse41.pmuldq" || Name ==
"avx2.pmul.dq" ||
3874 Name ==
"avx512.pmul.dq.512" ||
3875 Name.starts_with(
"avx512.mask.pmul.dq.")) {
3877 }
else if (Name ==
"sse.cvtsi2ss" || Name ==
"sse2.cvtsi2sd" ||
3878 Name ==
"sse.cvtsi642ss" || Name ==
"sse2.cvtsi642sd") {
3883 }
else if (Name ==
"avx512.cvtusi2sd") {
3888 }
else if (Name ==
"sse2.cvtss2sd") {
3890 Rep = Builder.CreateFPExt(
3893 }
else if (Name ==
"sse2.cvtdq2pd" || Name ==
"sse2.cvtdq2ps" ||
3894 Name ==
"avx.cvtdq2.pd.256" || Name ==
"avx.cvtdq2.ps.256" ||
3895 Name.starts_with(
"avx512.mask.cvtdq2pd.") ||
3896 Name.starts_with(
"avx512.mask.cvtudq2pd.") ||
3897 Name.starts_with(
"avx512.mask.cvtdq2ps.") ||
3898 Name.starts_with(
"avx512.mask.cvtudq2ps.") ||
3899 Name.starts_with(
"avx512.mask.cvtqq2pd.") ||
3900 Name.starts_with(
"avx512.mask.cvtuqq2pd.") ||
3901 Name ==
"avx512.mask.cvtqq2ps.256" ||
3902 Name ==
"avx512.mask.cvtqq2ps.512" ||
3903 Name ==
"avx512.mask.cvtuqq2ps.256" ||
3904 Name ==
"avx512.mask.cvtuqq2ps.512" || Name ==
"sse2.cvtps2pd" ||
3905 Name ==
"avx.cvt.ps2.pd.256" ||
3906 Name ==
"avx512.mask.cvtps2pd.128" ||
3907 Name ==
"avx512.mask.cvtps2pd.256") {
3912 unsigned NumDstElts = DstTy->getNumElements();
3913 if (NumDstElts < SrcTy->getNumElements()) {
3914 assert(NumDstElts == 2 &&
"Unexpected vector size");
3915 Rep = Builder.CreateShuffleVector(Rep, Rep,
ArrayRef<int>{0, 1});
3918 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3919 bool IsUnsigned = Name.contains(
"cvtu");
3921 Rep = Builder.CreateFPExt(Rep, DstTy,
"cvtps2pd");
3925 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3926 : Intrinsic::x86_avx512_sitofp_round;
3927 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3930 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy,
"cvt")
3931 : Builder.CreateSIToFP(Rep, DstTy,
"cvt");
3937 }
else if (Name.starts_with(
"avx512.mask.vcvtph2ps.") ||
3938 Name.starts_with(
"vcvtph2ps.")) {
3942 unsigned NumDstElts = DstTy->getNumElements();
3943 if (NumDstElts != SrcTy->getNumElements()) {
3944 assert(NumDstElts == 4 &&
"Unexpected vector size");
3945 Rep = Builder.CreateShuffleVector(Rep, Rep,
ArrayRef<int>{0, 1, 2, 3});
3947 Rep = Builder.CreateBitCast(
3949 Rep = Builder.CreateFPExt(Rep, DstTy,
"cvtph2ps");
3953 }
else if (Name.starts_with(
"avx512.mask.load")) {
3955 bool Aligned = Name[16] !=
'u';
3958 }
else if (Name.starts_with(
"avx512.mask.expand.load.")) {
3962 ResultTy->getNumElements());
3963 Rep = Builder.CreateIntrinsic(
3964 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3966 }
else if (Name.starts_with(
"avx512.mask.compress.store.")) {
3972 Rep = Builder.CreateIntrinsic(
3973 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3975 }
else if (Name.starts_with(
"avx512.mask.compress.") ||
3976 Name.starts_with(
"avx512.mask.expand.")) {
3980 ResultTy->getNumElements());
3982 bool IsCompress = Name[12] ==
'c';
3983 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3984 : Intrinsic::x86_avx512_mask_expand;
3985 Rep = Builder.CreateIntrinsic(
3987 }
else if (Name.starts_with(
"xop.vpcom")) {
3989 if (Name.ends_with(
"ub") || Name.ends_with(
"uw") || Name.ends_with(
"ud") ||
3990 Name.ends_with(
"uq"))
3992 else if (Name.ends_with(
"b") || Name.ends_with(
"w") ||
3993 Name.ends_with(
"d") || Name.ends_with(
"q"))
4002 Name = Name.substr(9);
4003 if (Name.starts_with(
"lt"))
4005 else if (Name.starts_with(
"le"))
4007 else if (Name.starts_with(
"gt"))
4009 else if (Name.starts_with(
"ge"))
4011 else if (Name.starts_with(
"eq"))
4013 else if (Name.starts_with(
"ne"))
4015 else if (Name.starts_with(
"false"))
4017 else if (Name.starts_with(
"true"))
4024 }
else if (Name.starts_with(
"xop.vpcmov")) {
4026 Value *NotSel = Builder.CreateNot(Sel);
4029 Rep = Builder.CreateOr(Sel0, Sel1);
4030 }
else if (Name.starts_with(
"xop.vprot") || Name.starts_with(
"avx512.prol") ||
4031 Name.starts_with(
"avx512.mask.prol")) {
4033 }
else if (Name.starts_with(
"avx512.pror") ||
4034 Name.starts_with(
"avx512.mask.pror")) {
4036 }
else if (Name.starts_with(
"avx512.vpshld.") ||
4037 Name.starts_with(
"avx512.mask.vpshld") ||
4038 Name.starts_with(
"avx512.maskz.vpshld")) {
4039 bool ZeroMask = Name[11] ==
'z';
4041 }
else if (Name.starts_with(
"avx512.vpshrd.") ||
4042 Name.starts_with(
"avx512.mask.vpshrd") ||
4043 Name.starts_with(
"avx512.maskz.vpshrd")) {
4044 bool ZeroMask = Name[11] ==
'z';
4046 }
else if (Name ==
"sse42.crc32.64.8") {
4049 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
4051 Rep = Builder.CreateZExt(Rep, CI->
getType(),
"");
4052 }
else if (Name.starts_with(
"avx.vbroadcast.s") ||
4053 Name.starts_with(
"avx512.vbroadcast.s")) {
4056 Type *EltTy = VecTy->getElementType();
4057 unsigned EltNum = VecTy->getNumElements();
4061 for (
unsigned I = 0;
I < EltNum; ++
I)
4062 Rep = Builder.CreateInsertElement(Rep,
Load, ConstantInt::get(I32Ty,
I));
4063 }
else if (Name.starts_with(
"sse41.pmovsx") ||
4064 Name.starts_with(
"sse41.pmovzx") ||
4065 Name.starts_with(
"avx2.pmovsx") ||
4066 Name.starts_with(
"avx2.pmovzx") ||
4067 Name.starts_with(
"avx512.mask.pmovsx") ||
4068 Name.starts_with(
"avx512.mask.pmovzx")) {
4070 unsigned NumDstElts = DstTy->getNumElements();
4074 for (
unsigned i = 0; i != NumDstElts; ++i)
4079 bool DoSext = Name.contains(
"pmovsx");
4081 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
4086 }
else if (Name ==
"avx512.mask.pmov.qd.256" ||
4087 Name ==
"avx512.mask.pmov.qd.512" ||
4088 Name ==
"avx512.mask.pmov.wb.256" ||
4089 Name ==
"avx512.mask.pmov.wb.512") {
4094 }
else if (Name.starts_with(
"avx.vbroadcastf128") ||
4095 Name ==
"avx2.vbroadcasti128") {
4101 if (NumSrcElts == 2)
4104 Rep = Builder.CreateShuffleVector(
Load,
4106 }
else if (Name.starts_with(
"avx512.mask.shuf.i") ||
4107 Name.starts_with(
"avx512.mask.shuf.f")) {
4112 unsigned ControlBitsMask = NumLanes - 1;
4113 unsigned NumControlBits = NumLanes / 2;
4116 for (
unsigned l = 0; l != NumLanes; ++l) {
4117 unsigned LaneMask = (
Imm >> (l * NumControlBits)) & ControlBitsMask;
4119 if (l >= NumLanes / 2)
4120 LaneMask += NumLanes;
4121 for (
unsigned i = 0; i != NumElementsInLane; ++i)
4122 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4128 }
else if (Name.starts_with(
"avx512.mask.broadcastf") ||
4129 Name.starts_with(
"avx512.mask.broadcasti")) {
4132 unsigned NumDstElts =
4136 for (
unsigned i = 0; i != NumDstElts; ++i)
4137 ShuffleMask[i] = i % NumSrcElts;
4143 }
else if (Name.starts_with(
"avx2.pbroadcast") ||
4144 Name.starts_with(
"avx2.vbroadcast") ||
4145 Name.starts_with(
"avx512.pbroadcast") ||
4146 Name.starts_with(
"avx512.mask.broadcast.s")) {
4153 Rep = Builder.CreateShuffleVector(
Op, M);
4158 }
else if (Name.starts_with(
"sse2.padds.") ||
4159 Name.starts_with(
"avx2.padds.") ||
4160 Name.starts_with(
"avx512.padds.") ||
4161 Name.starts_with(
"avx512.mask.padds.")) {
4163 }
else if (Name.starts_with(
"sse2.psubs.") ||
4164 Name.starts_with(
"avx2.psubs.") ||
4165 Name.starts_with(
"avx512.psubs.") ||
4166 Name.starts_with(
"avx512.mask.psubs.")) {
4168 }
else if (Name.starts_with(
"sse2.paddus.") ||
4169 Name.starts_with(
"avx2.paddus.") ||
4170 Name.starts_with(
"avx512.mask.paddus.")) {
4172 }
else if (Name.starts_with(
"sse2.psubus.") ||
4173 Name.starts_with(
"avx2.psubus.") ||
4174 Name.starts_with(
"avx512.mask.psubus.")) {
4176 }
else if (Name.starts_with(
"avx512.mask.palignr.")) {
4181 }
else if (Name.starts_with(
"avx512.mask.valign.")) {
4185 }
else if (Name ==
"sse2.psll.dq" || Name ==
"avx2.psll.dq") {
4190 }
else if (Name ==
"sse2.psrl.dq" || Name ==
"avx2.psrl.dq") {
4195 }
else if (Name ==
"sse2.psll.dq.bs" || Name ==
"avx2.psll.dq.bs" ||
4196 Name ==
"avx512.psll.dq.512") {
4200 }
else if (Name ==
"sse2.psrl.dq.bs" || Name ==
"avx2.psrl.dq.bs" ||
4201 Name ==
"avx512.psrl.dq.512") {
4205 }
else if (Name ==
"sse41.pblendw" || Name.starts_with(
"sse41.blendp") ||
4206 Name.starts_with(
"avx.blend.p") || Name ==
"avx2.pblendw" ||
4207 Name.starts_with(
"avx2.pblendd.")) {
4212 unsigned NumElts = VecTy->getNumElements();
4215 for (
unsigned i = 0; i != NumElts; ++i)
4216 Idxs[i] = ((
Imm >> (i % 8)) & 1) ? i + NumElts : i;
4218 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4219 }
else if (Name.starts_with(
"avx.vinsertf128.") ||
4220 Name ==
"avx2.vinserti128" ||
4221 Name.starts_with(
"avx512.mask.insert")) {
4225 unsigned DstNumElts =
4227 unsigned SrcNumElts =
4229 unsigned Scale = DstNumElts / SrcNumElts;
4236 for (
unsigned i = 0; i != SrcNumElts; ++i)
4238 for (
unsigned i = SrcNumElts; i != DstNumElts; ++i)
4239 Idxs[i] = SrcNumElts;
4240 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4254 for (
unsigned i = 0; i != DstNumElts; ++i)
4257 for (
unsigned i = 0; i != SrcNumElts; ++i)
4258 Idxs[i +
Imm * SrcNumElts] = i + DstNumElts;
4259 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4265 }
else if (Name.starts_with(
"avx.vextractf128.") ||
4266 Name ==
"avx2.vextracti128" ||
4267 Name.starts_with(
"avx512.mask.vextract")) {
4270 unsigned DstNumElts =
4272 unsigned SrcNumElts =
4274 unsigned Scale = SrcNumElts / DstNumElts;
4281 for (
unsigned i = 0; i != DstNumElts; ++i) {
4282 Idxs[i] = i + (
Imm * DstNumElts);
4284 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4290 }
else if (Name.starts_with(
"avx512.mask.perm.df.") ||
4291 Name.starts_with(
"avx512.mask.perm.di.")) {
4295 unsigned NumElts = VecTy->getNumElements();
4298 for (
unsigned i = 0; i != NumElts; ++i)
4299 Idxs[i] = (i & ~0x3) + ((
Imm >> (2 * (i & 0x3))) & 3);
4301 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4306 }
else if (Name.starts_with(
"avx.vperm2f128.") || Name ==
"avx2.vperm2i128") {
4318 unsigned HalfSize = NumElts / 2;
4330 unsigned StartIndex = (
Imm & 0x01) ? HalfSize : 0;
4331 for (
unsigned i = 0; i < HalfSize; ++i)
4332 ShuffleMask[i] = StartIndex + i;
4335 StartIndex = (
Imm & 0x10) ? HalfSize : 0;
4336 for (
unsigned i = 0; i < HalfSize; ++i)
4337 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4339 Rep = Builder.CreateShuffleVector(V0,
V1, ShuffleMask);
4341 }
else if (Name.starts_with(
"avx.vpermil.") || Name ==
"sse2.pshuf.d" ||
4342 Name.starts_with(
"avx512.mask.vpermil.p") ||
4343 Name.starts_with(
"avx512.mask.pshuf.d.")) {
4347 unsigned NumElts = VecTy->getNumElements();
4349 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4350 unsigned IdxMask = ((1 << IdxSize) - 1);
4356 for (
unsigned i = 0; i != NumElts; ++i)
4357 Idxs[i] = ((
Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4359 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4364 }
else if (Name ==
"sse2.pshufl.w" ||
4365 Name.starts_with(
"avx512.mask.pshufl.w.")) {
4370 if (Name ==
"sse2.pshufl.w" && NumElts % 8 != 0)
4374 for (
unsigned l = 0; l != NumElts; l += 8) {
4375 for (
unsigned i = 0; i != 4; ++i)
4376 Idxs[i + l] = ((
Imm >> (2 * i)) & 0x3) + l;
4377 for (
unsigned i = 4; i != 8; ++i)
4378 Idxs[i + l] = i + l;
4381 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4386 }
else if (Name ==
"sse2.pshufh.w" ||
4387 Name.starts_with(
"avx512.mask.pshufh.w.")) {
4392 if (Name ==
"sse2.pshufh.w" && NumElts % 8 != 0)
4396 for (
unsigned l = 0; l != NumElts; l += 8) {
4397 for (
unsigned i = 0; i != 4; ++i)
4398 Idxs[i + l] = i + l;
4399 for (
unsigned i = 0; i != 4; ++i)
4400 Idxs[i + l + 4] = ((
Imm >> (2 * i)) & 0x3) + 4 + l;
4403 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4408 }
else if (Name.starts_with(
"avx512.mask.shuf.p")) {
4415 unsigned HalfLaneElts = NumLaneElts / 2;
4418 for (
unsigned i = 0; i != NumElts; ++i) {
4420 Idxs[i] = i - (i % NumLaneElts);
4422 if ((i % NumLaneElts) >= HalfLaneElts)
4426 Idxs[i] += (
Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4429 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4433 }
else if (Name.starts_with(
"avx512.mask.movddup") ||
4434 Name.starts_with(
"avx512.mask.movshdup") ||
4435 Name.starts_with(
"avx512.mask.movsldup")) {
4441 if (Name.starts_with(
"avx512.mask.movshdup."))
4445 for (
unsigned l = 0; l != NumElts; l += NumLaneElts)
4446 for (
unsigned i = 0; i != NumLaneElts; i += 2) {
4447 Idxs[i + l + 0] = i + l +
Offset;
4448 Idxs[i + l + 1] = i + l +
Offset;
4451 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4455 }
else if (Name.starts_with(
"avx512.mask.punpckl") ||
4456 Name.starts_with(
"avx512.mask.unpckl.")) {
4463 for (
int l = 0; l != NumElts; l += NumLaneElts)
4464 for (
int i = 0; i != NumLaneElts; ++i)
4465 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4467 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4471 }
else if (Name.starts_with(
"avx512.mask.punpckh") ||
4472 Name.starts_with(
"avx512.mask.unpckh.")) {
4479 for (
int l = 0; l != NumElts; l += NumLaneElts)
4480 for (
int i = 0; i != NumLaneElts; ++i)
4481 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4483 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4487 }
else if (Name.starts_with(
"avx512.mask.and.") ||
4488 Name.starts_with(
"avx512.mask.pand.")) {
4491 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->
getArgOperand(0), ITy),
4493 Rep = Builder.CreateBitCast(Rep, FTy);
4496 }
else if (Name.starts_with(
"avx512.mask.andn.") ||
4497 Name.starts_with(
"avx512.mask.pandn.")) {
4500 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->
getArgOperand(0), ITy));
4501 Rep = Builder.CreateAnd(Rep,
4503 Rep = Builder.CreateBitCast(Rep, FTy);
4506 }
else if (Name.starts_with(
"avx512.mask.or.") ||
4507 Name.starts_with(
"avx512.mask.por.")) {
4510 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->
getArgOperand(0), ITy),
4512 Rep = Builder.CreateBitCast(Rep, FTy);
4515 }
else if (Name.starts_with(
"avx512.mask.xor.") ||
4516 Name.starts_with(
"avx512.mask.pxor.")) {
4519 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->
getArgOperand(0), ITy),
4521 Rep = Builder.CreateBitCast(Rep, FTy);
4524 }
else if (Name.starts_with(
"avx512.mask.padd.")) {
4528 }
else if (Name.starts_with(
"avx512.mask.psub.")) {
4532 }
else if (Name.starts_with(
"avx512.mask.pmull.")) {
4536 }
else if (Name.starts_with(
"avx512.mask.add.p")) {
4537 if (Name.ends_with(
".512")) {
4539 if (Name[17] ==
's')
4540 IID = Intrinsic::x86_avx512_add_ps_512;
4542 IID = Intrinsic::x86_avx512_add_pd_512;
4544 Rep = Builder.CreateIntrinsic(
4552 }
else if (Name.starts_with(
"avx512.mask.div.p")) {
4553 if (Name.ends_with(
".512")) {
4555 if (Name[17] ==
's')
4556 IID = Intrinsic::x86_avx512_div_ps_512;
4558 IID = Intrinsic::x86_avx512_div_pd_512;
4560 Rep = Builder.CreateIntrinsic(
4568 }
else if (Name.starts_with(
"avx512.mask.mul.p")) {
4569 if (Name.ends_with(
".512")) {
4571 if (Name[17] ==
's')
4572 IID = Intrinsic::x86_avx512_mul_ps_512;
4574 IID = Intrinsic::x86_avx512_mul_pd_512;
4576 Rep = Builder.CreateIntrinsic(
4584 }
else if (Name.starts_with(
"avx512.mask.sub.p")) {
4585 if (Name.ends_with(
".512")) {
4587 if (Name[17] ==
's')
4588 IID = Intrinsic::x86_avx512_sub_ps_512;
4590 IID = Intrinsic::x86_avx512_sub_pd_512;
4592 Rep = Builder.CreateIntrinsic(
4600 }
else if ((Name.starts_with(
"avx512.mask.max.p") ||
4601 Name.starts_with(
"avx512.mask.min.p")) &&
4602 Name.drop_front(18) ==
".512") {
4603 bool IsDouble = Name[17] ==
'd';
4604 bool IsMin = Name[13] ==
'i';
4606 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4607 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4610 Rep = Builder.CreateIntrinsic(
4615 }
else if (Name.starts_with(
"avx512.mask.lzcnt.")) {
4617 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->
getType(),
4618 {CI->getArgOperand(0), Builder.getInt1(false)});
4621 }
else if (Name.starts_with(
"avx512.mask.psll")) {
4622 bool IsImmediate = Name[16] ==
'i' || (Name.size() > 18 && Name[18] ==
'i');
4623 bool IsVariable = Name[16] ==
'v';
4624 char Size = Name[16] ==
'.' ? Name[17]
4625 : Name[17] ==
'.' ? Name[18]
4626 : Name[18] ==
'.' ? Name[19]
4630 if (IsVariable && Name[17] !=
'.') {
4631 if (
Size ==
'd' && Name[17] ==
'2')
4632 IID = Intrinsic::x86_avx2_psllv_q;
4633 else if (
Size ==
'd' && Name[17] ==
'4')
4634 IID = Intrinsic::x86_avx2_psllv_q_256;
4635 else if (
Size ==
's' && Name[17] ==
'4')
4636 IID = Intrinsic::x86_avx2_psllv_d;
4637 else if (
Size ==
's' && Name[17] ==
'8')
4638 IID = Intrinsic::x86_avx2_psllv_d_256;
4639 else if (
Size ==
'h' && Name[17] ==
'8')
4640 IID = Intrinsic::x86_avx512_psllv_w_128;
4641 else if (
Size ==
'h' && Name[17] ==
'1')
4642 IID = Intrinsic::x86_avx512_psllv_w_256;
4643 else if (Name[17] ==
'3' && Name[18] ==
'2')
4644 IID = Intrinsic::x86_avx512_psllv_w_512;
4647 }
else if (Name.ends_with(
".128")) {
4649 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4650 : Intrinsic::x86_sse2_psll_d;
4651 else if (
Size ==
'q')
4652 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4653 : Intrinsic::x86_sse2_psll_q;
4654 else if (
Size ==
'w')
4655 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4656 : Intrinsic::x86_sse2_psll_w;
4659 }
else if (Name.ends_with(
".256")) {
4661 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4662 : Intrinsic::x86_avx2_psll_d;
4663 else if (
Size ==
'q')
4664 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4665 : Intrinsic::x86_avx2_psll_q;
4666 else if (
Size ==
'w')
4667 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4668 : Intrinsic::x86_avx2_psll_w;
4673 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4674 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4675 : Intrinsic::x86_avx512_psll_d_512;
4676 else if (
Size ==
'q')
4677 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4678 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4679 : Intrinsic::x86_avx512_psll_q_512;
4680 else if (
Size ==
'w')
4681 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4682 : Intrinsic::x86_avx512_psll_w_512;
4688 }
else if (Name.starts_with(
"avx512.mask.psrl")) {
4689 bool IsImmediate = Name[16] ==
'i' || (Name.size() > 18 && Name[18] ==
'i');
4690 bool IsVariable = Name[16] ==
'v';
4691 char Size = Name[16] ==
'.' ? Name[17]
4692 : Name[17] ==
'.' ? Name[18]
4693 : Name[18] ==
'.' ? Name[19]
4697 if (IsVariable && Name[17] !=
'.') {
4698 if (
Size ==
'd' && Name[17] ==
'2')
4699 IID = Intrinsic::x86_avx2_psrlv_q;
4700 else if (
Size ==
'd' && Name[17] ==
'4')
4701 IID = Intrinsic::x86_avx2_psrlv_q_256;
4702 else if (
Size ==
's' && Name[17] ==
'4')
4703 IID = Intrinsic::x86_avx2_psrlv_d;
4704 else if (
Size ==
's' && Name[17] ==
'8')
4705 IID = Intrinsic::x86_avx2_psrlv_d_256;
4706 else if (
Size ==
'h' && Name[17] ==
'8')
4707 IID = Intrinsic::x86_avx512_psrlv_w_128;
4708 else if (
Size ==
'h' && Name[17] ==
'1')
4709 IID = Intrinsic::x86_avx512_psrlv_w_256;
4710 else if (Name[17] ==
'3' && Name[18] ==
'2')
4711 IID = Intrinsic::x86_avx512_psrlv_w_512;
4714 }
else if (Name.ends_with(
".128")) {
4716 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4717 : Intrinsic::x86_sse2_psrl_d;
4718 else if (
Size ==
'q')
4719 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4720 : Intrinsic::x86_sse2_psrl_q;
4721 else if (
Size ==
'w')
4722 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4723 : Intrinsic::x86_sse2_psrl_w;
4726 }
else if (Name.ends_with(
".256")) {
4728 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4729 : Intrinsic::x86_avx2_psrl_d;
4730 else if (
Size ==
'q')
4731 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4732 : Intrinsic::x86_avx2_psrl_q;
4733 else if (
Size ==
'w')
4734 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4735 : Intrinsic::x86_avx2_psrl_w;
4740 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4741 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4742 : Intrinsic::x86_avx512_psrl_d_512;
4743 else if (
Size ==
'q')
4744 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4745 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4746 : Intrinsic::x86_avx512_psrl_q_512;
4747 else if (
Size ==
'w')
4748 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4749 : Intrinsic::x86_avx512_psrl_w_512;
4755 }
else if (Name.starts_with(
"avx512.mask.psra")) {
4756 bool IsImmediate = Name[16] ==
'i' || (Name.size() > 18 && Name[18] ==
'i');
4757 bool IsVariable = Name[16] ==
'v';
4758 char Size = Name[16] ==
'.' ? Name[17]
4759 : Name[17] ==
'.' ? Name[18]
4760 : Name[18] ==
'.' ? Name[19]
4764 if (IsVariable && Name[17] !=
'.') {
4765 if (
Size ==
's' && Name[17] ==
'4')
4766 IID = Intrinsic::x86_avx2_psrav_d;
4767 else if (
Size ==
's' && Name[17] ==
'8')
4768 IID = Intrinsic::x86_avx2_psrav_d_256;
4769 else if (
Size ==
'h' && Name[17] ==
'8')
4770 IID = Intrinsic::x86_avx512_psrav_w_128;
4771 else if (
Size ==
'h' && Name[17] ==
'1')
4772 IID = Intrinsic::x86_avx512_psrav_w_256;
4773 else if (Name[17] ==
'3' && Name[18] ==
'2')
4774 IID = Intrinsic::x86_avx512_psrav_w_512;
4777 }
else if (Name.ends_with(
".128")) {
4779 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4780 : Intrinsic::x86_sse2_psra_d;
4781 else if (
Size ==
'q')
4782 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4783 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4784 : Intrinsic::x86_avx512_psra_q_128;
4785 else if (
Size ==
'w')
4786 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4787 : Intrinsic::x86_sse2_psra_w;
4790 }
else if (Name.ends_with(
".256")) {
4792 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4793 : Intrinsic::x86_avx2_psra_d;
4794 else if (
Size ==
'q')
4795 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4796 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4797 : Intrinsic::x86_avx512_psra_q_256;
4798 else if (
Size ==
'w')
4799 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4800 : Intrinsic::x86_avx2_psra_w;
4805 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4806 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4807 : Intrinsic::x86_avx512_psra_d_512;
4808 else if (
Size ==
'q')
4809 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4810 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4811 : Intrinsic::x86_avx512_psra_q_512;
4812 else if (
Size ==
'w')
4813 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4814 : Intrinsic::x86_avx512_psra_w_512;
4820 }
else if (Name.starts_with(
"avx512.mask.move.s")) {
4822 }
else if (Name.starts_with(
"avx512.cvtmask2")) {
4824 }
else if (Name.ends_with(
".movntdqa")) {
4828 LoadInst *LI = Builder.CreateAlignedLoad(
4833 }
else if (Name.starts_with(
"fma.vfmadd.") ||
4834 Name.starts_with(
"fma.vfmsub.") ||
4835 Name.starts_with(
"fma.vfnmadd.") ||
4836 Name.starts_with(
"fma.vfnmsub.")) {
4837 bool NegMul = Name[6] ==
'n';
4838 bool NegAcc = NegMul ? Name[8] ==
's' : Name[7] ==
's';
4839 bool IsScalar = NegMul ? Name[12] ==
's' : Name[11] ==
's';
4850 if (NegMul && !IsScalar)
4851 Ops[0] = Builder.CreateFNeg(
Ops[0]);
4852 if (NegMul && IsScalar)
4853 Ops[1] = Builder.CreateFNeg(
Ops[1]);
4855 Ops[2] = Builder.CreateFNeg(
Ops[2]);
4857 Rep = Builder.CreateIntrinsic(Intrinsic::fma,
Ops[0]->
getType(),
Ops);
4861 }
else if (Name.starts_with(
"fma4.vfmadd.s")) {
4869 Rep = Builder.CreateIntrinsic(Intrinsic::fma,
Ops[0]->
getType(),
Ops);
4873 }
else if (Name.starts_with(
"avx512.mask.vfmadd.s") ||
4874 Name.starts_with(
"avx512.maskz.vfmadd.s") ||
4875 Name.starts_with(
"avx512.mask3.vfmadd.s") ||
4876 Name.starts_with(
"avx512.mask3.vfmsub.s") ||
4877 Name.starts_with(
"avx512.mask3.vfnmsub.s")) {
4878 bool IsMask3 = Name[11] ==
'3';
4879 bool IsMaskZ = Name[11] ==
'z';
4881 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4882 bool NegMul = Name[2] ==
'n';
4883 bool NegAcc = NegMul ? Name[4] ==
's' : Name[3] ==
's';
4889 if (NegMul && (IsMask3 || IsMaskZ))
4890 A = Builder.CreateFNeg(
A);
4891 if (NegMul && !(IsMask3 || IsMaskZ))
4892 B = Builder.CreateFNeg(
B);
4894 C = Builder.CreateFNeg(
C);
4896 A = Builder.CreateExtractElement(
A, (
uint64_t)0);
4897 B = Builder.CreateExtractElement(
B, (
uint64_t)0);
4898 C = Builder.CreateExtractElement(
C, (
uint64_t)0);
4905 if (Name.back() ==
'd')
4906 IID = Intrinsic::x86_avx512_vfmadd_f64;
4908 IID = Intrinsic::x86_avx512_vfmadd_f32;
4909 Rep = Builder.CreateIntrinsic(IID,
Ops);
4911 Rep = Builder.CreateFMA(
A,
B,
C);
4920 if (NegAcc && IsMask3)
4925 Rep = Builder.CreateInsertElement(CI->
getArgOperand(IsMask3 ? 2 : 0), Rep,
4927 }
else if (Name.starts_with(
"avx512.mask.vfmadd.p") ||
4928 Name.starts_with(
"avx512.mask.vfnmadd.p") ||
4929 Name.starts_with(
"avx512.mask.vfnmsub.p") ||
4930 Name.starts_with(
"avx512.mask3.vfmadd.p") ||
4931 Name.starts_with(
"avx512.mask3.vfmsub.p") ||
4932 Name.starts_with(
"avx512.mask3.vfnmsub.p") ||
4933 Name.starts_with(
"avx512.maskz.vfmadd.p")) {
4934 bool IsMask3 = Name[11] ==
'3';
4935 bool IsMaskZ = Name[11] ==
'z';
4937 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4938 bool NegMul = Name[2] ==
'n';
4939 bool NegAcc = NegMul ? Name[4] ==
's' : Name[3] ==
's';
4945 if (NegMul && (IsMask3 || IsMaskZ))
4946 A = Builder.CreateFNeg(
A);
4947 if (NegMul && !(IsMask3 || IsMaskZ))
4948 B = Builder.CreateFNeg(
B);
4950 C = Builder.CreateFNeg(
C);
4957 if (Name[Name.size() - 5] ==
's')
4958 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4960 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4964 Rep = Builder.CreateFMA(
A,
B,
C);
4972 }
else if (Name.starts_with(
"fma.vfmsubadd.p")) {
4976 if (VecWidth == 128 && EltWidth == 32)
4977 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4978 else if (VecWidth == 256 && EltWidth == 32)
4979 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4980 else if (VecWidth == 128 && EltWidth == 64)
4981 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4982 else if (VecWidth == 256 && EltWidth == 64)
4983 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4989 Ops[2] = Builder.CreateFNeg(
Ops[2]);
4990 Rep = Builder.CreateIntrinsic(IID,
Ops);
4991 }
else if (Name.starts_with(
"avx512.mask.vfmaddsub.p") ||
4992 Name.starts_with(
"avx512.mask3.vfmaddsub.p") ||
4993 Name.starts_with(
"avx512.maskz.vfmaddsub.p") ||
4994 Name.starts_with(
"avx512.mask3.vfmsubadd.p")) {
4995 bool IsMask3 = Name[11] ==
'3';
4996 bool IsMaskZ = Name[11] ==
'z';
4998 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4999 bool IsSubAdd = Name[3] ==
's';
5003 if (Name[Name.size() - 5] ==
's')
5004 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
5006 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
5011 Ops[2] = Builder.CreateFNeg(
Ops[2]);
5013 Rep = Builder.CreateIntrinsic(IID,
Ops);
5022 Value *Odd = Builder.CreateCall(FMA,
Ops);
5023 Ops[2] = Builder.CreateFNeg(
Ops[2]);
5024 Value *Even = Builder.CreateCall(FMA,
Ops);
5030 for (
int i = 0; i != NumElts; ++i)
5031 Idxs[i] = i + (i % 2) * NumElts;
5033 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
5041 }
else if (Name.starts_with(
"avx512.mask.pternlog.") ||
5042 Name.starts_with(
"avx512.maskz.pternlog.")) {
5043 bool ZeroMask = Name[11] ==
'z';
5047 if (VecWidth == 128 && EltWidth == 32)
5048 IID = Intrinsic::x86_avx512_pternlog_d_128;
5049 else if (VecWidth == 256 && EltWidth == 32)
5050 IID = Intrinsic::x86_avx512_pternlog_d_256;
5051 else if (VecWidth == 512 && EltWidth == 32)
5052 IID = Intrinsic::x86_avx512_pternlog_d_512;
5053 else if (VecWidth == 128 && EltWidth == 64)
5054 IID = Intrinsic::x86_avx512_pternlog_q_128;
5055 else if (VecWidth == 256 && EltWidth == 64)
5056 IID = Intrinsic::x86_avx512_pternlog_q_256;
5057 else if (VecWidth == 512 && EltWidth == 64)
5058 IID = Intrinsic::x86_avx512_pternlog_q_512;
5064 Rep = Builder.CreateIntrinsic(IID, Args);
5068 }
else if (Name.starts_with(
"avx512.mask.vpmadd52") ||
5069 Name.starts_with(
"avx512.maskz.vpmadd52")) {
5070 bool ZeroMask = Name[11] ==
'z';
5071 bool High = Name[20] ==
'h' || Name[21] ==
'h';
5074 if (VecWidth == 128 && !
High)
5075 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5076 else if (VecWidth == 256 && !
High)
5077 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5078 else if (VecWidth == 512 && !
High)
5079 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5080 else if (VecWidth == 128 &&
High)
5081 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5082 else if (VecWidth == 256 &&
High)
5083 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5084 else if (VecWidth == 512 &&
High)
5085 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5091 Rep = Builder.CreateIntrinsic(IID, Args);
5095 }
else if (Name.starts_with(
"avx512.mask.vpermi2var.") ||
5096 Name.starts_with(
"avx512.mask.vpermt2var.") ||
5097 Name.starts_with(
"avx512.maskz.vpermt2var.")) {
5098 bool ZeroMask = Name[11] ==
'z';
5099 bool IndexForm = Name[17] ==
'i';
5101 }
else if (Name.starts_with(
"avx512.mask.vpdpbusd.") ||
5102 Name.starts_with(
"avx512.maskz.vpdpbusd.") ||
5103 Name.starts_with(
"avx512.mask.vpdpbusds.") ||
5104 Name.starts_with(
"avx512.maskz.vpdpbusds.")) {
5105 bool ZeroMask = Name[11] ==
'z';
5106 bool IsSaturating = Name[ZeroMask ? 21 : 20] ==
's';
5109 if (VecWidth == 128 && !IsSaturating)
5110 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5111 else if (VecWidth == 256 && !IsSaturating)
5112 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5113 else if (VecWidth == 512 && !IsSaturating)
5114 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5115 else if (VecWidth == 128 && IsSaturating)
5116 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5117 else if (VecWidth == 256 && IsSaturating)
5118 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5119 else if (VecWidth == 512 && IsSaturating)
5120 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5130 if (Args[1]->
getType()->isVectorTy() &&
5133 ->isIntegerTy(32) &&
5134 Args[2]->
getType()->isVectorTy() &&
5137 ->isIntegerTy(32)) {
5138 Type *NewArgType =
nullptr;
5139 if (VecWidth == 128)
5141 else if (VecWidth == 256)
5143 else if (VecWidth == 512)
5149 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5150 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5153 Rep = Builder.CreateIntrinsic(IID, Args);
5157 }
else if (Name.starts_with(
"avx512.mask.vpdpwssd.") ||
5158 Name.starts_with(
"avx512.maskz.vpdpwssd.") ||
5159 Name.starts_with(
"avx512.mask.vpdpwssds.") ||
5160 Name.starts_with(
"avx512.maskz.vpdpwssds.")) {
5161 bool ZeroMask = Name[11] ==
'z';
5162 bool IsSaturating = Name[ZeroMask ? 21 : 20] ==
's';
5165 if (VecWidth == 128 && !IsSaturating)
5166 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5167 else if (VecWidth == 256 && !IsSaturating)
5168 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5169 else if (VecWidth == 512 && !IsSaturating)
5170 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5171 else if (VecWidth == 128 && IsSaturating)
5172 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5173 else if (VecWidth == 256 && IsSaturating)
5174 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5175 else if (VecWidth == 512 && IsSaturating)
5176 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5186 if (Args[1]->
getType()->isVectorTy() &&
5189 ->isIntegerTy(32) &&
5190 Args[2]->
getType()->isVectorTy() &&
5193 ->isIntegerTy(32)) {
5194 Type *NewArgType =
nullptr;
5195 if (VecWidth == 128)
5197 else if (VecWidth == 256)
5199 else if (VecWidth == 512)
5205 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5206 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5209 Rep = Builder.CreateIntrinsic(IID, Args);
5213 }
else if (Name ==
"addcarryx.u32" || Name ==
"addcarryx.u64" ||
5214 Name ==
"addcarry.u32" || Name ==
"addcarry.u64" ||
5215 Name ==
"subborrow.u32" || Name ==
"subborrow.u64") {
5217 if (Name[0] ==
'a' && Name.back() ==
'2')
5218 IID = Intrinsic::x86_addcarry_32;
5219 else if (Name[0] ==
'a' && Name.back() ==
'4')
5220 IID = Intrinsic::x86_addcarry_64;
5221 else if (Name[0] ==
's' && Name.back() ==
'2')
5222 IID = Intrinsic::x86_subborrow_32;
5223 else if (Name[0] ==
's' && Name.back() ==
'4')
5224 IID = Intrinsic::x86_subborrow_64;
5231 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5234 Value *
Data = Builder.CreateExtractValue(NewCall, 1);
5237 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5241 }
else if (Name.starts_with(
"avx512.mask.") &&
5244 }
else if (Name.starts_with(
"bmi.pdep.")) {
5246 }
else if (Name.starts_with(
"bmi.pext.")) {
5256 if (Name.starts_with(
"neon.bfcvt")) {
5257 if (Name.starts_with(
"neon.bfcvtn2")) {
5259 std::iota(LoMask.
begin(), LoMask.
end(), 0);
5261 std::iota(ConcatMask.
begin(), ConcatMask.
end(), 0);
5262 Value *Inactive = Builder.CreateShuffleVector(CI->
getOperand(0), LoMask);
5265 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5266 }
else if (Name.starts_with(
"neon.bfcvtn")) {
5268 std::iota(ConcatMask.
begin(), ConcatMask.
end(), 0);
5272 dbgs() <<
"Trunc: " << *Trunc <<
"\n";
5273 return Builder.CreateShuffleVector(
5276 return Builder.CreateFPTrunc(CI->
getOperand(0),
5279 }
else if (Name.starts_with(
"sve.fcvt")) {
5282 .
Case(
"sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5283 .
Case(
"sve.fcvtnt.bf16f32",
5284 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5296 if (Args[1]->
getType() != BadPredTy)
5299 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5300 BadPredTy, Args[1]);
5301 Args[1] = Builder.CreateIntrinsic(
5302 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5304 return Builder.CreateIntrinsic(NewID, Args,
nullptr,
5308 if (Name ==
"neon.vcvtfp2hf")
5309 return Builder.CreateBitCast(
5310 Builder.CreateFPTrunc(
5314 if (Name ==
"neon.vcvthf2fp")
5315 return Builder.CreateFPExt(
5316 Builder.CreateBitCast(
5326 if (Name ==
"mve.vctp64.old") {
5329 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5332 Value *
C1 = Builder.CreateIntrinsic(
5333 Intrinsic::arm_mve_pred_v2i,
5335 return Builder.CreateIntrinsic(
5336 Intrinsic::arm_mve_pred_i2v,
5338 }
else if (Name ==
"mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5339 Name ==
"mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5340 Name ==
"mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5341 Name ==
"mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5343 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5344 Name ==
"mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5345 Name ==
"mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5346 Name ==
"mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5348 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5349 Name ==
"mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5350 Name ==
"cde.vcx1q.predicated.v2i64.v4i1" ||
5351 Name ==
"cde.vcx1qa.predicated.v2i64.v4i1" ||
5352 Name ==
"cde.vcx2q.predicated.v2i64.v4i1" ||
5353 Name ==
"cde.vcx2qa.predicated.v2i64.v4i1" ||
5354 Name ==
"cde.vcx3q.predicated.v2i64.v4i1" ||
5355 Name ==
"cde.vcx3qa.predicated.v2i64.v4i1") {
5356 std::vector<Type *> Tys;
5360 case Intrinsic::arm_mve_mull_int_predicated:
5361 case Intrinsic::arm_mve_vqdmull_predicated:
5362 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5365 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5366 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5367 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5371 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5375 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5379 case Intrinsic::arm_cde_vcx1q_predicated:
5380 case Intrinsic::arm_cde_vcx1qa_predicated:
5381 case Intrinsic::arm_cde_vcx2q_predicated:
5382 case Intrinsic::arm_cde_vcx2qa_predicated:
5383 case Intrinsic::arm_cde_vcx3q_predicated:
5384 case Intrinsic::arm_cde_vcx3qa_predicated:
5391 std::vector<Value *>
Ops;
5393 Type *Ty =
Op->getType();
5394 if (Ty->getScalarSizeInBits() == 1) {
5395 Value *
C1 = Builder.CreateIntrinsic(
5396 Intrinsic::arm_mve_pred_v2i,
5398 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty},
C1);
5403 return Builder.CreateIntrinsic(ID, Tys,
Ops,
nullptr,
5418 auto UpgradeLegacyWMMAIUIntrinsicCall =
5423 Args.push_back(Builder.getFalse());
5427 F->getParent(),
F->getIntrinsicID(), OverloadTys);
5434 auto *NewCall =
cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5438 NewCall->copyMetadata(*CI);
5442 if (
F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5443 assert(CI->
arg_size() == 7 &&
"Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5444 "intrinsic should have 7 arguments");
5447 return UpgradeLegacyWMMAIUIntrinsicCall(
F, CI, Builder, {
T1, T2});
5449 if (
F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5450 assert(CI->
arg_size() == 8 &&
"Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5451 "intrinsic should have 8 arguments");
5456 return UpgradeLegacyWMMAIUIntrinsicCall(
F, CI, Builder, {
T1, T2, T3, T4});
5459 switch (
F->getIntrinsicID()) {
5462 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5463 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5464 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5465 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5466 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5467 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5482 if (
F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5485 F->getParent(),
F->getIntrinsicID(), Overloads);
5490 auto *NewCall =
cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5494 NewCall->copyMetadata(*CI);
5495 NewCall->takeName(CI);
5500 if (Name.starts_with(
"fcmp.") || Name.starts_with(
"icmp.")) {
5506 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5507 CI->
getType(), Intrinsic::amdgcn_ballot, Cmp);
5515 if (Name.starts_with(
"addrspacecast.nonnull")) {
5518 Value *ASC = Builder.CreateAddrSpaceCast(
5541 if (NumOperands < 3)
5554 bool IsVolatile =
false;
5558 if (NumOperands > 3)
5563 if (NumOperands > 5) {
5565 IsVolatile = !VolatileArg || !VolatileArg->
isZero();
5579 if (VT->getElementType()->isIntegerTy(16)) {
5582 Val = Builder.CreateBitCast(Val, AsBF16);
5590 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5592 unsigned AddrSpace = PtrTy->getAddressSpace();
5595 RMW->
setMetadata(
"amdgpu.no.fine.grained.memory", EmptyMD);
5597 RMW->
setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5602 MDNode *RangeNotPrivate =
5605 RMW->
setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5611 return Builder.CreateBitCast(RMW, RetTy);
5632 return MAV->getMetadata();
5641 if (Name ==
"label") {
5643 }
else if (Name ==
"assign") {
5650 }
else if (Name ==
"declare") {
5654 }
else if (Name ==
"addr") {
5664 unwrapMAVOp(CI, 1), ExprNode,
nullptr,
nullptr,
nullptr);
5665 }
else if (Name ==
"value") {
5668 unsigned ExprOp = 2;
5683 assert(DR &&
"Unhandled intrinsic kind in upgrade to DbgRecord");
5691 int64_t OffsetVal =
Offset->getSExtValue();
5692 return Builder.CreateIntrinsic(OffsetVal >= 0
5693 ? Intrinsic::vector_splice_left
5694 : Intrinsic::vector_splice_right,
5696 {CI->getArgOperand(0), CI->getArgOperand(1),
5697 Builder.getInt32(std::abs(OffsetVal))});
5702 if (Name.starts_with(
"to.fp16")) {
5704 Builder.CreateFPTrunc(CI->
getArgOperand(0), Builder.getHalfTy());
5705 return Builder.CreateBitCast(Cast, CI->
getType());
5708 if (Name.starts_with(
"from.fp16")) {
5710 Builder.CreateBitCast(CI->
getArgOperand(0), Builder.getHalfTy());
5711 return Builder.CreateFPExt(Cast, CI->
getType());
5770 else if (Opcode == Instruction::ICmp)
5773 else if (Opcode == Instruction::FCmp)
5776 else if (Opcode == Instruction::Select)
5781 Rep = Builder.CreateIntrinsic(CI->
getType(), IntrinsicID, Args, {});
5793 if (Defaults.empty())
5796 unsigned OldArgCount = CI->
arg_size();
5797 unsigned NewArgCount = NewFn->
arg_size();
5799 if (OldArgCount < FirstDefault)
5803 if (OldArgCount > NewArgCount)
5808 if (OldArgCount == NewArgCount) {
5820 for (
unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5821 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5822 "missing argument outside the default range");
5823 Type *ParamTy = NewFT->getParamType(Idx);
5828 NewArgs.
push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5834 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5865 if (!Name.consume_front(
"llvm."))
5868 bool IsX86 = Name.consume_front(
"x86.");
5869 bool IsNVVM = Name.consume_front(
"nvvm.");
5870 bool IsAArch64 = Name.consume_front(
"aarch64.");
5871 bool IsARM = Name.consume_front(
"arm.");
5872 bool IsAMDGCN = Name.consume_front(
"amdgcn.");
5873 bool IsDbg = Name.consume_front(
"dbg.");
5875 (Name.consume_front(
"experimental.vector.splice") ||
5876 Name.consume_front(
"vector.splice")) &&
5877 !(Name.starts_with(
".left") || Name.starts_with(
".right"));
5878 Value *Rep =
nullptr;
5880 if (!IsX86 && Name ==
"stackprotectorcheck") {
5882 }
else if (IsNVVM) {
5886 }
else if (IsAArch64) {
5890 }
else if (IsAMDGCN) {
5894 }
else if (IsOldSplice) {
5896 }
else if (Name.consume_front(
"convert.")) {
5898 }
else if (Name ==
"lifetime.start.i64" || Name ==
"lifetime.end.i64") {
5913 const auto &DefaultCase = [&]() ->
void {
5921 "Unknown function for CallBase upgrade and isn't just a name change");
5929 "Return type must have changed");
5930 assert(OldST->getNumElements() ==
5932 "Must have same number of elements");
5935 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5938 for (
unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5939 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5940 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5961 case Intrinsic::arm_neon_vst1:
5962 case Intrinsic::arm_neon_vst2:
5963 case Intrinsic::arm_neon_vst3:
5964 case Intrinsic::arm_neon_vst4:
5965 case Intrinsic::arm_neon_vst2lane:
5966 case Intrinsic::arm_neon_vst3lane:
5967 case Intrinsic::arm_neon_vst4lane: {
5969 NewCall = Builder.CreateCall(NewFn, Args);
5972 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5973 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5974 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5979 NewCall = Builder.CreateCall(NewFn, Args);
5982 case Intrinsic::aarch64_sve_ld3_sret:
5983 case Intrinsic::aarch64_sve_ld4_sret:
5984 case Intrinsic::aarch64_sve_ld2_sret: {
5992 Name = Name.substr(5);
5999 unsigned MinElts = RetTy->getMinNumElements() /
N;
6001 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
6003 for (
unsigned I = 0;
I <
N;
I++) {
6004 Value *SRet = Builder.CreateExtractValue(NewLdCall,
I);
6005 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet,
I * MinElts);
6011 case Intrinsic::coro_end_async:
6012 case Intrinsic::coro_end: {
6014 if (NewFn->
getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
6016 NewCall = Builder.CreateCall(NewFn, Args);
6021 CI->
getModule(), Intrinsic::coro_is_in_ramp);
6022 Value *InRamp = Builder.CreateCall(IsInRamp);
6032 case Intrinsic::vector_extract: {
6034 Name = Name.substr(5);
6035 if (!Name.starts_with(
"aarch64.sve.tuple.get")) {
6040 unsigned MinElts = RetTy->getMinNumElements();
6043 NewCall = Builder.CreateCall(NewFn, {CI->
getArgOperand(0), NewIdx});
6047 case Intrinsic::vector_insert: {
6049 Name = Name.substr(5);
6050 if (!Name.starts_with(
"aarch64.sve.tuple")) {
6054 if (Name.starts_with(
"aarch64.sve.tuple.set")) {
6059 NewCall = Builder.CreateCall(
6063 if (Name.starts_with(
"aarch64.sve.tuple.create")) {
6069 assert(
N > 1 &&
"Create is expected to be between 2-4");
6072 unsigned MinElts = RetTy->getMinNumElements() /
N;
6073 for (
unsigned I = 0;
I <
N;
I++) {
6075 Ret = Builder.CreateInsertVector(RetTy, Ret, V,
I * MinElts);
6082 case Intrinsic::arm_neon_bfdot:
6083 case Intrinsic::arm_neon_bfmmla:
6084 case Intrinsic::arm_neon_bfmlalb:
6085 case Intrinsic::arm_neon_bfmlalt:
6086 case Intrinsic::aarch64_neon_bfdot:
6087 case Intrinsic::aarch64_neon_bfmmla:
6088 case Intrinsic::aarch64_neon_bfmlalb:
6089 case Intrinsic::aarch64_neon_bfmlalt: {
6092 "Mismatch between function args and call args");
6093 size_t OperandWidth =
6095 assert((OperandWidth == 64 || OperandWidth == 128) &&
6096 "Unexpected operand width");
6098 auto Iter = CI->
args().begin();
6099 Args.push_back(*Iter++);
6100 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6101 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6102 NewCall = Builder.CreateCall(NewFn, Args);
6106 case Intrinsic::bitreverse:
6107 NewCall = Builder.CreateCall(NewFn, {CI->
getArgOperand(0)});
6110 case Intrinsic::ctlz:
6111 case Intrinsic::cttz: {
6118 Builder.CreateCall(NewFn, {CI->
getArgOperand(0), Builder.getFalse()});
6122 case Intrinsic::objectsize: {
6123 Value *NullIsUnknownSize =
6127 NewCall = Builder.CreateCall(
6132 case Intrinsic::ctpop:
6133 NewCall = Builder.CreateCall(NewFn, {CI->
getArgOperand(0)});
6135 case Intrinsic::dbg_value: {
6137 Name = Name.substr(5);
6139 if (Name.starts_with(
"dbg.addr")) {
6153 if (
Offset->isNullValue()) {
6154 NewCall = Builder.CreateCall(
6163 case Intrinsic::ptr_annotation:
6171 NewCall = Builder.CreateCall(
6180 case Intrinsic::var_annotation:
6187 NewCall = Builder.CreateCall(
6196 case Intrinsic::riscv_aes32dsi:
6197 case Intrinsic::riscv_aes32dsmi:
6198 case Intrinsic::riscv_aes32esi:
6199 case Intrinsic::riscv_aes32esmi:
6200 case Intrinsic::riscv_sm4ks:
6201 case Intrinsic::riscv_sm4ed: {
6211 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6212 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6218 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6219 Value *Res = NewCall;
6221 Res = Builder.CreateIntCast(NewCall, CI->
getType(),
true);
6227 case Intrinsic::nvvm_mapa_shared_cluster: {
6231 Value *Res = NewCall;
6232 Res = Builder.CreateAddrSpaceCast(
6239 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6241 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6243 Args[0] = Builder.CreateAddrSpaceCast(
6247 Args.push_back(Builder.getInt32(0));
6249 NewCall = Builder.CreateCall(NewFn, Args);
6255 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6260 for (
unsigned I = 0;
I < 4; ++
I)
6262 Args.push_back(Builder.getInt32(0));
6263 Args.push_back(Builder.getInt32(0));
6266 Args.push_back(Builder.getInt1(
false));
6267 Args.push_back(Builder.getInt32(0));
6269 NewCall = Builder.CreateCall(NewFn, Args);
6275 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6278 Args[0] = Builder.CreateAddrSpaceCast(
6281 NewCall = Builder.CreateCall(NewFn, Args);
6288#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6289 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6291#undef G2S_CLUSTER_CASE
6296 Args[0] = Builder.CreateAddrSpaceCast(
6302 Args.push_back(Builder.getInt32(0));
6304 NewCall = Builder.CreateCall(NewFn, Args);
6311#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6312 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6320 "expected only the trailing flag_valid_pattern to be missing");
6321 Args.push_back(Builder.getInt32(0));
6323 NewCall = Builder.CreateCall(NewFn, Args);
6329#undef NVVM_TMA_G2S_MODES
6332 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6333 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6334 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6335 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6336 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6337 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6338 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6339 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6341 Name.consume_front(
"llvm.nvvm.cp.async.bulk.tensor.reduce.");
6345 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6346 NewCall = Builder.CreateCall(NewFn, Args);
6350 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cta_space_cluster:
6351 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cluster_space_cluster:
6352 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cta_space_cluster:
6353 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cluster_space_cluster:
6354 case Intrinsic::nvvm_mbarrier_arrive_scope_cta_space_cluster:
6355 case Intrinsic::nvvm_mbarrier_arrive_scope_cluster_space_cluster:
6356 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cta_space_cluster:
6357 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cluster_space_cluster:
6358 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cta_space_cluster:
6359 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cluster_space_cluster:
6360 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cta_space_cluster:
6361 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cluster_space_cluster:
6362 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cta_space_cluster:
6363 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cluster_space_cluster:
6364 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cta_space_cluster:
6365 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cluster_space_cluster:
6366 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cta_space_cluster:
6367 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cluster_space_cluster:
6368 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cta_space_cluster:
6369 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cluster_space_cluster: {
6372 Args.push_back(Builder.getInt32(0));
6373 Args.push_back(Builder.getFalse());
6374 NewCall = Builder.CreateCall(NewFn, Args);
6380 case Intrinsic::nvvm_mbarrier_init: {
6384 if (Args.size() == 2)
6385 Args.push_back(Builder.getInt32(0));
6386 NewCall = Builder.CreateCall(NewFn, Args);
6389 case Intrinsic::riscv_sha256sig0:
6390 case Intrinsic::riscv_sha256sig1:
6391 case Intrinsic::riscv_sha256sum0:
6392 case Intrinsic::riscv_sha256sum1:
6393 case Intrinsic::riscv_sm3p0:
6394 case Intrinsic::riscv_sm3p1: {
6401 Builder.CreateTrunc(CI->
getArgOperand(0), Builder.getInt32Ty());
6403 NewCall = Builder.CreateCall(NewFn, Arg);
6405 Builder.CreateIntCast(NewCall, CI->
getType(),
true);
6412 case Intrinsic::x86_xop_vfrcz_ss:
6413 case Intrinsic::x86_xop_vfrcz_sd:
6414 NewCall = Builder.CreateCall(NewFn, {CI->
getArgOperand(1)});
6417 case Intrinsic::x86_xop_vpermil2pd:
6418 case Intrinsic::x86_xop_vpermil2ps:
6419 case Intrinsic::x86_xop_vpermil2pd_256:
6420 case Intrinsic::x86_xop_vpermil2ps_256: {
6424 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6425 NewCall = Builder.CreateCall(NewFn, Args);
6429 case Intrinsic::x86_sse41_ptestc:
6430 case Intrinsic::x86_sse41_ptestz:
6431 case Intrinsic::x86_sse41_ptestnzc: {
6445 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy,
"cast");
6446 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy,
"cast");
6448 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6452 case Intrinsic::x86_rdtscp: {
6458 NewCall = Builder.CreateCall(NewFn);
6460 Value *
Data = Builder.CreateExtractValue(NewCall, 1);
6463 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6471 case Intrinsic::x86_sse41_insertps:
6472 case Intrinsic::x86_sse41_dppd:
6473 case Intrinsic::x86_sse41_dpps:
6474 case Intrinsic::x86_sse41_mpsadbw:
6475 case Intrinsic::x86_avx_dp_ps_256:
6476 case Intrinsic::x86_avx2_mpsadbw: {
6482 Args.back() = Builder.CreateTrunc(Args.back(),
Type::getInt8Ty(
C),
"trunc");
6483 NewCall = Builder.CreateCall(NewFn, Args);
6487 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6488 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6489 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6490 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6491 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6492 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6498 NewCall = Builder.CreateCall(NewFn, Args);
6507 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6508 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6509 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6510 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6511 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6512 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6516 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6517 Args[1] = Builder.CreateBitCast(
6520 NewCall = Builder.CreateCall(NewFn, Args);
6521 Value *Res = Builder.CreateBitCast(
6529 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6530 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6531 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6535 Args[1] = Builder.CreateBitCast(
6537 Args[2] = Builder.CreateBitCast(
6540 NewCall = Builder.CreateCall(NewFn, Args);
6544 case Intrinsic::amdgcn_asyncmark:
6545 case Intrinsic::amdgcn_wait_asyncmark: {
6549 Args.push_back(Builder.getInt32(0));
6554 NewCall = Builder.CreateCall(NewFn, Args, Bundles);
6562 case Intrinsic::thread_pointer: {
6563 NewCall = Builder.CreateCall(NewFn, {});
6567 case Intrinsic::memcpy:
6568 case Intrinsic::memmove:
6569 case Intrinsic::memset: {
6585 NewCall = Builder.CreateCall(NewFn, Args);
6588 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6589 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6590 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6595 MemCI->setDestAlignment(
Align->getMaybeAlignValue());
6598 MTI->setSourceAlignment(
Align->getMaybeAlignValue());
6602 case Intrinsic::masked_load:
6603 case Intrinsic::masked_gather:
6604 case Intrinsic::masked_store:
6605 case Intrinsic::masked_scatter: {
6611 auto GetMaybeAlign = [](
Value *
Op) {
6613 uint64_t Val = CI->getZExtValue();
6621 auto GetAlign = [&](
Value *
Op) {
6630 case Intrinsic::masked_load:
6631 NewCall = Builder.CreateMaskedLoad(
6635 case Intrinsic::masked_gather:
6636 NewCall = Builder.CreateMaskedGather(
6642 case Intrinsic::masked_store:
6643 NewCall = Builder.CreateMaskedStore(
6647 case Intrinsic::masked_scatter:
6648 NewCall = Builder.CreateMaskedScatter(
6650 DL.getValueOrABITypeAlignment(
6664 case Intrinsic::lifetime_start:
6665 case Intrinsic::lifetime_end: {
6677 NewCall = Builder.CreateLifetimeStart(Ptr);
6679 NewCall = Builder.CreateLifetimeEnd(Ptr);
6688 case Intrinsic::x86_avx512_vpdpbusd_128:
6689 case Intrinsic::x86_avx512_vpdpbusd_256:
6690 case Intrinsic::x86_avx512_vpdpbusd_512:
6691 case Intrinsic::x86_avx512_vpdpbusds_128:
6692 case Intrinsic::x86_avx512_vpdpbusds_256:
6693 case Intrinsic::x86_avx512_vpdpbusds_512:
6694 case Intrinsic::x86_avx2_vpdpbssd_128:
6695 case Intrinsic::x86_avx2_vpdpbssd_256:
6696 case Intrinsic::x86_avx10_vpdpbssd_512:
6697 case Intrinsic::x86_avx2_vpdpbssds_128:
6698 case Intrinsic::x86_avx2_vpdpbssds_256:
6699 case Intrinsic::x86_avx10_vpdpbssds_512:
6700 case Intrinsic::x86_avx2_vpdpbsud_128:
6701 case Intrinsic::x86_avx2_vpdpbsud_256:
6702 case Intrinsic::x86_avx10_vpdpbsud_512:
6703 case Intrinsic::x86_avx2_vpdpbsuds_128:
6704 case Intrinsic::x86_avx2_vpdpbsuds_256:
6705 case Intrinsic::x86_avx10_vpdpbsuds_512:
6706 case Intrinsic::x86_avx2_vpdpbuud_128:
6707 case Intrinsic::x86_avx2_vpdpbuud_256:
6708 case Intrinsic::x86_avx10_vpdpbuud_512:
6709 case Intrinsic::x86_avx2_vpdpbuuds_128:
6710 case Intrinsic::x86_avx2_vpdpbuuds_256:
6711 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6716 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6717 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6719 NewCall = Builder.CreateCall(NewFn, Args);
6722 case Intrinsic::x86_avx512_vpdpwssd_128:
6723 case Intrinsic::x86_avx512_vpdpwssd_256:
6724 case Intrinsic::x86_avx512_vpdpwssd_512:
6725 case Intrinsic::x86_avx512_vpdpwssds_128:
6726 case Intrinsic::x86_avx512_vpdpwssds_256:
6727 case Intrinsic::x86_avx512_vpdpwssds_512:
6728 case Intrinsic::x86_avx2_vpdpwsud_128:
6729 case Intrinsic::x86_avx2_vpdpwsud_256:
6730 case Intrinsic::x86_avx10_vpdpwsud_512:
6731 case Intrinsic::x86_avx2_vpdpwsuds_128:
6732 case Intrinsic::x86_avx2_vpdpwsuds_256:
6733 case Intrinsic::x86_avx10_vpdpwsuds_512:
6734 case Intrinsic::x86_avx2_vpdpwusd_128:
6735 case Intrinsic::x86_avx2_vpdpwusd_256:
6736 case Intrinsic::x86_avx10_vpdpwusd_512:
6737 case Intrinsic::x86_avx2_vpdpwusds_128:
6738 case Intrinsic::x86_avx2_vpdpwusds_256:
6739 case Intrinsic::x86_avx10_vpdpwusds_512:
6740 case Intrinsic::x86_avx2_vpdpwuud_128:
6741 case Intrinsic::x86_avx2_vpdpwuud_256:
6742 case Intrinsic::x86_avx10_vpdpwuud_512:
6743 case Intrinsic::x86_avx2_vpdpwuuds_128:
6744 case Intrinsic::x86_avx2_vpdpwuuds_256:
6745 case Intrinsic::x86_avx10_vpdpwuuds_512:
6750 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6751 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6753 NewCall = Builder.CreateCall(NewFn, Args);
6756 assert(NewCall &&
"Should have either set this variable or returned through "
6757 "the default case");
6764 assert(
F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
6778 F->eraseFromParent();
6784 if (NumOperands == 0)
6792 if (NumOperands == 3) {
6796 Metadata *Elts2[] = {ScalarType, ScalarType,
6812 if (NumOperands == 0 || NumOperands % 3 != 0)
6817 for (
unsigned I = 2;
I < NumOperands;
I += 3) {
6822 if (Upgraded ==
Tag)
6843 New->replaceOperandWith(0, New);
6849 unsigned NumOperands =
Domain->getNumOperands();
6850 bool HadDescription = NumOperands == 2;
6852 if (NumOperands == 0 || NumOperands > 2 ||
6877 if (Scope->getNumOperands() < 2 || !
isa<MDNode>(Scope->getOperand(1)))
6882 if (UpgradedDomain ==
Domain)
6886 Ops[1] = UpgradedDomain;
6888 Upgraded[Scope] = Upgrade;
6895 if (
MDNode *Upgrade = UpgradedAliasScopes.
lookup(&ScopeList))
6905 Ops.push_back(Scope);
6911 UpgradedAliasScopes[&ScopeList] = Upgrade;
6917 if (
Opc != Instruction::BitCast)
6921 Type *SrcTy = V->getType();
6938 if (
Opc != Instruction::BitCast)
6941 Type *SrcTy =
C->getType();
6958 if (Flag.getNumOperands() < 3)
6959 return std::nullopt;
6961 return Name->getString();
6962 return std::nullopt;
6968 if (!
getIROptions(M.getContext()).auto_upgrade_debug_info)
6976 if (
NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6977 auto OpIt =
find_if(ModFlags->operands(), [](
const MDNode *Flag) {
6978 if (auto Name = getModuleFlagNameSafely(*Flag))
6979 return *Name ==
"Debug Info Version";
6982 if (OpIt != ModFlags->op_end()) {
6983 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6990 bool BrokenDebugInfo =
false;
6993 if (!BrokenDebugInfo)
6999 M.getContext().diagnose(Diag);
7006 M.getContext().diagnose(DiagVersion);
7016 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
7019 if (
F->hasFnAttribute(Attr)) {
7022 StringRef S =
F->getFnAttribute(Attr).getValueAsString();
7024 auto [Part, Rest] = S.
split(
',');
7030 const unsigned Dim = DimC -
'x';
7031 assert(Dim < 3 &&
"Unexpected dim char");
7041 F->addFnAttr(Attr, NewAttr);
7045 return S ==
"x" || S ==
"y" || S ==
"z";
7050 if (
K ==
"kernel") {
7062 const unsigned Idx = (AlignIdxValuePair >> 16);
7063 const Align StackAlign =
Align(AlignIdxValuePair & 0xFFFF);
7068 if (
K ==
"maxclusterrank" ||
K ==
"cluster_max_blocks") {
7073 if (
K ==
"minctasm") {
7078 if (
K ==
"maxnreg") {
7083 if (
K.consume_front(
"maxntid") &&
isXYZ(
K)) {
7087 if (
K.consume_front(
"reqntid") &&
isXYZ(
K)) {
7091 if (
K.consume_front(
"cluster_dim_") &&
isXYZ(
K)) {
7095 if (
K ==
"grid_constant") {
7110 NamedMDNode *NamedMD = M.getNamedMetadata(
"nvvm.annotations");
7117 if (!SeenNodes.
insert(MD).second)
7124 assert((MD->getNumOperands() % 2) == 1 &&
"Invalid number of operands");
7131 for (
unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
7133 const MDOperand &V = MD->getOperand(j + 1);
7139 if (NewOperands.
size() > 1)
7152 const char *MarkerKey =
"clang.arc.retainAutoreleasedReturnValueMarker";
7153 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
7154 if (ModRetainReleaseMarker) {
7160 ID->getString().split(ValueComp,
"#");
7161 if (ValueComp.
size() == 2) {
7162 std::string NewValue = ValueComp[0].str() +
";" + ValueComp[1].str();
7166 M.eraseNamedMetadata(ModRetainReleaseMarker);
7177 auto UpgradeToIntrinsic = [&](
const char *OldFunc,
7203 bool InvalidCast =
false;
7205 for (
unsigned I = 0, E = CI->
arg_size();
I != E; ++
I) {
7218 Arg = Builder.CreateBitCast(Arg, NewFuncTy->
getParamType(
I));
7220 Args.push_back(Arg);
7227 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
7232 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->
getType());
7245 UpgradeToIntrinsic(
"clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7253 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7254 {
"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7255 {
"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7256 {
"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7257 {
"objc_autoreleaseReturnValue",
7258 llvm::Intrinsic::objc_autoreleaseReturnValue},
7259 {
"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7260 {
"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7261 {
"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7262 {
"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7263 {
"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7264 {
"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7265 {
"objc_release", llvm::Intrinsic::objc_release},
7266 {
"objc_retain", llvm::Intrinsic::objc_retain},
7267 {
"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7268 {
"objc_retainAutoreleaseReturnValue",
7269 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7270 {
"objc_retainAutoreleasedReturnValue",
7271 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7272 {
"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7273 {
"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7274 {
"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7275 {
"objc_unsafeClaimAutoreleasedReturnValue",
7276 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7277 {
"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7278 {
"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7279 {
"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7280 {
"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7281 {
"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7282 {
"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7283 {
"objc_arc_annotation_topdown_bbstart",
7284 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7285 {
"objc_arc_annotation_topdown_bbend",
7286 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7287 {
"objc_arc_annotation_bottomup_bbstart",
7288 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7289 {
"objc_arc_annotation_bottomup_bbend",
7290 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7292 for (
auto &
I : RuntimeFuncs)
7293 UpgradeToIntrinsic(
I.first,
I.second);
7317 std::optional<bool> UseAddressDisc;
7320 if (
const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7321 for (
const MDNode *Flag : ModFlags->operands()) {
7323 if (Name && (*Name ==
"ptrauth-init-fini" ||
7324 *Name ==
"ptrauth-init-fini-address-discrimination"))
7329 auto UpgradeSinglePointer = [&UseAddressDisc](
Constant *CV) ->
Constant * {
7330 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7331 constexpr unsigned ExpectedAddressMarker = 1;
7334 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7337 bool HasAddressDisc;
7338 if (!CPA->hasAddressDiscriminator())
7339 HasAddressDisc =
false;
7340 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7341 HasAddressDisc =
true;
7345 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7348 UseAddressDisc = HasAddressDisc;
7349 return CPA->getPointer();
7353 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7356 for (
const char *Name : {
"llvm.global_ctors",
"llvm.global_dtors"}) {
7358 if (!GV || !GV->hasInitializer())
7362 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7365 std::vector<Constant *> NewStructors;
7366 NewStructors.reserve(OldStructorsArray->getNumOperands());
7368 for (
Use &U : OldStructorsArray->operands()) {
7377 Func = UpgradeSinglePointer(Func);
7381 NewStructors.push_back(
7390 if (GlobalArraysToUpgrade.
empty())
7392 assert(UseAddressDisc.has_value());
7394 for (
auto [GV, NewInit] : GlobalArraysToUpgrade)
7395 GV->setInitializer(NewInit);
7398 M.addModuleFlag(
Module::Error,
"ptrauth-init-fini-address-discrimination",
7408 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7412 bool HasObjCFlag =
false, HasClassProperties =
false;
7413 bool HasClassROSigning =
false;
7414 bool HasSwiftVersionFlag =
false;
7415 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7422 if (
Op->getNumOperands() != 3)
7436 if (ID->getString() ==
"Objective-C Image Info Version")
7438 if (ID->getString() ==
"Objective-C Class Properties")
7439 HasClassProperties =
true;
7440 if (ID->getString() ==
"Objective-C Enforce ClassRO Pointer Signing")
7441 HasClassROSigning =
true;
7443 if (ID->getString() ==
"PIC Level") {
7444 if (
auto *Behavior =
7446 uint64_t V = Behavior->getLimitedValue();
7452 if (ID->getString() ==
"PIE Level")
7453 if (
auto *Behavior =
7461 if (ID->getString() ==
"branch-target-enforcement" ||
7462 (ID->getString().starts_with(
"sign-return-address") &&
7463 ID->getString() !=
"sign-return-address-harden")) {
7464 if (
auto *Behavior =
7470 Op->getOperand(1),
Op->getOperand(2)};
7480 if (ID->getString() ==
"Objective-C Image Info Section") {
7483 Value->getString().split(ValueComp,
" ");
7484 if (ValueComp.
size() != 1) {
7485 std::string NewValue;
7486 for (
auto &S : ValueComp)
7487 NewValue += S.str();
7498 if (ID->getString() ==
"Objective-C Garbage Collection") {
7501 assert(Md->getValue() &&
"Expected non-empty metadata");
7502 auto Type = Md->getValue()->getType();
7505 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7506 if ((Val & 0xff) != Val) {
7507 HasSwiftVersionFlag =
true;
7508 SwiftABIVersion = (Val & 0xff00) >> 8;
7509 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7510 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7521 if (ID->getString() ==
"amdgpu_code_object_version") {
7524 MDString::get(M.getContext(),
"amdhsa_code_object_version"),
7533 if (M.getTargetTriple().isPPC() && ID->getString() ==
"float-abi") {
7562 if (HasObjCFlag && !HasClassProperties) {
7572 if (HasObjCFlag && !HasClassROSigning) {
7574 "Objective-C Enforce ClassRO Pointer Signing", (
uint32_t)0);
7578 if (HasSwiftVersionFlag) {
7582 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7584 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7592 NamedMDNode *CFIConsts = M.getNamedMetadata(
"cfi.functions");
7596 auto MatchesVersion = [](
const MDNode *
Op) {
7597 return Op->getNumOperands() >= 3 &&
7611 assert(!MatchesVersion(
Op) &&
"Unexpected mix of CFIConstant formats");
7612 assert(
Op->getNumOperands() >= 2 &&
7613 "Expected at least 2 operands - name and linkage type");
7625 for (
unsigned J = 2, EJ =
Op->getNumOperands(); J != EJ; ++J)
7636 auto TrimSpaces = [](
StringRef Section) -> std::string {
7638 Section.split(Components,
',');
7643 for (
auto Component : Components)
7644 OS <<
',' << Component.trim();
7649 for (
auto &GV : M.globals()) {
7650 if (!GV.hasSection())
7655 if (!Section.starts_with(
"__DATA, __objc_catlist"))
7660 GV.setSection(TrimSpaces(Section));
7669 for (
auto Kind : {LLVMContext::MD_alias_scope, LLVMContext::MD_noalias}) {
7670 if (
MDNode *Old =
I.getMetadata(Kind)) {
7673 I.setMetadata(Kind, New);
7678 if (Decl->getNumOperands() > ArgNo)
7679 if (
auto *ListAsValue =
7684 Decl->setOperand(ArgNo,
7703struct StrictFPUpgradeVisitor :
public InstVisitor<StrictFPUpgradeVisitor> {
7704 StrictFPUpgradeVisitor() =
default;
7707 if (!
Call.isStrictFP())
7713 Call.removeFnAttr(Attribute::StrictFP);
7714 Call.addFnAttr(Attribute::NoBuiltin);
7719struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7720 :
public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7721 AMDGPUUnsafeFPAtomicsUpgradeVisitor() =
default;
7723 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7738 if (!
F.isDeclaration() && !
F.hasFnAttribute(Attribute::StrictFP)) {
7739 StrictFPUpgradeVisitor SFPV;
7744 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7745 F.getReturnType(),
F.getAttributes().getRetAttrs()));
7746 for (
auto &Arg :
F.args())
7748 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7750 bool AddingAttrs =
false, RemovingAttrs =
false;
7751 AttrBuilder AttrsToAdd(
F.getContext());
7756 if (
Attribute A =
F.getFnAttribute(
"implicit-section-name");
7757 A.isValid() &&
A.isStringAttribute()) {
7758 F.setSection(
A.getValueAsString());
7760 RemovingAttrs =
true;
7764 A.isValid() &&
A.isStringAttribute()) {
7767 AddingAttrs = RemovingAttrs =
true;
7770 if (
Attribute A =
F.getFnAttribute(
"uniform-work-group-size");
7771 A.isValid() &&
A.isStringAttribute() && !
A.getValueAsString().empty()) {
7773 RemovingAttrs =
true;
7774 if (
A.getValueAsString() ==
"true") {
7775 AttrsToAdd.addAttribute(
"uniform-work-group-size");
7784 if (
Attribute A =
F.getFnAttribute(
"amdgpu-unsafe-fp-atomics");
7787 if (
A.getValueAsBool()) {
7788 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7794 AttrsToRemove.
addAttribute(
"amdgpu-unsafe-fp-atomics");
7795 RemovingAttrs =
true;
7802 bool HandleDenormalMode =
false;
7804 if (
Attribute Attr =
F.getFnAttribute(
"denormal-fp-math"); Attr.isValid()) {
7807 DenormalFPMath = ParsedMode;
7809 AddingAttrs = RemovingAttrs =
true;
7810 HandleDenormalMode =
true;
7814 if (
Attribute Attr =
F.getFnAttribute(
"denormal-fp-math-f32");
7818 DenormalFPMathF32 = ParsedMode;
7820 AddingAttrs = RemovingAttrs =
true;
7821 HandleDenormalMode =
true;
7825 if (HandleDenormalMode)
7826 AttrsToAdd.addDenormalFPEnvAttr(
7830 F.removeFnAttrs(AttrsToRemove);
7833 F.addFnAttrs(AttrsToAdd);
7839 if (!
F.hasFnAttribute(FnAttrName)) {
7840 F.addFnAttr(FnAttrName,
Value);
7842 <<
"\", function: " <<
F.getName() <<
"\n");
7850 if (!
F.hasFnAttribute(FnAttrName)) {
7852 F.addFnAttr(FnAttrName);
7854 <<
", function: " <<
F.getName() <<
"\n");
7857 auto A =
F.getFnAttribute(FnAttrName);
7858 if (
"false" ==
A.getValueAsString()) {
7859 F.removeFnAttr(FnAttrName);
7861 <<
"=\"false\", function: " <<
F.getName() <<
"\n");
7862 }
else if (
"true" ==
A.getValueAsString()) {
7863 F.removeFnAttr(FnAttrName);
7864 F.addFnAttr(FnAttrName);
7866 <<
"=\"true\", function: " <<
F.getName() <<
"\n");
7873 M.setModuleFlag(Behavior,
Key, Val);
7874 LLVM_DEBUG(
dbgs() <<
"Converted module flag: " <<
"{" << Behavior <<
", "
7875 <<
Key <<
", " << Val <<
"}\n");
7879 Triple T(M.getTargetTriple());
7880 if (!
T.isThumb() && !
T.isARM() && !
T.isAArch64())
7883 uint64_t BTEValue = 0;
7884 uint64_t BPPLRValue = 0;
7885 uint64_t GCSValue = 0;
7886 uint64_t SRAValue = 0;
7887 uint64_t SRAALLValue = 0;
7888 uint64_t SRABKeyValue = 0;
7890 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7894 if (
Op->getNumOperands() != 3)
7903 uint64_t *ValPtr = IDStr ==
"branch-target-enforcement" ? &BTEValue
7904 : IDStr ==
"branch-protection-pauth-lr" ? &BPPLRValue
7905 : IDStr ==
"guarded-control-stack" ? &GCSValue
7906 : IDStr ==
"sign-return-address" ? &SRAValue
7907 : IDStr ==
"sign-return-address-all" ? &SRAALLValue
7908 : IDStr ==
"sign-return-address-with-bkey"
7914 *ValPtr = CI->getZExtValue();
7918 LLVM_DEBUG(
dbgs() <<
"Found module flag: " << IDStr <<
"(" << *ValPtr
7923 bool BTE = BTEValue == 1;
7924 bool BPPLR = BPPLRValue == 1;
7925 bool GCS = GCSValue == 1;
7926 bool SRA = SRAValue == 1;
7929 if (SRA && SRAALLValue == 1)
7930 SignTypeValue =
"all";
7933 if (SRA && SRABKeyValue == 1)
7934 SignKeyValue =
"b_key";
7936 for (
Function &
F : M.getFunctionList()) {
7937 if (
F.isDeclaration())
7944 if (
auto A =
F.getFnAttribute(
"sign-return-address");
7945 A.isValid() &&
"none" ==
A.getValueAsString()) {
7946 F.removeFnAttr(
"sign-return-address");
7947 F.removeFnAttr(
"sign-return-address-key");
7963 if (SRAALLValue == 1)
7965 if (SRABKeyValue == 1) {
7994 if (
T->getNumOperands() < 1)
7999 if (S->getString().starts_with(
"llvm.vectorizer."))
8005 StringRef OldPrefix =
"llvm.vectorizer.";
8008 if (OldTag ==
"llvm.vectorizer.unroll")
8020 if (
T->getNumOperands() < 1)
8032 if (!OldTag->getString().starts_with(
"llvm.vectorizer."))
8045 Ops.reserve(
T->getNumOperands());
8046 Ops.push_back(NewTag);
8047 for (
unsigned I = 1,
E =
T->getNumOperands();
I !=
E; ++
I)
8048 Ops.push_back(
T->getOperand(
I));
8065 if (
T->isDistinct()) {
8066 for (
unsigned I = 0, E =
T->getNumOperands();
I < E; ++
I) {
8078 Ops.reserve(
T->getNumOperands());
8089 if ((
T.isSPIR() || (
T.isSPIRV() && !
T.isSPIRVLogical())) &&
8090 !
DL.contains(
"-G") && !
DL.starts_with(
"G")) {
8091 return DL.empty() ? std::string(
"G1") : (
DL +
"-G1").str();
8094 if (
T.isLoongArch64() ||
T.isRISCV64()) {
8096 auto I =
DL.find(
"-n64-");
8098 return (
DL.take_front(
I) +
"-n32:64-" +
DL.drop_front(
I + 5)).str();
8103 std::string Res =
DL.str();
8106 if (!
DL.contains(
"-G") && !
DL.starts_with(
"G"))
8107 Res.append(Res.empty() ?
"G1" :
"-G1");
8115 if (!
DL.contains(
"-ni") && !
DL.starts_with(
"ni"))
8116 Res.append(
"-ni:7:8:9");
8118 if (
DL.ends_with(
"ni:7"))
8120 if (
DL.ends_with(
"ni:7:8"))
8125 if (!
DL.contains(
"-p7") && !
DL.starts_with(
"p7"))
8126 Res.append(
"-p7:160:256:256:32");
8127 if (!
DL.contains(
"-p8") && !
DL.starts_with(
"p8"))
8128 Res.append(
"-p8:128:128:128:48");
8129 constexpr StringRef OldP8(
"-p8:128:128-");
8130 if (
DL.contains(OldP8))
8131 Res.replace(Res.find(OldP8), OldP8.
size(),
"-p8:128:128:128:48-");
8132 if (!
DL.contains(
"-p9") && !
DL.starts_with(
"p9"))
8133 Res.append(
"-p9:192:256:256:32");
8138 for (
StringRef AS : {
"p10",
"p11",
"p12",
"p13",
"p14",
"p15"}) {
8139 if (!
DL.contains((
"-" + AS).str()) && !
DL.starts_with(AS))
8140 Res.append((
"-" + AS +
":32:32").str());
8145 if (!
DL.contains(
"m:e"))
8146 Res = Res.empty() ?
"m:e" :
"m:e-" + Res;
8151 if (
T.isSystemZ() && !
DL.empty()) {
8153 if (!
DL.contains(
"-S64"))
8154 return "E-S64" +
DL.drop_front(1).str();
8158 auto AddPtr32Ptr64AddrSpaces = [&
DL, &Res]() {
8161 StringRef AddrSpaces{
"-p270:32:32-p271:32:32-p272:64:64"};
8162 if (!
DL.contains(AddrSpaces)) {
8164 Regex R(
"^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
8165 if (R.match(Res, &
Groups))
8171 if (
T.isAArch64()) {
8173 if (!
DL.empty() && !
DL.contains(
"-Fn32"))
8174 Res.append(
"-Fn32");
8175 AddPtr32Ptr64AddrSpaces();
8179 if (
T.isSPARC() || (
T.isMIPS64() && !
DL.contains(
"m:m")) ||
T.isPPC64() ||
8183 std::string I64 =
"-i64:64";
8184 std::string I128 =
"-i128:128";
8186 size_t Pos = Res.find(I64);
8187 if (Pos !=
size_t(-1))
8188 Res.insert(Pos + I64.size(), I128);
8192 if (
T.isPPC() &&
T.isOSAIX() && !
DL.contains(
"f64:32:64") && !
DL.empty()) {
8193 size_t Pos = Res.find(
"-S128");
8196 Res.insert(Pos,
"-f64:32:64");
8201 if (
T.isARM() && !
DL.empty() && !
DL.contains(
"Fi") && !
DL.contains(
"Fn")) {
8203 size_t Pos = Res.
find(p3232);
8205 Res.insert(Pos + p3232.
size(),
"-Fi8");
8211 AddPtr32Ptr64AddrSpaces();
8219 if (!
T.isOSIAMCU()) {
8220 std::string I128 =
"-i128:128";
8223 Regex R(
"^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
8224 if (R.match(Res, &
Groups))
8232 if (
T.isWindowsMSVCEnvironment() && !
T.isArch64Bit()) {
8234 auto I =
Ref.find(
"-f80:32-");
8236 Res = (
Ref.take_front(
I) +
"-f80:128-" +
Ref.drop_front(
I + 8)).str();
8244 Attribute A =
B.getAttribute(
"no-frame-pointer-elim");
8247 FramePointer =
A.getValueAsString() ==
"true" ?
"all" :
"none";
8248 B.removeAttribute(
"no-frame-pointer-elim");
8250 if (
B.contains(
"no-frame-pointer-elim-non-leaf")) {
8252 if (FramePointer !=
"all")
8253 FramePointer =
"non-leaf";
8254 B.removeAttribute(
"no-frame-pointer-elim-non-leaf");
8256 if (!FramePointer.
empty())
8257 B.addAttribute(
"frame-pointer", FramePointer);
8259 A =
B.getAttribute(
"null-pointer-is-valid");
8262 bool NullPointerIsValid =
A.getValueAsString() ==
"true";
8263 B.removeAttribute(
"null-pointer-is-valid");
8264 if (NullPointerIsValid)
8265 B.addAttribute(Attribute::NullPointerIsValid);
8268 A =
B.getAttribute(
"uniform-work-group-size");
8272 bool IsTrue = Val ==
"true";
8273 B.removeAttribute(
"uniform-work-group-size");
8275 B.addAttribute(
"uniform-work-group-size");
8286 return OBD.
getTag() ==
"clang.arc.attachedcall" &&
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
#define G2S_ID(ID_SUFFIX, NAME)
static Metadata * upgradeLoopArgument(Metadata *MD)
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
static bool isXYZ(StringRef S)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID(&IIDs)[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu)
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static bool shouldUpgradeVPIntrinsic(StringRef Name)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
#define G2S_CTA_ID(ID_SUFFIX, NAME)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static constexpr Intrinsic::ID NVVMFMAIIDs[2][3]
static constexpr Intrinsic::ID NVVMFMulIIDs[2][3]
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static unsigned getFunctionalOpcodeForVP(StringRef Name)
static MDNode * rebuildScopeOrDomainNode(MDNode *Old, SmallVectorImpl< Metadata * > &Ops)
Rebuild the alias scope or domain Old with Ops, keeping its identity: a self reference has to point a...
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
static Intrinsic::ID shouldUpgradeNVVMBarrierMulticastIntrinsic(Function *F, StringRef Name)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static MDNode * upgradeAliasScope(MDNode *Scope, DenseMap< MDNode *, MDNode * > &Upgraded)
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static MDNode * upgradeAliasScopeDomain(MDNode *Domain, DenseMap< MDNode *, MDNode * > &Upgraded)
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
#define NVVM_TMA_G2S_MODES(M)
static Value * upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradePtrauthInitFiniArrays(Module &M)
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
static constexpr Intrinsic::ID NVVMFAddIIDs[2][3]
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
NVPTX address space definition.
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
static const X86InstrFMA3Group Groups[]
Class for arbitrary precision integers.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Class to represent array types.
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
StructType * getType() const
Specialization - reduce amount of casting.
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
A parsed version of the target data layout string in and methods for querying it.
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Base class for non-instruction debug metadata records that have positions within IR.
void setDebugLoc(DebugLoc Loc)
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Convenience struct for specifying and reasoning about fast-math flags.
void setApproxFunc(bool B=true)
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
const Function & getFunction() const
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
LinkageTypes getLinkage() const
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
This is an important class for using LLVM in a threaded context.
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
static MDTuple * getDistinct(LLVMContext &Context, ArrayRef< Metadata * > MDs)
const MDOperand & getOperand(unsigned I) const
ArrayRef< MDOperand > operands() const
op_iterator op_end() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
op_iterator op_begin() const
LLVMContext & getContext() const
Tracking metadata reference owned by Metadata.
LLVM_ABI StringRef getString() const
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
@ Min
Takes the min of the two values, which are required to be integers.
@ Max
Takes the max of the two values, which are required to be integers.
LLVM_ABI void setOperand(unsigned I, MDNode *New)
LLVM_ABI MDNode * getOperand(unsigned i) const
LLVM_ABI unsigned getNumOperands() const
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
iterator_range< op_iterator > operands()
LLVM_ABI void addOperand(MDNode *M)
ArrayRef< InputTy > inputs() const
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
ArrayRef< int > getShuffleMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
static constexpr size_t npos
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Class to represent struct types.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< user_iterator > users()
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Base class of all SIMD vector types.
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ PRIVATE_ADDRESS
Address space for private memory.
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
static const int NoAliasScopeDeclScopeArg
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
@ ADDRESS_SPACE_SHARED_CLUSTER
constexpr StringLiteral GridConstant("nvvm.grid_constant")
constexpr StringLiteral MaxNTID("nvvm.maxntid")
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
constexpr StringLiteral MinCTASm("nvvm.minctasm")
constexpr StringLiteral ReqNTID("nvvm.reqntid")
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
LLVM_ABI void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
@ Load
The value being inserted comes from a load (InsertElement only).
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
LongDoubleFormat
The floating-point format used for the target's "long double" type.
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
LLVM_ABI void UpgradeInstructionMetadata(Module &M)
Discover metadata used by instructions in this module, and apply the suitable upgrade.
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
std::string utostr(uint64_t X, bool isNeg=false)
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
auto dyn_cast_or_null(const Y &Val)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
LLVM_ABI MDNode * UpgradeTBAAStructNode(MDNode &TBAAStructNode)
If the given !tbaa.struct node has old-style scalar field tags, return an equivalent node with each f...
LLVM_ABI MDNode * UpgradeAliasScopeList(MDNode &ScopeList, DenseMap< MDNode *, MDNode * > &UpgradedAliasScopes)
Mark any domains from scopes in ScopeList that don't have a disjointness flag as non-disjoint.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
OperandBundleDefT< Value * > OperandBundleDef
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ Dynamic
Denotes mode unknown at compile time.
@ TowardPositive
roundTowardPositive.
@ TowardNegative
roundTowardNegative.
ArrayRef(const T &OneElt) -> ArrayRef< T >
const IROptions & getIROptions(const LLVMContext &C)
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getInvalid()
constexpr bool isValid() const
static constexpr DenormalMode getIEEE()
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.