30#include "llvm/IR/IntrinsicsAArch64.h"
31#include "llvm/IR/IntrinsicsARM.h"
32#include "llvm/IR/IntrinsicsNVPTX.h"
33#include "llvm/IR/IntrinsicsRISCV.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/IR/IntrinsicsX86.h"
50 cl::desc(
"Disable autoupgrade of debug info"));
60 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
75 Type *LastArgType =
F->getFunctionType()->getParamType(
76 F->getFunctionType()->getNumParams() - 1);
91 if (
F->getReturnType()->isVectorTy())
101 if (
F->getReturnType()->getScalarType()->isBFloatTy())
111 if (
F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
125 if (
Name.consume_front(
"avx."))
126 return (
Name.starts_with(
"blend.p") ||
127 Name ==
"cvt.ps2.pd.256" ||
128 Name ==
"cvtdq2.pd.256" ||
129 Name ==
"cvtdq2.ps.256" ||
130 Name.starts_with(
"movnt.") ||
131 Name.starts_with(
"sqrt.p") ||
132 Name.starts_with(
"storeu.") ||
133 Name.starts_with(
"vbroadcast.s") ||
134 Name.starts_with(
"vbroadcastf128") ||
135 Name.starts_with(
"vextractf128.") ||
136 Name.starts_with(
"vinsertf128.") ||
137 Name.starts_with(
"vperm2f128.") ||
138 Name.starts_with(
"vpermil."));
140 if (
Name.consume_front(
"avx2."))
141 return (
Name ==
"movntdqa" ||
142 Name.starts_with(
"pabs.") ||
143 Name.starts_with(
"padds.") ||
144 Name.starts_with(
"paddus.") ||
145 Name.starts_with(
"pblendd.") ||
147 Name.starts_with(
"pbroadcast") ||
148 Name.starts_with(
"pcmpeq.") ||
149 Name.starts_with(
"pcmpgt.") ||
150 Name.starts_with(
"pmax") ||
151 Name.starts_with(
"pmin") ||
152 Name.starts_with(
"pmovsx") ||
153 Name.starts_with(
"pmovzx") ||
155 Name ==
"pmulu.dq" ||
156 Name.starts_with(
"psll.dq") ||
157 Name.starts_with(
"psrl.dq") ||
158 Name.starts_with(
"psubs.") ||
159 Name.starts_with(
"psubus.") ||
160 Name.starts_with(
"vbroadcast") ||
161 Name ==
"vbroadcasti128" ||
162 Name ==
"vextracti128" ||
163 Name ==
"vinserti128" ||
164 Name ==
"vperm2i128");
166 if (
Name.consume_front(
"avx512.")) {
167 if (
Name.consume_front(
"mask."))
169 return (
Name.starts_with(
"add.p") ||
170 Name.starts_with(
"and.") ||
171 Name.starts_with(
"andn.") ||
172 Name.starts_with(
"broadcast.s") ||
173 Name.starts_with(
"broadcastf32x4.") ||
174 Name.starts_with(
"broadcastf32x8.") ||
175 Name.starts_with(
"broadcastf64x2.") ||
176 Name.starts_with(
"broadcastf64x4.") ||
177 Name.starts_with(
"broadcasti32x4.") ||
178 Name.starts_with(
"broadcasti32x8.") ||
179 Name.starts_with(
"broadcasti64x2.") ||
180 Name.starts_with(
"broadcasti64x4.") ||
181 Name.starts_with(
"cmp.b") ||
182 Name.starts_with(
"cmp.d") ||
183 Name.starts_with(
"cmp.q") ||
184 Name.starts_with(
"cmp.w") ||
185 Name.starts_with(
"compress.b") ||
186 Name.starts_with(
"compress.d") ||
187 Name.starts_with(
"compress.p") ||
188 Name.starts_with(
"compress.q") ||
189 Name.starts_with(
"compress.store.") ||
190 Name.starts_with(
"compress.w") ||
191 Name.starts_with(
"conflict.") ||
192 Name.starts_with(
"cvtdq2pd.") ||
193 Name.starts_with(
"cvtdq2ps.") ||
194 Name ==
"cvtpd2dq.256" ||
195 Name ==
"cvtpd2ps.256" ||
196 Name ==
"cvtps2pd.128" ||
197 Name ==
"cvtps2pd.256" ||
198 Name.starts_with(
"cvtqq2pd.") ||
199 Name ==
"cvtqq2ps.256" ||
200 Name ==
"cvtqq2ps.512" ||
201 Name ==
"cvttpd2dq.256" ||
202 Name ==
"cvttps2dq.128" ||
203 Name ==
"cvttps2dq.256" ||
204 Name.starts_with(
"cvtudq2pd.") ||
205 Name.starts_with(
"cvtudq2ps.") ||
206 Name.starts_with(
"cvtuqq2pd.") ||
207 Name ==
"cvtuqq2ps.256" ||
208 Name ==
"cvtuqq2ps.512" ||
209 Name.starts_with(
"dbpsadbw.") ||
210 Name.starts_with(
"div.p") ||
211 Name.starts_with(
"expand.b") ||
212 Name.starts_with(
"expand.d") ||
213 Name.starts_with(
"expand.load.") ||
214 Name.starts_with(
"expand.p") ||
215 Name.starts_with(
"expand.q") ||
216 Name.starts_with(
"expand.w") ||
217 Name.starts_with(
"fpclass.p") ||
218 Name.starts_with(
"insert") ||
219 Name.starts_with(
"load.") ||
220 Name.starts_with(
"loadu.") ||
221 Name.starts_with(
"lzcnt.") ||
222 Name.starts_with(
"max.p") ||
223 Name.starts_with(
"min.p") ||
224 Name.starts_with(
"movddup") ||
225 Name.starts_with(
"move.s") ||
226 Name.starts_with(
"movshdup") ||
227 Name.starts_with(
"movsldup") ||
228 Name.starts_with(
"mul.p") ||
229 Name.starts_with(
"or.") ||
230 Name.starts_with(
"pabs.") ||
231 Name.starts_with(
"packssdw.") ||
232 Name.starts_with(
"packsswb.") ||
233 Name.starts_with(
"packusdw.") ||
234 Name.starts_with(
"packuswb.") ||
235 Name.starts_with(
"padd.") ||
236 Name.starts_with(
"padds.") ||
237 Name.starts_with(
"paddus.") ||
238 Name.starts_with(
"palignr.") ||
239 Name.starts_with(
"pand.") ||
240 Name.starts_with(
"pandn.") ||
241 Name.starts_with(
"pavg") ||
242 Name.starts_with(
"pbroadcast") ||
243 Name.starts_with(
"pcmpeq.") ||
244 Name.starts_with(
"pcmpgt.") ||
245 Name.starts_with(
"perm.df.") ||
246 Name.starts_with(
"perm.di.") ||
247 Name.starts_with(
"permvar.") ||
248 Name.starts_with(
"pmaddubs.w.") ||
249 Name.starts_with(
"pmaddw.d.") ||
250 Name.starts_with(
"pmax") ||
251 Name.starts_with(
"pmin") ||
252 Name ==
"pmov.qd.256" ||
253 Name ==
"pmov.qd.512" ||
254 Name ==
"pmov.wb.256" ||
255 Name ==
"pmov.wb.512" ||
256 Name.starts_with(
"pmovsx") ||
257 Name.starts_with(
"pmovzx") ||
258 Name.starts_with(
"pmul.dq.") ||
259 Name.starts_with(
"pmul.hr.sw.") ||
260 Name.starts_with(
"pmulh.w.") ||
261 Name.starts_with(
"pmulhu.w.") ||
262 Name.starts_with(
"pmull.") ||
263 Name.starts_with(
"pmultishift.qb.") ||
264 Name.starts_with(
"pmulu.dq.") ||
265 Name.starts_with(
"por.") ||
266 Name.starts_with(
"prol.") ||
267 Name.starts_with(
"prolv.") ||
268 Name.starts_with(
"pror.") ||
269 Name.starts_with(
"prorv.") ||
270 Name.starts_with(
"pshuf.b.") ||
271 Name.starts_with(
"pshuf.d.") ||
272 Name.starts_with(
"pshufh.w.") ||
273 Name.starts_with(
"pshufl.w.") ||
274 Name.starts_with(
"psll.d") ||
275 Name.starts_with(
"psll.q") ||
276 Name.starts_with(
"psll.w") ||
277 Name.starts_with(
"pslli") ||
278 Name.starts_with(
"psllv") ||
279 Name.starts_with(
"psra.d") ||
280 Name.starts_with(
"psra.q") ||
281 Name.starts_with(
"psra.w") ||
282 Name.starts_with(
"psrai") ||
283 Name.starts_with(
"psrav") ||
284 Name.starts_with(
"psrl.d") ||
285 Name.starts_with(
"psrl.q") ||
286 Name.starts_with(
"psrl.w") ||
287 Name.starts_with(
"psrli") ||
288 Name.starts_with(
"psrlv") ||
289 Name.starts_with(
"psub.") ||
290 Name.starts_with(
"psubs.") ||
291 Name.starts_with(
"psubus.") ||
292 Name.starts_with(
"pternlog.") ||
293 Name.starts_with(
"punpckh") ||
294 Name.starts_with(
"punpckl") ||
295 Name.starts_with(
"pxor.") ||
296 Name.starts_with(
"shuf.f") ||
297 Name.starts_with(
"shuf.i") ||
298 Name.starts_with(
"shuf.p") ||
299 Name.starts_with(
"sqrt.p") ||
300 Name.starts_with(
"store.b.") ||
301 Name.starts_with(
"store.d.") ||
302 Name.starts_with(
"store.p") ||
303 Name.starts_with(
"store.q.") ||
304 Name.starts_with(
"store.w.") ||
305 Name ==
"store.ss" ||
306 Name.starts_with(
"storeu.") ||
307 Name.starts_with(
"sub.p") ||
308 Name.starts_with(
"ucmp.") ||
309 Name.starts_with(
"unpckh.") ||
310 Name.starts_with(
"unpckl.") ||
311 Name.starts_with(
"valign.") ||
312 Name ==
"vcvtph2ps.128" ||
313 Name ==
"vcvtph2ps.256" ||
314 Name.starts_with(
"vextract") ||
315 Name.starts_with(
"vfmadd.") ||
316 Name.starts_with(
"vfmaddsub.") ||
317 Name.starts_with(
"vfnmadd.") ||
318 Name.starts_with(
"vfnmsub.") ||
319 Name.starts_with(
"vpdpbusd.") ||
320 Name.starts_with(
"vpdpbusds.") ||
321 Name.starts_with(
"vpdpwssd.") ||
322 Name.starts_with(
"vpdpwssds.") ||
323 Name.starts_with(
"vpermi2var.") ||
324 Name.starts_with(
"vpermil.p") ||
325 Name.starts_with(
"vpermilvar.") ||
326 Name.starts_with(
"vpermt2var.") ||
327 Name.starts_with(
"vpmadd52") ||
328 Name.starts_with(
"vpshld.") ||
329 Name.starts_with(
"vpshldv.") ||
330 Name.starts_with(
"vpshrd.") ||
331 Name.starts_with(
"vpshrdv.") ||
332 Name.starts_with(
"vpshufbitqmb.") ||
333 Name.starts_with(
"xor."));
335 if (
Name.consume_front(
"mask3."))
337 return (
Name.starts_with(
"vfmadd.") ||
338 Name.starts_with(
"vfmaddsub.") ||
339 Name.starts_with(
"vfmsub.") ||
340 Name.starts_with(
"vfmsubadd.") ||
341 Name.starts_with(
"vfnmsub."));
343 if (
Name.consume_front(
"maskz."))
345 return (
Name.starts_with(
"pternlog.") ||
346 Name.starts_with(
"vfmadd.") ||
347 Name.starts_with(
"vfmaddsub.") ||
348 Name.starts_with(
"vpdpbusd.") ||
349 Name.starts_with(
"vpdpbusds.") ||
350 Name.starts_with(
"vpdpwssd.") ||
351 Name.starts_with(
"vpdpwssds.") ||
352 Name.starts_with(
"vpermt2var.") ||
353 Name.starts_with(
"vpmadd52") ||
354 Name.starts_with(
"vpshldv.") ||
355 Name.starts_with(
"vpshrdv."));
358 return (
Name ==
"movntdqa" ||
359 Name ==
"pmul.dq.512" ||
360 Name ==
"pmulu.dq.512" ||
361 Name.starts_with(
"broadcastm") ||
362 Name.starts_with(
"cmp.p") ||
363 Name.starts_with(
"cvtb2mask.") ||
364 Name.starts_with(
"cvtd2mask.") ||
365 Name.starts_with(
"cvtmask2") ||
366 Name.starts_with(
"cvtq2mask.") ||
367 Name ==
"cvtusi2sd" ||
368 Name.starts_with(
"cvtw2mask.") ||
373 Name ==
"kortestc.w" ||
374 Name ==
"kortestz.w" ||
375 Name.starts_with(
"kunpck") ||
378 Name.starts_with(
"padds.") ||
379 Name.starts_with(
"pbroadcast") ||
380 Name.starts_with(
"prol") ||
381 Name.starts_with(
"pror") ||
382 Name.starts_with(
"psll.dq") ||
383 Name.starts_with(
"psrl.dq") ||
384 Name.starts_with(
"psubs.") ||
385 Name.starts_with(
"ptestm") ||
386 Name.starts_with(
"ptestnm") ||
387 Name.starts_with(
"storent.") ||
388 Name.starts_with(
"vbroadcast.s") ||
389 Name.starts_with(
"vpshld.") ||
390 Name.starts_with(
"vpshrd."));
393 if (
Name.consume_front(
"fma."))
394 return (
Name.starts_with(
"vfmadd.") ||
395 Name.starts_with(
"vfmsub.") ||
396 Name.starts_with(
"vfmsubadd.") ||
397 Name.starts_with(
"vfnmadd.") ||
398 Name.starts_with(
"vfnmsub."));
400 if (
Name.consume_front(
"fma4."))
401 return Name.starts_with(
"vfmadd.s");
403 if (
Name.consume_front(
"sse."))
404 return (
Name ==
"add.ss" ||
405 Name ==
"cvtsi2ss" ||
406 Name ==
"cvtsi642ss" ||
409 Name.starts_with(
"sqrt.p") ||
411 Name.starts_with(
"storeu.") ||
414 if (
Name.consume_front(
"sse2."))
415 return (
Name ==
"add.sd" ||
416 Name ==
"cvtdq2pd" ||
417 Name ==
"cvtdq2ps" ||
418 Name ==
"cvtps2pd" ||
419 Name ==
"cvtsi2sd" ||
420 Name ==
"cvtsi642sd" ||
421 Name ==
"cvtss2sd" ||
424 Name.starts_with(
"padds.") ||
425 Name.starts_with(
"paddus.") ||
426 Name.starts_with(
"pcmpeq.") ||
427 Name.starts_with(
"pcmpgt.") ||
432 Name ==
"pmulu.dq" ||
433 Name.starts_with(
"pshuf") ||
434 Name.starts_with(
"psll.dq") ||
435 Name.starts_with(
"psrl.dq") ||
436 Name.starts_with(
"psubs.") ||
437 Name.starts_with(
"psubus.") ||
438 Name.starts_with(
"sqrt.p") ||
440 Name ==
"storel.dq" ||
441 Name.starts_with(
"storeu.") ||
444 if (
Name.consume_front(
"sse41."))
445 return (
Name.starts_with(
"blendp") ||
446 Name ==
"movntdqa" ||
456 Name.starts_with(
"pmovsx") ||
457 Name.starts_with(
"pmovzx") ||
460 if (
Name.consume_front(
"sse42."))
461 return Name ==
"crc32.64.8";
463 if (
Name.consume_front(
"sse4a."))
464 return Name.starts_with(
"movnt.");
466 if (
Name.consume_front(
"ssse3."))
467 return (
Name ==
"pabs.b.128" ||
468 Name ==
"pabs.d.128" ||
469 Name ==
"pabs.w.128");
471 if (
Name.consume_front(
"xop."))
472 return (
Name ==
"vpcmov" ||
473 Name ==
"vpcmov.256" ||
474 Name.starts_with(
"vpcom") ||
475 Name.starts_with(
"vprot"));
477 return (
Name ==
"addcarry.u32" ||
478 Name ==
"addcarry.u64" ||
479 Name ==
"addcarryx.u32" ||
480 Name ==
"addcarryx.u64" ||
481 Name ==
"subborrow.u32" ||
482 Name ==
"subborrow.u64" ||
483 Name.starts_with(
"vcvtph2ps."));
489 if (!
Name.consume_front(
"x86."))
497 if (
Name ==
"rdtscp") {
499 if (
F->getFunctionType()->getNumParams() == 0)
504 Intrinsic::x86_rdtscp);
511 if (
Name.consume_front(
"sse41.ptest")) {
513 .
Case(
"c", Intrinsic::x86_sse41_ptestc)
514 .
Case(
"z", Intrinsic::x86_sse41_ptestz)
515 .
Case(
"nzc", Intrinsic::x86_sse41_ptestnzc)
528 .
Case(
"sse41.insertps", Intrinsic::x86_sse41_insertps)
529 .
Case(
"sse41.dppd", Intrinsic::x86_sse41_dppd)
530 .
Case(
"sse41.dpps", Intrinsic::x86_sse41_dpps)
531 .
Case(
"sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
532 .
Case(
"avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
533 .
Case(
"avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
538 if (
Name.consume_front(
"avx512.mask.cmp.")) {
541 .
Case(
"pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
542 .
Case(
"pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
543 .
Case(
"pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
544 .
Case(
"ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
545 .
Case(
"ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
546 .
Case(
"ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
553 if (
Name.consume_front(
"avx512bf16.")) {
556 .
Case(
"cvtne2ps2bf16.128",
557 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
558 .
Case(
"cvtne2ps2bf16.256",
559 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
560 .
Case(
"cvtne2ps2bf16.512",
561 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
562 .
Case(
"mask.cvtneps2bf16.128",
563 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
564 .
Case(
"cvtneps2bf16.256",
565 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
566 .
Case(
"cvtneps2bf16.512",
567 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
574 .
Case(
"dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
575 .
Case(
"dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
576 .
Case(
"dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
583 if (
Name.consume_front(
"xop.")) {
585 if (
Name.starts_with(
"vpermil2")) {
588 auto Idx =
F->getFunctionType()->getParamType(2);
589 if (
Idx->isFPOrFPVectorTy()) {
590 unsigned IdxSize =
Idx->getPrimitiveSizeInBits();
591 unsigned EltSize =
Idx->getScalarSizeInBits();
592 if (EltSize == 64 && IdxSize == 128)
593 ID = Intrinsic::x86_xop_vpermil2pd;
594 else if (EltSize == 32 && IdxSize == 128)
595 ID = Intrinsic::x86_xop_vpermil2ps;
596 else if (EltSize == 64 && IdxSize == 256)
597 ID = Intrinsic::x86_xop_vpermil2pd_256;
599 ID = Intrinsic::x86_xop_vpermil2ps_256;
601 }
else if (
F->arg_size() == 2)
604 .
Case(
"vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
605 .
Case(
"vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
616 if (
Name ==
"seh.recoverfp") {
629 if (
Name.starts_with(
"rbit")) {
632 F->arg_begin()->getType());
636 if (
Name ==
"thread.pointer") {
643 bool Neon =
Name.consume_front(
"neon.");
648 if (
Name.consume_front(
"bfdot.")) {
652 .
Cases(
"v2f32.v8i8",
"v4f32.v16i8",
657 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
658 assert((OperandWidth == 64 || OperandWidth == 128) &&
659 "Unexpected operand width");
661 std::array<Type *, 2> Tys{
672 if (
Name.consume_front(
"bfm")) {
674 if (
Name.consume_back(
".v4f32.v16i8")) {
714 F->arg_begin()->getType());
718 if (
Name.consume_front(
"vst")) {
720 static const Regex vstRegex(
"^([1234]|[234]lane)\\.v[a-z0-9]*$");
724 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
725 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
728 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
729 Intrinsic::arm_neon_vst4lane};
731 auto fArgs =
F->getFunctionType()->params();
732 Type *Tys[] = {fArgs[0], fArgs[1]};
735 StoreInts[fArgs.size() - 3], Tys);
738 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
747 if (
Name.consume_front(
"mve.")) {
749 if (
Name ==
"vctp64") {
750 if (cast<FixedVectorType>(
F->getReturnType())->getNumElements() == 4) {
760 if (
Name.consume_back(
".v4i1")) {
762 if (
Name.consume_back(
".predicated.v2i64.v4i32"))
764 return Name ==
"mull.int" ||
Name ==
"vqdmull";
766 if (
Name.consume_back(
".v2i64")) {
768 bool IsGather =
Name.consume_front(
"vldr.gather.");
769 if (IsGather ||
Name.consume_front(
"vstr.scatter.")) {
770 if (
Name.consume_front(
"base.")) {
772 Name.consume_front(
"wb.");
775 return Name ==
"predicated.v2i64";
778 if (
Name.consume_front(
"offset.predicated."))
779 return Name == (IsGather ?
"v2i64.p0i64" :
"p0i64.v2i64") ||
780 Name == (IsGather ?
"v2i64.p0" :
"p0.v2i64");
793 if (
Name.consume_front(
"cde.vcx")) {
795 if (
Name.consume_back(
".predicated.v2i64.v4i1"))
812 F->arg_begin()->getType());
816 if (
Name.starts_with(
"addp")) {
818 if (
F->arg_size() != 2)
820 VectorType *Ty = dyn_cast<VectorType>(
F->getReturnType());
821 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
823 Intrinsic::aarch64_neon_faddp, Ty);
829 if (
Name.consume_front(
"sve.")) {
831 if (
Name.consume_front(
"bf")) {
832 if (
Name.consume_back(
".lane")) {
836 .
Case(
"dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
837 .
Case(
"mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
838 .
Case(
"mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
849 if (
Name.consume_front(
"addqv")) {
851 if (!
F->getReturnType()->isFPOrFPVectorTy())
854 auto Args =
F->getFunctionType()->params();
855 Type *Tys[] = {
F->getReturnType(), Args[1]};
857 Intrinsic::aarch64_sve_faddqv, Tys);
861 if (
Name.consume_front(
"ld")) {
863 static const Regex LdRegex(
"^[234](.nxv[a-z0-9]+|$)");
866 cast<VectorType>(
F->getReturnType())->getElementType();
868 cast<VectorType>(
F->arg_begin()->getType())->getElementCount();
869 Type *Ty = VectorType::get(ScalarTy, EC);
871 Intrinsic::aarch64_sve_ld2_sret,
872 Intrinsic::aarch64_sve_ld3_sret,
873 Intrinsic::aarch64_sve_ld4_sret,
876 LoadIDs[
Name[0] -
'2'], Ty);
882 if (
Name.consume_front(
"tuple.")) {
884 if (
Name.starts_with(
"get")) {
886 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
888 Intrinsic::vector_extract, Tys);
892 if (
Name.starts_with(
"set")) {
894 auto Args =
F->getFunctionType()->params();
895 Type *Tys[] = {Args[0], Args[2], Args[1]};
897 Intrinsic::vector_insert, Tys);
901 static const Regex CreateTupleRegex(
"^create[234](.nxv[a-z0-9]+|$)");
904 auto Args =
F->getFunctionType()->params();
905 Type *Tys[] = {
F->getReturnType(), Args[1]};
907 Intrinsic::vector_insert, Tys);
919 if (
Name.consume_front(
"abs."))
921 .
Case(
"bf16", Intrinsic::nvvm_abs_bf16)
922 .
Case(
"bf16x2", Intrinsic::nvvm_abs_bf16x2)
925 if (
Name.consume_front(
"fma.rn."))
927 .
Case(
"bf16", Intrinsic::nvvm_fma_rn_bf16)
928 .
Case(
"bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
929 .
Case(
"ftz.bf16", Intrinsic::nvvm_fma_rn_ftz_bf16)
930 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fma_rn_ftz_bf16x2)
931 .
Case(
"ftz.relu.bf16", Intrinsic::nvvm_fma_rn_ftz_relu_bf16)
932 .
Case(
"ftz.relu.bf16x2", Intrinsic::nvvm_fma_rn_ftz_relu_bf16x2)
933 .
Case(
"ftz.sat.bf16", Intrinsic::nvvm_fma_rn_ftz_sat_bf16)
934 .
Case(
"ftz.sat.bf16x2", Intrinsic::nvvm_fma_rn_ftz_sat_bf16x2)
935 .
Case(
"relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
936 .
Case(
"relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
937 .
Case(
"sat.bf16", Intrinsic::nvvm_fma_rn_sat_bf16)
938 .
Case(
"sat.bf16x2", Intrinsic::nvvm_fma_rn_sat_bf16x2)
941 if (
Name.consume_front(
"fmax."))
943 .
Case(
"bf16", Intrinsic::nvvm_fmax_bf16)
944 .
Case(
"bf16x2", Intrinsic::nvvm_fmax_bf16x2)
945 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
946 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
947 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
948 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
949 .
Case(
"ftz.nan.xorsign.abs.bf16",
950 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
951 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
952 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
953 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
954 .
Case(
"ftz.xorsign.abs.bf16x2",
955 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
956 .
Case(
"nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
957 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
958 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
959 .
Case(
"nan.xorsign.abs.bf16x2",
960 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
961 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
962 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
965 if (
Name.consume_front(
"fmin."))
967 .
Case(
"bf16", Intrinsic::nvvm_fmin_bf16)
968 .
Case(
"bf16x2", Intrinsic::nvvm_fmin_bf16x2)
969 .
Case(
"ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
970 .
Case(
"ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
971 .
Case(
"ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
972 .
Case(
"ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
973 .
Case(
"ftz.nan.xorsign.abs.bf16",
974 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
975 .
Case(
"ftz.nan.xorsign.abs.bf16x2",
976 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
977 .
Case(
"ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
978 .
Case(
"ftz.xorsign.abs.bf16x2",
979 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
980 .
Case(
"nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
981 .
Case(
"nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
982 .
Case(
"nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
983 .
Case(
"nan.xorsign.abs.bf16x2",
984 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
985 .
Case(
"xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
986 .
Case(
"xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
989 if (
Name.consume_front(
"neg."))
991 .
Case(
"bf16", Intrinsic::nvvm_neg_bf16)
992 .
Case(
"bf16x2", Intrinsic::nvvm_neg_bf16x2)
999 bool CanUpgradeDebugIntrinsicsToRecords) {
1000 assert(
F &&
"Illegal to upgrade a non-existent Function.");
1005 if (!
Name.consume_front(
"llvm.") ||
Name.empty())
1011 bool IsArm =
Name.consume_front(
"arm.");
1012 if (IsArm ||
Name.consume_front(
"aarch64.")) {
1018 if (
Name.consume_front(
"amdgcn.")) {
1019 if (
Name ==
"alignbit") {
1022 {F->getReturnType()});
1026 if (
Name.consume_front(
"atomic.")) {
1027 if (
Name.starts_with(
"inc") ||
Name.starts_with(
"dec")) {
1036 if (
Name.starts_with(
"ds.fadd") ||
Name.starts_with(
"ds.fmin") ||
1037 Name.starts_with(
"ds.fmax")) {
1044 if (
Name.starts_with(
"ldexp.")) {
1047 F->getParent(), Intrinsic::ldexp,
1048 {F->getReturnType(), F->getArg(1)->getType()});
1057 if (
F->arg_size() == 1) {
1065 F->arg_begin()->getType());
1070 if (
F->arg_size() == 2 &&
Name ==
"coro.end") {
1079 if (
Name.consume_front(
"dbg.")) {
1081 if (CanUpgradeDebugIntrinsicsToRecords &&
1082 F->getParent()->IsNewDbgInfoFormat) {
1083 if (
Name ==
"addr" ||
Name ==
"value" ||
Name ==
"assign" ||
1084 Name ==
"declare" ||
Name ==
"label") {
1093 if (
Name ==
"addr" || (
Name ==
"value" &&
F->arg_size() == 4)) {
1102 if (
Name.consume_front(
"experimental.vector.")) {
1105 .
StartsWith(
"extract.", Intrinsic::vector_extract)
1106 .
StartsWith(
"insert.", Intrinsic::vector_insert)
1107 .
StartsWith(
"splice.", Intrinsic::vector_splice)
1108 .
StartsWith(
"reverse.", Intrinsic::vector_reverse)
1109 .
StartsWith(
"interleave2.", Intrinsic::vector_interleave2)
1110 .
StartsWith(
"deinterleave2.", Intrinsic::vector_deinterleave2)
1113 const auto *FT =
F->getFunctionType();
1115 if (
ID == Intrinsic::vector_extract ||
1116 ID == Intrinsic::vector_interleave2)
1119 if (
ID != Intrinsic::vector_interleave2)
1121 if (
ID == Intrinsic::vector_insert)
1129 if (
Name.consume_front(
"reduce.")) {
1131 static const Regex R(
"^([a-z]+)\\.[a-z][0-9]+");
1134 .
Case(
"add", Intrinsic::vector_reduce_add)
1135 .
Case(
"mul", Intrinsic::vector_reduce_mul)
1136 .
Case(
"and", Intrinsic::vector_reduce_and)
1137 .
Case(
"or", Intrinsic::vector_reduce_or)
1138 .
Case(
"xor", Intrinsic::vector_reduce_xor)
1139 .
Case(
"smax", Intrinsic::vector_reduce_smax)
1140 .
Case(
"smin", Intrinsic::vector_reduce_smin)
1141 .
Case(
"umax", Intrinsic::vector_reduce_umax)
1142 .
Case(
"umin", Intrinsic::vector_reduce_umin)
1143 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
1144 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
1149 static const Regex R2(
"^v2\\.([a-z]+)\\.[fi][0-9]+");
1154 .
Case(
"fadd", Intrinsic::vector_reduce_fadd)
1155 .
Case(
"fmul", Intrinsic::vector_reduce_fmul)
1160 auto Args =
F->getFunctionType()->params();
1171 if (
Name.starts_with(
"flt.rounds")) {
1178 if (
Name.starts_with(
"invariant.group.barrier")) {
1180 auto Args =
F->getFunctionType()->params();
1181 Type* ObjectPtr[1] = {Args[0]};
1184 Intrinsic::launder_invariant_group, ObjectPtr);
1193 .StartsWith(
"memcpy.", Intrinsic::memcpy)
1194 .StartsWith(
"memmove.", Intrinsic::memmove)
1196 if (
F->arg_size() == 5) {
1200 F->getFunctionType()->params().slice(0, 3);
1205 if (
Name.starts_with(
"memset.") &&
F->arg_size() == 5) {
1208 const auto *FT =
F->getFunctionType();
1209 Type *ParamTypes[2] = {
1210 FT->getParamType(0),
1220 if (
Name.consume_front(
"nvvm.")) {
1222 if (
F->arg_size() == 1) {
1225 .
Cases(
"brev32",
"brev64", Intrinsic::bitreverse)
1226 .
Case(
"clz.i", Intrinsic::ctlz)
1227 .
Case(
"popc.i", Intrinsic::ctpop)
1231 {F->getReturnType()});
1237 if (!
F->getReturnType()->getScalarType()->isBFloatTy()) {
1249 bool Expand =
false;
1250 if (
Name.consume_front(
"abs."))
1252 Expand =
Name ==
"i" ||
Name ==
"ll";
1253 else if (
Name ==
"clz.ll" ||
Name ==
"popc.ll" ||
Name ==
"h2f")
1255 else if (
Name.consume_front(
"max.") ||
Name.consume_front(
"min."))
1259 else if (
Name.consume_front(
"atomic.load.add."))
1261 Expand =
Name.starts_with(
"f32.p") ||
Name.starts_with(
"f64.p");
1276 if (
Name.starts_with(
"objectsize.")) {
1277 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
1278 if (
F->arg_size() == 2 ||
F->arg_size() == 3 ||
1290 if (
Name.starts_with(
"ptr.annotation.") &&
F->arg_size() == 4) {
1293 F->getParent(), Intrinsic::ptr_annotation,
1294 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1300 if (
Name.consume_front(
"riscv.")) {
1303 .
Case(
"aes32dsi", Intrinsic::riscv_aes32dsi)
1304 .
Case(
"aes32dsmi", Intrinsic::riscv_aes32dsmi)
1305 .
Case(
"aes32esi", Intrinsic::riscv_aes32esi)
1306 .
Case(
"aes32esmi", Intrinsic::riscv_aes32esmi)
1309 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1322 if (!
F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1323 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1332 .
StartsWith(
"sha256sig0", Intrinsic::riscv_sha256sig0)
1333 .
StartsWith(
"sha256sig1", Intrinsic::riscv_sha256sig1)
1334 .
StartsWith(
"sha256sum0", Intrinsic::riscv_sha256sum0)
1335 .
StartsWith(
"sha256sum1", Intrinsic::riscv_sha256sum1)
1340 if (
F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1352 if (
Name ==
"stackprotectorcheck") {
1359 if (
Name ==
"var.annotation" &&
F->arg_size() == 4) {
1362 F->getParent(), Intrinsic::var_annotation,
1363 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1370 if (
Name.consume_front(
"wasm.")) {
1373 .
StartsWith(
"fma.", Intrinsic::wasm_relaxed_madd)
1374 .
StartsWith(
"fms.", Intrinsic::wasm_relaxed_nmadd)
1375 .
StartsWith(
"laneselect.", Intrinsic::wasm_relaxed_laneselect)
1384 if (
Name.consume_front(
"dot.i8x16.i7x16.")) {
1386 .
Case(
"signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1388 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1406 auto *
ST = dyn_cast<StructType>(
F->getReturnType());
1407 if (ST && (!
ST->isLiteral() ||
ST->isPacked()) &&
1416 auto *FT =
F->getFunctionType();
1418 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1419 std::string
Name =
F->getName().str();
1422 Name,
F->getParent());
1433 if (Result != std::nullopt) {
1446 bool CanUpgradeDebugIntrinsicsToRecords) {
1450 assert(
F != NewFn &&
"Intrinsic function upgraded to the same function");
1462 GV->
getName() ==
"llvm.global_dtors")) ||
1477 unsigned N =
Init->getNumOperands();
1478 std::vector<Constant *> NewCtors(
N);
1479 for (
unsigned i = 0; i !=
N; ++i) {
1480 auto Ctor = cast<Constant>(
Init->getOperand(i));
1482 Ctor->getAggregateElement(1),
1495 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1496 unsigned NumElts = ResultTy->getNumElements() * 8;
1510 for (
unsigned l = 0; l != NumElts; l += 16)
1511 for (
unsigned i = 0; i != 16; ++i) {
1512 unsigned Idx = NumElts + i - Shift;
1514 Idx -= NumElts - 16;
1515 Idxs[l + i] =
Idx + l;
1529 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1530 unsigned NumElts = ResultTy->getNumElements() * 8;
1544 for (
unsigned l = 0; l != NumElts; l += 16)
1545 for (
unsigned i = 0; i != 16; ++i) {
1546 unsigned Idx = i + Shift;
1548 Idx += NumElts - 16;
1549 Idxs[l + i] =
Idx + l;
1570 for (
unsigned i = 0; i != NumElts; ++i)
1582 if (
const auto *
C = dyn_cast<Constant>(Mask))
1583 if (
C->isAllOnesValue())
1587 cast<FixedVectorType>(Op0->
getType())->getNumElements());
1594 if (
const auto *
C = dyn_cast<Constant>(Mask))
1595 if (
C->isAllOnesValue())
1599 Mask->getType()->getIntegerBitWidth());
1612 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1614 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1615 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
1616 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
1621 ShiftVal &= (NumElts - 1);
1630 if (ShiftVal > 16) {
1638 for (
unsigned l = 0; l < NumElts; l += 16) {
1639 for (
unsigned i = 0; i != 16; ++i) {
1640 unsigned Idx = ShiftVal + i;
1641 if (!IsVALIGN &&
Idx >= 16)
1642 Idx += NumElts - 16;
1643 Indices[l + i] =
Idx + l;
1648 Op1, Op0,
ArrayRef(Indices, NumElts),
"palignr");
1654 bool ZeroMask,
bool IndexForm) {
1660 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1661 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1662 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1663 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1664 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1665 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1666 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1667 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1668 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1669 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1670 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1671 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1672 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1673 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1674 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1675 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1676 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1677 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1678 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1679 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1680 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1681 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1682 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1683 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1684 else if (VecWidth == 128 && EltWidth == 16)
1685 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1686 else if (VecWidth == 256 && EltWidth == 16)
1687 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1688 else if (VecWidth == 512 && EltWidth == 16)
1689 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1690 else if (VecWidth == 128 && EltWidth == 8)
1691 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1692 else if (VecWidth == 256 && EltWidth == 8)
1693 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1694 else if (VecWidth == 512 && EltWidth == 8)
1695 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1731 bool IsRotateRight) {
1740 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1745 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1766 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1769 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1772 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1775 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1778 Pred = ICmpInst::ICMP_EQ;
1781 Pred = ICmpInst::ICMP_NE;
1797 bool IsShiftRight,
bool ZeroMask) {
1810 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1815 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1835 const Align Alignment =
1837 ?
Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
1841 if (
const auto *
C = dyn_cast<Constant>(Mask))
1842 if (
C->isAllOnesValue())
1846 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1856 const Align Alignment =
1864 if (
const auto *
C = dyn_cast<Constant>(Mask))
1865 if (
C->isAllOnesValue())
1869 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1893 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1900 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1916 unsigned NumElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
1918 const auto *
C = dyn_cast<Constant>(Mask);
1919 if (!
C || !
C->isAllOnesValue())
1925 for (
unsigned i = 0; i != NumElts; ++i)
1927 for (
unsigned i = NumElts; i != 8; ++i)
1928 Indices[i] = NumElts + i % NumElts;
1939 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1945 }
else if (
CC == 7) {
1952 case 0: Pred = ICmpInst::ICMP_EQ;
break;
1953 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
1954 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
1955 case 4: Pred = ICmpInst::ICMP_NE;
break;
1956 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
1957 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
1993 unsigned NumElts = cast<FixedVectorType>(CI.
getType())->getNumElements();
1995 return Builder.
CreateSExt(Mask, ReturnOp,
"vpmovm2");
2006 if (
Name.starts_with(
"max.p")) {
2007 if (VecWidth == 128 && EltWidth == 32)
2008 IID = Intrinsic::x86_sse_max_ps;
2009 else if (VecWidth == 128 && EltWidth == 64)
2010 IID = Intrinsic::x86_sse2_max_pd;
2011 else if (VecWidth == 256 && EltWidth == 32)
2012 IID = Intrinsic::x86_avx_max_ps_256;
2013 else if (VecWidth == 256 && EltWidth == 64)
2014 IID = Intrinsic::x86_avx_max_pd_256;
2017 }
else if (
Name.starts_with(
"min.p")) {
2018 if (VecWidth == 128 && EltWidth == 32)
2019 IID = Intrinsic::x86_sse_min_ps;
2020 else if (VecWidth == 128 && EltWidth == 64)
2021 IID = Intrinsic::x86_sse2_min_pd;
2022 else if (VecWidth == 256 && EltWidth == 32)
2023 IID = Intrinsic::x86_avx_min_ps_256;
2024 else if (VecWidth == 256 && EltWidth == 64)
2025 IID = Intrinsic::x86_avx_min_pd_256;
2028 }
else if (
Name.starts_with(
"pshuf.b.")) {
2029 if (VecWidth == 128)
2030 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2031 else if (VecWidth == 256)
2032 IID = Intrinsic::x86_avx2_pshuf_b;
2033 else if (VecWidth == 512)
2034 IID = Intrinsic::x86_avx512_pshuf_b_512;
2037 }
else if (
Name.starts_with(
"pmul.hr.sw.")) {
2038 if (VecWidth == 128)
2039 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2040 else if (VecWidth == 256)
2041 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2042 else if (VecWidth == 512)
2043 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2046 }
else if (
Name.starts_with(
"pmulh.w.")) {
2047 if (VecWidth == 128)
2048 IID = Intrinsic::x86_sse2_pmulh_w;
2049 else if (VecWidth == 256)
2050 IID = Intrinsic::x86_avx2_pmulh_w;
2051 else if (VecWidth == 512)
2052 IID = Intrinsic::x86_avx512_pmulh_w_512;
2055 }
else if (
Name.starts_with(
"pmulhu.w.")) {
2056 if (VecWidth == 128)
2057 IID = Intrinsic::x86_sse2_pmulhu_w;
2058 else if (VecWidth == 256)
2059 IID = Intrinsic::x86_avx2_pmulhu_w;
2060 else if (VecWidth == 512)
2061 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2064 }
else if (
Name.starts_with(
"pmaddw.d.")) {
2065 if (VecWidth == 128)
2066 IID = Intrinsic::x86_sse2_pmadd_wd;
2067 else if (VecWidth == 256)
2068 IID = Intrinsic::x86_avx2_pmadd_wd;
2069 else if (VecWidth == 512)
2070 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2073 }
else if (
Name.starts_with(
"pmaddubs.w.")) {
2074 if (VecWidth == 128)
2075 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2076 else if (VecWidth == 256)
2077 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2078 else if (VecWidth == 512)
2079 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2082 }
else if (
Name.starts_with(
"packsswb.")) {
2083 if (VecWidth == 128)
2084 IID = Intrinsic::x86_sse2_packsswb_128;
2085 else if (VecWidth == 256)
2086 IID = Intrinsic::x86_avx2_packsswb;
2087 else if (VecWidth == 512)
2088 IID = Intrinsic::x86_avx512_packsswb_512;
2091 }
else if (
Name.starts_with(
"packssdw.")) {
2092 if (VecWidth == 128)
2093 IID = Intrinsic::x86_sse2_packssdw_128;
2094 else if (VecWidth == 256)
2095 IID = Intrinsic::x86_avx2_packssdw;
2096 else if (VecWidth == 512)
2097 IID = Intrinsic::x86_avx512_packssdw_512;
2100 }
else if (
Name.starts_with(
"packuswb.")) {
2101 if (VecWidth == 128)
2102 IID = Intrinsic::x86_sse2_packuswb_128;
2103 else if (VecWidth == 256)
2104 IID = Intrinsic::x86_avx2_packuswb;
2105 else if (VecWidth == 512)
2106 IID = Intrinsic::x86_avx512_packuswb_512;
2109 }
else if (
Name.starts_with(
"packusdw.")) {
2110 if (VecWidth == 128)
2111 IID = Intrinsic::x86_sse41_packusdw;
2112 else if (VecWidth == 256)
2113 IID = Intrinsic::x86_avx2_packusdw;
2114 else if (VecWidth == 512)
2115 IID = Intrinsic::x86_avx512_packusdw_512;
2118 }
else if (
Name.starts_with(
"vpermilvar.")) {
2119 if (VecWidth == 128 && EltWidth == 32)
2120 IID = Intrinsic::x86_avx_vpermilvar_ps;
2121 else if (VecWidth == 128 && EltWidth == 64)
2122 IID = Intrinsic::x86_avx_vpermilvar_pd;
2123 else if (VecWidth == 256 && EltWidth == 32)
2124 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2125 else if (VecWidth == 256 && EltWidth == 64)
2126 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2127 else if (VecWidth == 512 && EltWidth == 32)
2128 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2129 else if (VecWidth == 512 && EltWidth == 64)
2130 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2133 }
else if (
Name ==
"cvtpd2dq.256") {
2134 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2135 }
else if (
Name ==
"cvtpd2ps.256") {
2136 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2137 }
else if (
Name ==
"cvttpd2dq.256") {
2138 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2139 }
else if (
Name ==
"cvttps2dq.128") {
2140 IID = Intrinsic::x86_sse2_cvttps2dq;
2141 }
else if (
Name ==
"cvttps2dq.256") {
2142 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2143 }
else if (
Name.starts_with(
"permvar.")) {
2145 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2146 IID = Intrinsic::x86_avx2_permps;
2147 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2148 IID = Intrinsic::x86_avx2_permd;
2149 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2150 IID = Intrinsic::x86_avx512_permvar_df_256;
2151 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2152 IID = Intrinsic::x86_avx512_permvar_di_256;
2153 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2154 IID = Intrinsic::x86_avx512_permvar_sf_512;
2155 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2156 IID = Intrinsic::x86_avx512_permvar_si_512;
2157 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2158 IID = Intrinsic::x86_avx512_permvar_df_512;
2159 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2160 IID = Intrinsic::x86_avx512_permvar_di_512;
2161 else if (VecWidth == 128 && EltWidth == 16)
2162 IID = Intrinsic::x86_avx512_permvar_hi_128;
2163 else if (VecWidth == 256 && EltWidth == 16)
2164 IID = Intrinsic::x86_avx512_permvar_hi_256;
2165 else if (VecWidth == 512 && EltWidth == 16)
2166 IID = Intrinsic::x86_avx512_permvar_hi_512;
2167 else if (VecWidth == 128 && EltWidth == 8)
2168 IID = Intrinsic::x86_avx512_permvar_qi_128;
2169 else if (VecWidth == 256 && EltWidth == 8)
2170 IID = Intrinsic::x86_avx512_permvar_qi_256;
2171 else if (VecWidth == 512 && EltWidth == 8)
2172 IID = Intrinsic::x86_avx512_permvar_qi_512;
2175 }
else if (
Name.starts_with(
"dbpsadbw.")) {
2176 if (VecWidth == 128)
2177 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2178 else if (VecWidth == 256)
2179 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2180 else if (VecWidth == 512)
2181 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2184 }
else if (
Name.starts_with(
"pmultishift.qb.")) {
2185 if (VecWidth == 128)
2186 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2187 else if (VecWidth == 256)
2188 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2189 else if (VecWidth == 512)
2190 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2193 }
else if (
Name.starts_with(
"conflict.")) {
2194 if (
Name[9] ==
'd' && VecWidth == 128)
2195 IID = Intrinsic::x86_avx512_conflict_d_128;
2196 else if (
Name[9] ==
'd' && VecWidth == 256)
2197 IID = Intrinsic::x86_avx512_conflict_d_256;
2198 else if (
Name[9] ==
'd' && VecWidth == 512)
2199 IID = Intrinsic::x86_avx512_conflict_d_512;
2200 else if (
Name[9] ==
'q' && VecWidth == 128)
2201 IID = Intrinsic::x86_avx512_conflict_q_128;
2202 else if (
Name[9] ==
'q' && VecWidth == 256)
2203 IID = Intrinsic::x86_avx512_conflict_q_256;
2204 else if (
Name[9] ==
'q' && VecWidth == 512)
2205 IID = Intrinsic::x86_avx512_conflict_q_512;
2208 }
else if (
Name.starts_with(
"pavg.")) {
2209 if (
Name[5] ==
'b' && VecWidth == 128)
2210 IID = Intrinsic::x86_sse2_pavg_b;
2211 else if (
Name[5] ==
'b' && VecWidth == 256)
2212 IID = Intrinsic::x86_avx2_pavg_b;
2213 else if (
Name[5] ==
'b' && VecWidth == 512)
2214 IID = Intrinsic::x86_avx512_pavg_b_512;
2215 else if (
Name[5] ==
'w' && VecWidth == 128)
2216 IID = Intrinsic::x86_sse2_pavg_w;
2217 else if (
Name[5] ==
'w' && VecWidth == 256)
2218 IID = Intrinsic::x86_avx2_pavg_w;
2219 else if (
Name[5] ==
'w' && VecWidth == 512)
2220 IID = Intrinsic::x86_avx512_pavg_w_512;
2241 if (AsmStr->find(
"mov\tfp") == 0 &&
2242 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
2243 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
2244 AsmStr->replace(Pos, 1,
";");
2250 if (
Name ==
"mve.vctp64.old") {
2258 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2259 {VectorType::get(Builder.getInt1Ty(), 2, false)}),
2263 F->getParent(), Intrinsic::arm_mve_pred_i2v,
2264 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2266 }
else if (
Name ==
"mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
2267 Name ==
"mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
2268 Name ==
"mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
2269 Name ==
"mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
2271 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
2272 Name ==
"mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
2273 Name ==
"mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
2274 Name ==
"mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
2276 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
2277 Name ==
"mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
2278 Name ==
"cde.vcx1q.predicated.v2i64.v4i1" ||
2279 Name ==
"cde.vcx1qa.predicated.v2i64.v4i1" ||
2280 Name ==
"cde.vcx2q.predicated.v2i64.v4i1" ||
2281 Name ==
"cde.vcx2qa.predicated.v2i64.v4i1" ||
2282 Name ==
"cde.vcx3q.predicated.v2i64.v4i1" ||
2283 Name ==
"cde.vcx3qa.predicated.v2i64.v4i1") {
2284 std::vector<Type *> Tys;
2288 case Intrinsic::arm_mve_mull_int_predicated:
2289 case Intrinsic::arm_mve_vqdmull_predicated:
2290 case Intrinsic::arm_mve_vldr_gather_base_predicated:
2293 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
2294 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
2295 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
2299 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
2303 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
2307 case Intrinsic::arm_cde_vcx1q_predicated:
2308 case Intrinsic::arm_cde_vcx1qa_predicated:
2309 case Intrinsic::arm_cde_vcx2q_predicated:
2310 case Intrinsic::arm_cde_vcx2qa_predicated:
2311 case Intrinsic::arm_cde_vcx3q_predicated:
2312 case Intrinsic::arm_cde_vcx3qa_predicated:
2319 std::vector<Value *> Ops;
2321 Type *Ty =
Op->getType();
2325 F->getParent(), Intrinsic::arm_mve_pred_v2i,
2326 {VectorType::get(Builder.getInt1Ty(), 4, false)}),
2330 Intrinsic::arm_mve_pred_i2v, {V2I1Ty}),
2358 if (NumOperands < 3)
2371 bool IsVolatile =
false;
2375 if (NumOperands > 3)
2380 if (NumOperands > 5) {
2382 IsVolatile = !VolatileArg || !VolatileArg->
isZero();
2388 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
2389 Order = AtomicOrdering::SequentiallyConsistent;
2396 if (VT->getElementType()->isIntegerTy(16)) {
2409 if (PtrTy->getAddressSpace() != 3) {
2421template <
typename MDType>
2424 return dyn_cast<MDType>(MAV->getMetadata());
2433 if (
Name ==
"label") {
2435 }
else if (
Name ==
"assign") {
2437 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
2438 unwrapMAVOp<DIExpression>(CI, 2), unwrapMAVOp<DIAssignID>(CI, 3),
2439 unwrapMAVOp<Metadata>(CI, 4), unwrapMAVOp<DIExpression>(CI, 5),
2441 }
else if (
Name ==
"declare") {
2443 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
2444 unwrapMAVOp<DIExpression>(CI, 2), CI->
getDebugLoc(),
2445 DbgVariableRecord::LocationType::Declare);
2446 }
else if (
Name ==
"addr") {
2451 unwrapMAVOp<DILocalVariable>(CI, 1), Expr,
2453 }
else if (
Name ==
"value") {
2456 unsigned ExprOp = 2;
2466 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, VarOp),
2467 unwrapMAVOp<DIExpression>(CI, ExprOp), CI->
getDebugLoc());
2469 assert(DR &&
"Unhandled intrinsic kind in upgrade to DbgRecord");
2488 bool FallthroughToDefaultUpgrade =
false;
2492 assert(
Name.starts_with(
"llvm.") &&
"Intrinsic doesn't start with 'llvm.'");
2495 bool IsX86 =
Name.consume_front(
"x86.");
2496 bool IsNVVM =
Name.consume_front(
"nvvm.");
2497 bool IsARM =
Name.consume_front(
"arm.");
2498 bool IsAMDGCN =
Name.consume_front(
"amdgcn.");
2499 bool IsDbg =
Name.consume_front(
"dbg.");
2501 if (IsX86 &&
Name.starts_with(
"sse4a.movnt.")) {
2512 Type *SrcEltTy = cast<VectorType>(Arg1->
getType())->getElementType();
2513 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
2519 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2526 if (IsX86 && (
Name.starts_with(
"avx.movnt.") ||
2527 Name.starts_with(
"avx512.storent."))) {
2538 PointerType::getUnqual(Arg1->
getType()),
2543 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
2550 if (IsX86 &&
Name ==
"sse2.storel.dq") {
2558 PointerType::getUnqual(Elt->
getType()),
2567 if (IsX86 && (
Name.starts_with(
"sse.storeu.") ||
2568 Name.starts_with(
"sse2.storeu.") ||
2569 Name.starts_with(
"avx.storeu."))) {
2574 PointerType::getUnqual(Arg1->
getType()),
2583 if (IsX86 &&
Name ==
"avx512.mask.store.ss") {
2593 if (IsX86 &&
Name.starts_with(
"avx512.mask.store")) {
2604 Value *Rep =
nullptr;
2606 if (IsX86 && (
Name.starts_with(
"sse2.pcmp") ||
2607 Name.starts_with(
"avx2.pcmp"))) {
2609 bool CmpEq =
Name[9] ==
'e';
2610 Rep = Builder.
CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
2613 }
else if (IsX86 && (
Name.starts_with(
"avx512.broadcastm"))) {
2621 }
else if (IsX86 && (
Name ==
"sse.sqrt.ss" ||
2622 Name ==
"sse2.sqrt.sd")) {
2626 Intrinsic::sqrt, Elt0->
getType());
2629 }
else if (IsX86 && (
Name.starts_with(
"avx.sqrt.p") ||
2630 Name.starts_with(
"sse2.sqrt.p") ||
2631 Name.starts_with(
"sse.sqrt.p"))) {
2635 {CI->getArgOperand(0)});
2636 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.sqrt.p"))) {
2639 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2641 : Intrinsic::x86_avx512_sqrt_pd_512;
2650 {CI->getArgOperand(0)});
2654 }
else if (IsX86 && (
Name.starts_with(
"avx512.ptestm") ||
2655 Name.starts_with(
"avx512.ptestnm"))) {
2663 Name.starts_with(
"avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2666 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.pbroadcast"))){
2672 }
else if (IsX86 && (
Name.starts_with(
"avx512.kunpck"))) {
2677 for (
unsigned i = 0; i != NumElts; ++i)
2690 }
else if (IsX86 &&
Name ==
"avx512.kand.w") {
2695 }
else if (IsX86 &&
Name ==
"avx512.kandn.w") {
2701 }
else if (IsX86 &&
Name ==
"avx512.kor.w") {
2706 }
else if (IsX86 &&
Name ==
"avx512.kxor.w") {
2711 }
else if (IsX86 &&
Name ==
"avx512.kxnor.w") {
2717 }
else if (IsX86 &&
Name ==
"avx512.knot.w") {
2722 (
Name ==
"avx512.kortestz.w" ||
Name ==
"avx512.kortestc.w")) {
2728 if (
Name[14] ==
'c')
2729 C = ConstantInt::getAllOnesValue(Builder.
getInt16Ty());
2731 C = ConstantInt::getNullValue(Builder.
getInt16Ty());
2734 }
else if (IsX86 && (
Name ==
"sse.add.ss" ||
Name ==
"sse2.add.sd" ||
2735 Name ==
"sse.sub.ss" ||
Name ==
"sse2.sub.sd" ||
2736 Name ==
"sse.mul.ss" ||
Name ==
"sse2.mul.sd" ||
2737 Name ==
"sse.div.ss" ||
Name ==
"sse2.div.sd")) {
2740 ConstantInt::get(I32Ty, 0));
2742 ConstantInt::get(I32Ty, 0));
2744 if (
Name.contains(
".add."))
2746 else if (
Name.contains(
".sub."))
2748 else if (
Name.contains(
".mul."))
2753 ConstantInt::get(I32Ty, 0));
2754 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.pcmp")) {
2756 bool CmpEq =
Name[16] ==
'e';
2758 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.vpshufbitqmb.")) {
2764 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
break;
2765 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
break;
2766 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
break;
2770 { CI->getOperand(0), CI->getArgOperand(1) });
2772 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.fpclass.p")) {
2777 if (VecWidth == 128 && EltWidth == 32)
2778 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2779 else if (VecWidth == 256 && EltWidth == 32)
2780 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2781 else if (VecWidth == 512 && EltWidth == 32)
2782 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2783 else if (VecWidth == 128 && EltWidth == 64)
2784 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2785 else if (VecWidth == 256 && EltWidth == 64)
2786 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2787 else if (VecWidth == 512 && EltWidth == 64)
2788 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2793 { CI->getOperand(0), CI->getArgOperand(1) });
2795 }
else if (IsX86 &&
Name.starts_with(
"avx512.cmp.p")) {
2797 Type *OpTy = Args[0]->getType();
2801 if (VecWidth == 128 && EltWidth == 32)
2802 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2803 else if (VecWidth == 256 && EltWidth == 32)
2804 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2805 else if (VecWidth == 512 && EltWidth == 32)
2806 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2807 else if (VecWidth == 128 && EltWidth == 64)
2808 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2809 else if (VecWidth == 256 && EltWidth == 64)
2810 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2811 else if (VecWidth == 512 && EltWidth == 64)
2812 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2817 if (VecWidth == 512)
2819 Args.push_back(Mask);
2823 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.cmp.")) {
2825 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2827 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.ucmp.")) {
2828 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2830 }
else if (IsX86 && (
Name.starts_with(
"avx512.cvtb2mask.") ||
2831 Name.starts_with(
"avx512.cvtw2mask.") ||
2832 Name.starts_with(
"avx512.cvtd2mask.") ||
2833 Name.starts_with(
"avx512.cvtq2mask."))) {
2836 Rep = Builder.
CreateICmp(ICmpInst::ICMP_SLT,
Op, Zero);
2838 }
else if(IsX86 && (
Name ==
"ssse3.pabs.b.128" ||
2839 Name ==
"ssse3.pabs.w.128" ||
2840 Name ==
"ssse3.pabs.d.128" ||
2841 Name.starts_with(
"avx2.pabs") ||
2842 Name.starts_with(
"avx512.mask.pabs"))) {
2844 }
else if (IsX86 && (
Name ==
"sse41.pmaxsb" ||
2845 Name ==
"sse2.pmaxs.w" ||
2846 Name ==
"sse41.pmaxsd" ||
2847 Name.starts_with(
"avx2.pmaxs") ||
2848 Name.starts_with(
"avx512.mask.pmaxs"))) {
2850 }
else if (IsX86 && (
Name ==
"sse2.pmaxu.b" ||
2851 Name ==
"sse41.pmaxuw" ||
2852 Name ==
"sse41.pmaxud" ||
2853 Name.starts_with(
"avx2.pmaxu") ||
2854 Name.starts_with(
"avx512.mask.pmaxu"))) {
2856 }
else if (IsX86 && (
Name ==
"sse41.pminsb" ||
2857 Name ==
"sse2.pmins.w" ||
2858 Name ==
"sse41.pminsd" ||
2859 Name.starts_with(
"avx2.pmins") ||
2860 Name.starts_with(
"avx512.mask.pmins"))) {
2862 }
else if (IsX86 && (
Name ==
"sse2.pminu.b" ||
2863 Name ==
"sse41.pminuw" ||
2864 Name ==
"sse41.pminud" ||
2865 Name.starts_with(
"avx2.pminu") ||
2866 Name.starts_with(
"avx512.mask.pminu"))) {
2868 }
else if (IsX86 && (
Name ==
"sse2.pmulu.dq" ||
2869 Name ==
"avx2.pmulu.dq" ||
2870 Name ==
"avx512.pmulu.dq.512" ||
2871 Name.starts_with(
"avx512.mask.pmulu.dq."))) {
2873 }
else if (IsX86 && (
Name ==
"sse41.pmuldq" ||
2874 Name ==
"avx2.pmul.dq" ||
2875 Name ==
"avx512.pmul.dq.512" ||
2876 Name.starts_with(
"avx512.mask.pmul.dq."))) {
2878 }
else if (IsX86 && (
Name ==
"sse.cvtsi2ss" ||
2879 Name ==
"sse2.cvtsi2sd" ||
2880 Name ==
"sse.cvtsi642ss" ||
2881 Name ==
"sse2.cvtsi642sd")) {
2884 cast<VectorType>(CI->
getType())->getElementType());
2886 }
else if (IsX86 &&
Name ==
"avx512.cvtusi2sd") {
2889 cast<VectorType>(CI->
getType())->getElementType());
2891 }
else if (IsX86 &&
Name ==
"sse2.cvtss2sd") {
2894 Rep, cast<VectorType>(CI->
getType())->getElementType());
2896 }
else if (IsX86 && (
Name ==
"sse2.cvtdq2pd" ||
2897 Name ==
"sse2.cvtdq2ps" ||
2898 Name ==
"avx.cvtdq2.pd.256" ||
2899 Name ==
"avx.cvtdq2.ps.256" ||
2900 Name.starts_with(
"avx512.mask.cvtdq2pd.") ||
2901 Name.starts_with(
"avx512.mask.cvtudq2pd.") ||
2902 Name.starts_with(
"avx512.mask.cvtdq2ps.") ||
2903 Name.starts_with(
"avx512.mask.cvtudq2ps.") ||
2904 Name.starts_with(
"avx512.mask.cvtqq2pd.") ||
2905 Name.starts_with(
"avx512.mask.cvtuqq2pd.") ||
2906 Name ==
"avx512.mask.cvtqq2ps.256" ||
2907 Name ==
"avx512.mask.cvtqq2ps.512" ||
2908 Name ==
"avx512.mask.cvtuqq2ps.256" ||
2909 Name ==
"avx512.mask.cvtuqq2ps.512" ||
2910 Name ==
"sse2.cvtps2pd" ||
2911 Name ==
"avx.cvt.ps2.pd.256" ||
2912 Name ==
"avx512.mask.cvtps2pd.128" ||
2913 Name ==
"avx512.mask.cvtps2pd.256")) {
2914 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2916 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2918 unsigned NumDstElts = DstTy->getNumElements();
2919 if (NumDstElts < SrcTy->getNumElements()) {
2920 assert(NumDstElts == 2 &&
"Unexpected vector size");
2924 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2925 bool IsUnsigned =
Name.contains(
"cvtu");
2927 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtps2pd");
2930 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2931 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2932 : Intrinsic::x86_avx512_sitofp_round;
2937 Rep = IsUnsigned ? Builder.
CreateUIToFP(Rep, DstTy,
"cvt")
2944 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vcvtph2ps.") ||
2945 Name.starts_with(
"vcvtph2ps."))) {
2946 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2948 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2949 unsigned NumDstElts = DstTy->getNumElements();
2950 if (NumDstElts != SrcTy->getNumElements()) {
2951 assert(NumDstElts == 4 &&
"Unexpected vector size");
2956 Rep = Builder.
CreateFPExt(Rep, DstTy,
"cvtph2ps");
2960 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.load")) {
2966 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.expand.load.")) {
2967 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2968 Type *PtrTy = ResultTy->getElementType();
2975 ResultTy->getNumElements());
2978 Intrinsic::masked_expandload,
2981 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.compress.store.")) {
2983 Type *PtrTy = ResultTy->getElementType();
2991 cast<FixedVectorType>(ResultTy)->getNumElements());
2994 Intrinsic::masked_compressstore,
2997 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.compress.") ||
2998 Name.starts_with(
"avx512.mask.expand."))) {
2999 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
3002 ResultTy->getNumElements());
3004 bool IsCompress =
Name[12] ==
'c';
3005 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3006 : Intrinsic::x86_avx512_mask_expand;
3010 }
else if (IsX86 &&
Name.starts_with(
"xop.vpcom")) {
3012 if (
Name.ends_with(
"ub") ||
Name.ends_with(
"uw") ||
Name.ends_with(
"ud") ||
3013 Name.ends_with(
"uq"))
3015 else if (
Name.ends_with(
"b") ||
Name.ends_with(
"w") ||
Name.ends_with(
"d") ||
3016 Name.ends_with(
"q"))
3023 Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3026 if (
Name.starts_with(
"lt"))
3028 else if (
Name.starts_with(
"le"))
3030 else if (
Name.starts_with(
"gt"))
3032 else if (
Name.starts_with(
"ge"))
3034 else if (
Name.starts_with(
"eq"))
3036 else if (
Name.starts_with(
"ne"))
3038 else if (
Name.starts_with(
"false"))
3040 else if (
Name.starts_with(
"true"))
3047 }
else if (IsX86 &&
Name.starts_with(
"xop.vpcmov")) {
3052 Rep = Builder.
CreateOr(Sel0, Sel1);
3053 }
else if (IsX86 && (
Name.starts_with(
"xop.vprot") ||
3054 Name.starts_with(
"avx512.prol") ||
3055 Name.starts_with(
"avx512.mask.prol"))) {
3057 }
else if (IsX86 && (
Name.starts_with(
"avx512.pror") ||
3058 Name.starts_with(
"avx512.mask.pror"))) {
3060 }
else if (IsX86 && (
Name.starts_with(
"avx512.vpshld.") ||
3061 Name.starts_with(
"avx512.mask.vpshld") ||
3062 Name.starts_with(
"avx512.maskz.vpshld"))) {
3063 bool ZeroMask =
Name[11] ==
'z';
3065 }
else if (IsX86 && (
Name.starts_with(
"avx512.vpshrd.") ||
3066 Name.starts_with(
"avx512.mask.vpshrd") ||
3067 Name.starts_with(
"avx512.maskz.vpshrd"))) {
3068 bool ZeroMask =
Name[11] ==
'z';
3070 }
else if (IsX86 &&
Name ==
"sse42.crc32.64.8") {
3072 Intrinsic::x86_sse42_crc32_32_8);
3076 }
else if (IsX86 && (
Name.starts_with(
"avx.vbroadcast.s") ||
3077 Name.starts_with(
"avx512.vbroadcast.s"))) {
3079 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3080 Type *EltTy = VecTy->getElementType();
3081 unsigned EltNum = VecTy->getNumElements();
3085 for (
unsigned I = 0;
I < EltNum; ++
I)
3087 ConstantInt::get(I32Ty,
I));
3088 }
else if (IsX86 && (
Name.starts_with(
"sse41.pmovsx") ||
3089 Name.starts_with(
"sse41.pmovzx") ||
3090 Name.starts_with(
"avx2.pmovsx") ||
3091 Name.starts_with(
"avx2.pmovzx") ||
3092 Name.starts_with(
"avx512.mask.pmovsx") ||
3093 Name.starts_with(
"avx512.mask.pmovzx"))) {
3094 auto *DstTy = cast<FixedVectorType>(CI->
getType());
3095 unsigned NumDstElts = DstTy->getNumElements();
3099 for (
unsigned i = 0; i != NumDstElts; ++i)
3105 bool DoSext =
Name.contains(
"pmovsx");
3112 }
else if (
Name ==
"avx512.mask.pmov.qd.256" ||
3113 Name ==
"avx512.mask.pmov.qd.512" ||
3114 Name ==
"avx512.mask.pmov.wb.256" ||
3115 Name ==
"avx512.mask.pmov.wb.512") {
3120 }
else if (IsX86 && (
Name.starts_with(
"avx.vbroadcastf128") ||
3121 Name ==
"avx2.vbroadcasti128")) {
3123 Type *EltTy = cast<VectorType>(CI->
getType())->getElementType();
3127 PointerType::getUnqual(VT));
3129 if (NumSrcElts == 2)
3134 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.shuf.i") ||
3135 Name.starts_with(
"avx512.mask.shuf.f"))) {
3136 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3140 unsigned ControlBitsMask = NumLanes - 1;
3141 unsigned NumControlBits = NumLanes / 2;
3144 for (
unsigned l = 0; l != NumLanes; ++l) {
3145 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3147 if (l >= NumLanes / 2)
3148 LaneMask += NumLanes;
3149 for (
unsigned i = 0; i != NumElementsInLane; ++i)
3150 ShuffleMask.
push_back(LaneMask * NumElementsInLane + i);
3156 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.broadcastf") ||
3157 Name.starts_with(
"avx512.mask.broadcasti"))) {
3158 unsigned NumSrcElts =
3161 unsigned NumDstElts =
3162 cast<FixedVectorType>(CI->
getType())->getNumElements();
3165 for (
unsigned i = 0; i != NumDstElts; ++i)
3166 ShuffleMask[i] = i % NumSrcElts;
3173 }
else if (IsX86 && (
Name.starts_with(
"avx2.pbroadcast") ||
3174 Name.starts_with(
"avx2.vbroadcast") ||
3175 Name.starts_with(
"avx512.pbroadcast") ||
3176 Name.starts_with(
"avx512.mask.broadcast.s"))) {
3188 }
else if (IsX86 && (
Name.starts_with(
"sse2.padds.") ||
3189 Name.starts_with(
"avx2.padds.") ||
3190 Name.starts_with(
"avx512.padds.") ||
3191 Name.starts_with(
"avx512.mask.padds."))) {
3193 }
else if (IsX86 && (
Name.starts_with(
"sse2.psubs.") ||
3194 Name.starts_with(
"avx2.psubs.") ||
3195 Name.starts_with(
"avx512.psubs.") ||
3196 Name.starts_with(
"avx512.mask.psubs."))) {
3198 }
else if (IsX86 && (
Name.starts_with(
"sse2.paddus.") ||
3199 Name.starts_with(
"avx2.paddus.") ||
3200 Name.starts_with(
"avx512.mask.paddus."))) {
3202 }
else if (IsX86 && (
Name.starts_with(
"sse2.psubus.") ||
3203 Name.starts_with(
"avx2.psubus.") ||
3204 Name.starts_with(
"avx512.mask.psubus."))) {
3206 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.palignr.")) {
3211 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.valign.")) {
3216 }
else if (IsX86 && (
Name ==
"sse2.psll.dq" ||
3217 Name ==
"avx2.psll.dq")) {
3219 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3222 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq" ||
3223 Name ==
"avx2.psrl.dq")) {
3225 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3228 }
else if (IsX86 && (
Name ==
"sse2.psll.dq.bs" ||
3229 Name ==
"avx2.psll.dq.bs" ||
3230 Name ==
"avx512.psll.dq.512")) {
3232 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3234 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq.bs" ||
3235 Name ==
"avx2.psrl.dq.bs" ||
3236 Name ==
"avx512.psrl.dq.512")) {
3238 unsigned Shift = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3240 }
else if (IsX86 && (
Name ==
"sse41.pblendw" ||
3241 Name.starts_with(
"sse41.blendp") ||
3242 Name.starts_with(
"avx.blend.p") ||
3243 Name ==
"avx2.pblendw" ||
3244 Name.starts_with(
"avx2.pblendd."))) {
3247 unsigned Imm = cast <ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3248 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3249 unsigned NumElts = VecTy->getNumElements();
3252 for (
unsigned i = 0; i != NumElts; ++i)
3253 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
3256 }
else if (IsX86 && (
Name.starts_with(
"avx.vinsertf128.") ||
3257 Name ==
"avx2.vinserti128" ||
3258 Name.starts_with(
"avx512.mask.insert"))) {
3261 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3262 unsigned DstNumElts =
3263 cast<FixedVectorType>(CI->
getType())->getNumElements();
3264 unsigned SrcNumElts =
3265 cast<FixedVectorType>(Op1->
getType())->getNumElements();
3266 unsigned Scale = DstNumElts / SrcNumElts;
3273 for (
unsigned i = 0; i != SrcNumElts; ++i)
3275 for (
unsigned i = SrcNumElts; i != DstNumElts; ++i)
3276 Idxs[i] = SrcNumElts;
3291 for (
unsigned i = 0; i != DstNumElts; ++i)
3294 for (
unsigned i = 0; i != SrcNumElts; ++i)
3295 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3302 }
else if (IsX86 && (
Name.starts_with(
"avx.vextractf128.") ||
3303 Name ==
"avx2.vextracti128" ||
3304 Name.starts_with(
"avx512.mask.vextract"))) {
3306 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3307 unsigned DstNumElts =
3308 cast<FixedVectorType>(CI->
getType())->getNumElements();
3309 unsigned SrcNumElts =
3310 cast<FixedVectorType>(Op0->
getType())->getNumElements();
3311 unsigned Scale = SrcNumElts / DstNumElts;
3318 for (
unsigned i = 0; i != DstNumElts; ++i) {
3319 Idxs[i] = i + (Imm * DstNumElts);
3327 }
else if (!IsX86 &&
Name ==
"stackprotectorcheck") {
3329 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.perm.df.") ||
3330 Name.starts_with(
"avx512.mask.perm.di."))) {
3332 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3333 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3334 unsigned NumElts = VecTy->getNumElements();
3337 for (
unsigned i = 0; i != NumElts; ++i)
3338 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3345 }
else if (IsX86 && (
Name.starts_with(
"avx.vperm2f128.") ||
3346 Name ==
"avx2.vperm2i128")) {
3355 uint8_t Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3357 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3358 unsigned HalfSize = NumElts / 2;
3370 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3371 for (
unsigned i = 0; i < HalfSize; ++i)
3372 ShuffleMask[i] = StartIndex + i;
3375 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3376 for (
unsigned i = 0; i < HalfSize; ++i)
3377 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3381 }
else if (IsX86 && (
Name.starts_with(
"avx.vpermil.") ||
3382 Name ==
"sse2.pshuf.d" ||
3383 Name.starts_with(
"avx512.mask.vpermil.p") ||
3384 Name.starts_with(
"avx512.mask.pshuf.d."))) {
3386 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3387 auto *VecTy = cast<FixedVectorType>(CI->
getType());
3388 unsigned NumElts = VecTy->getNumElements();
3390 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3391 unsigned IdxMask = ((1 << IdxSize) - 1);
3397 for (
unsigned i = 0; i != NumElts; ++i)
3398 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3405 }
else if (IsX86 && (
Name ==
"sse2.pshufl.w" ||
3406 Name.starts_with(
"avx512.mask.pshufl.w."))) {
3408 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3409 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3412 for (
unsigned l = 0; l != NumElts; l += 8) {
3413 for (
unsigned i = 0; i != 4; ++i)
3414 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3415 for (
unsigned i = 4; i != 8; ++i)
3416 Idxs[i + l] = i + l;
3424 }
else if (IsX86 && (
Name ==
"sse2.pshufh.w" ||
3425 Name.starts_with(
"avx512.mask.pshufh.w."))) {
3427 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
3428 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3431 for (
unsigned l = 0; l != NumElts; l += 8) {
3432 for (
unsigned i = 0; i != 4; ++i)
3433 Idxs[i + l] = i + l;
3434 for (
unsigned i = 0; i != 4; ++i)
3435 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3443 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.shuf.p")) {
3446 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
3447 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3450 unsigned HalfLaneElts = NumLaneElts / 2;
3453 for (
unsigned i = 0; i != NumElts; ++i) {
3455 Idxs[i] = i - (i % NumLaneElts);
3457 if ((i % NumLaneElts) >= HalfLaneElts)
3461 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3468 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.movddup") ||
3469 Name.starts_with(
"avx512.mask.movshdup") ||
3470 Name.starts_with(
"avx512.mask.movsldup"))) {
3472 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3476 if (
Name.starts_with(
"avx512.mask.movshdup."))
3480 for (
unsigned l = 0; l != NumElts; l += NumLaneElts)
3481 for (
unsigned i = 0; i != NumLaneElts; i += 2) {
3482 Idxs[i + l + 0] = i + l +
Offset;
3483 Idxs[i + l + 1] = i + l +
Offset;
3490 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.punpckl") ||
3491 Name.starts_with(
"avx512.mask.unpckl."))) {
3494 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3498 for (
int l = 0; l != NumElts; l += NumLaneElts)
3499 for (
int i = 0; i != NumLaneElts; ++i)
3500 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3506 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.punpckh") ||
3507 Name.starts_with(
"avx512.mask.unpckh."))) {
3510 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3514 for (
int l = 0; l != NumElts; l += NumLaneElts)
3515 for (
int i = 0; i != NumLaneElts; ++i)
3516 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3522 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.and.") ||
3523 Name.starts_with(
"avx512.mask.pand."))) {
3525 VectorType *ITy = VectorType::getInteger(FTy);
3531 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.andn.") ||
3532 Name.starts_with(
"avx512.mask.pandn."))) {
3534 VectorType *ITy = VectorType::getInteger(FTy);
3541 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.or.") ||
3542 Name.starts_with(
"avx512.mask.por."))) {
3544 VectorType *ITy = VectorType::getInteger(FTy);
3550 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.xor.") ||
3551 Name.starts_with(
"avx512.mask.pxor."))) {
3553 VectorType *ITy = VectorType::getInteger(FTy);
3559 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.padd.")) {
3563 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psub.")) {
3567 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.pmull.")) {
3571 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.add.p")) {
3572 if (
Name.ends_with(
".512")) {
3574 if (
Name[17] ==
's')
3575 IID = Intrinsic::x86_avx512_add_ps_512;
3577 IID = Intrinsic::x86_avx512_add_pd_512;
3580 { CI->getArgOperand(0), CI->getArgOperand(1),
3581 CI->getArgOperand(4) });
3587 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.div.p")) {
3588 if (
Name.ends_with(
".512")) {
3590 if (
Name[17] ==
's')
3591 IID = Intrinsic::x86_avx512_div_ps_512;
3593 IID = Intrinsic::x86_avx512_div_pd_512;
3596 { CI->getArgOperand(0), CI->getArgOperand(1),
3597 CI->getArgOperand(4) });
3603 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.mul.p")) {
3604 if (
Name.ends_with(
".512")) {
3606 if (
Name[17] ==
's')
3607 IID = Intrinsic::x86_avx512_mul_ps_512;
3609 IID = Intrinsic::x86_avx512_mul_pd_512;
3612 { CI->getArgOperand(0), CI->getArgOperand(1),
3613 CI->getArgOperand(4) });
3619 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.sub.p")) {
3620 if (
Name.ends_with(
".512")) {
3622 if (
Name[17] ==
's')
3623 IID = Intrinsic::x86_avx512_sub_ps_512;
3625 IID = Intrinsic::x86_avx512_sub_pd_512;
3628 { CI->getArgOperand(0), CI->getArgOperand(1),
3629 CI->getArgOperand(4) });
3635 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.max.p") ||
3636 Name.starts_with(
"avx512.mask.min.p")) &&
3637 Name.drop_front(18) ==
".512") {
3638 bool IsDouble =
Name[17] ==
'd';
3639 bool IsMin =
Name[13] ==
'i';
3641 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
3642 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
3647 { CI->getArgOperand(0), CI->getArgOperand(1),
3648 CI->getArgOperand(4) });
3651 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.lzcnt.")) {
3655 { CI->getArgOperand(0), Builder.getInt1(false) });
3658 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psll")) {
3659 bool IsImmediate =
Name[16] ==
'i' ||
3660 (
Name.size() > 18 &&
Name[18] ==
'i');
3661 bool IsVariable =
Name[16] ==
'v';
3668 if (IsVariable &&
Name[17] !=
'.') {
3669 if (
Size ==
'd' &&
Name[17] ==
'2')
3670 IID = Intrinsic::x86_avx2_psllv_q;
3671 else if (
Size ==
'd' &&
Name[17] ==
'4')
3672 IID = Intrinsic::x86_avx2_psllv_q_256;
3673 else if (
Size ==
's' &&
Name[17] ==
'4')
3674 IID = Intrinsic::x86_avx2_psllv_d;
3675 else if (
Size ==
's' &&
Name[17] ==
'8')
3676 IID = Intrinsic::x86_avx2_psllv_d_256;
3677 else if (
Size ==
'h' &&
Name[17] ==
'8')
3678 IID = Intrinsic::x86_avx512_psllv_w_128;
3679 else if (
Size ==
'h' &&
Name[17] ==
'1')
3680 IID = Intrinsic::x86_avx512_psllv_w_256;
3681 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3682 IID = Intrinsic::x86_avx512_psllv_w_512;
3685 }
else if (
Name.ends_with(
".128")) {
3687 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3688 : Intrinsic::x86_sse2_psll_d;
3689 else if (
Size ==
'q')
3690 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3691 : Intrinsic::x86_sse2_psll_q;
3692 else if (
Size ==
'w')
3693 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3694 : Intrinsic::x86_sse2_psll_w;
3697 }
else if (
Name.ends_with(
".256")) {
3699 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3700 : Intrinsic::x86_avx2_psll_d;
3701 else if (
Size ==
'q')
3702 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3703 : Intrinsic::x86_avx2_psll_q;
3704 else if (
Size ==
'w')
3705 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3706 : Intrinsic::x86_avx2_psll_w;
3711 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3712 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3713 Intrinsic::x86_avx512_psll_d_512;
3714 else if (
Size ==
'q')
3715 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3716 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3717 Intrinsic::x86_avx512_psll_q_512;
3718 else if (
Size ==
'w')
3719 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3720 : Intrinsic::x86_avx512_psll_w_512;
3726 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psrl")) {
3727 bool IsImmediate =
Name[16] ==
'i' ||
3728 (
Name.size() > 18 &&
Name[18] ==
'i');
3729 bool IsVariable =
Name[16] ==
'v';
3736 if (IsVariable &&
Name[17] !=
'.') {
3737 if (
Size ==
'd' &&
Name[17] ==
'2')
3738 IID = Intrinsic::x86_avx2_psrlv_q;
3739 else if (
Size ==
'd' &&
Name[17] ==
'4')
3740 IID = Intrinsic::x86_avx2_psrlv_q_256;
3741 else if (
Size ==
's' &&
Name[17] ==
'4')
3742 IID = Intrinsic::x86_avx2_psrlv_d;
3743 else if (
Size ==
's' &&
Name[17] ==
'8')
3744 IID = Intrinsic::x86_avx2_psrlv_d_256;
3745 else if (
Size ==
'h' &&
Name[17] ==
'8')
3746 IID = Intrinsic::x86_avx512_psrlv_w_128;
3747 else if (
Size ==
'h' &&
Name[17] ==
'1')
3748 IID = Intrinsic::x86_avx512_psrlv_w_256;
3749 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3750 IID = Intrinsic::x86_avx512_psrlv_w_512;
3753 }
else if (
Name.ends_with(
".128")) {
3755 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3756 : Intrinsic::x86_sse2_psrl_d;
3757 else if (
Size ==
'q')
3758 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3759 : Intrinsic::x86_sse2_psrl_q;
3760 else if (
Size ==
'w')
3761 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3762 : Intrinsic::x86_sse2_psrl_w;
3765 }
else if (
Name.ends_with(
".256")) {
3767 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3768 : Intrinsic::x86_avx2_psrl_d;
3769 else if (
Size ==
'q')
3770 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3771 : Intrinsic::x86_avx2_psrl_q;
3772 else if (
Size ==
'w')
3773 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3774 : Intrinsic::x86_avx2_psrl_w;
3779 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3780 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3781 Intrinsic::x86_avx512_psrl_d_512;
3782 else if (
Size ==
'q')
3783 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3784 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3785 Intrinsic::x86_avx512_psrl_q_512;
3786 else if (
Size ==
'w')
3787 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3788 : Intrinsic::x86_avx512_psrl_w_512;
3794 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.psra")) {
3795 bool IsImmediate =
Name[16] ==
'i' ||
3796 (
Name.size() > 18 &&
Name[18] ==
'i');
3797 bool IsVariable =
Name[16] ==
'v';
3804 if (IsVariable &&
Name[17] !=
'.') {
3805 if (
Size ==
's' &&
Name[17] ==
'4')
3806 IID = Intrinsic::x86_avx2_psrav_d;
3807 else if (
Size ==
's' &&
Name[17] ==
'8')
3808 IID = Intrinsic::x86_avx2_psrav_d_256;
3809 else if (
Size ==
'h' &&
Name[17] ==
'8')
3810 IID = Intrinsic::x86_avx512_psrav_w_128;
3811 else if (
Size ==
'h' &&
Name[17] ==
'1')
3812 IID = Intrinsic::x86_avx512_psrav_w_256;
3813 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3814 IID = Intrinsic::x86_avx512_psrav_w_512;
3817 }
else if (
Name.ends_with(
".128")) {
3819 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3820 : Intrinsic::x86_sse2_psra_d;
3821 else if (
Size ==
'q')
3822 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3823 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3824 Intrinsic::x86_avx512_psra_q_128;
3825 else if (
Size ==
'w')
3826 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3827 : Intrinsic::x86_sse2_psra_w;
3830 }
else if (
Name.ends_with(
".256")) {
3832 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3833 : Intrinsic::x86_avx2_psra_d;
3834 else if (
Size ==
'q')
3835 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3836 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3837 Intrinsic::x86_avx512_psra_q_256;
3838 else if (
Size ==
'w')
3839 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3840 : Intrinsic::x86_avx2_psra_w;
3845 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3846 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3847 Intrinsic::x86_avx512_psra_d_512;
3848 else if (
Size ==
'q')
3849 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3850 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3851 Intrinsic::x86_avx512_psra_q_512;
3852 else if (
Size ==
'w')
3853 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3854 : Intrinsic::x86_avx512_psra_w_512;
3860 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.move.s")) {
3862 }
else if (IsX86 &&
Name.starts_with(
"avx512.cvtmask2")) {
3864 }
else if (IsX86 &&
Name.ends_with(
".movntdqa")) {
3872 Ptr, PointerType::getUnqual(CI->
getType()),
"cast");
3876 LI->
setMetadata(LLVMContext::MD_nontemporal, Node);
3878 }
else if (IsX86 && (
Name.starts_with(
"fma.vfmadd.") ||
3879 Name.starts_with(
"fma.vfmsub.") ||
3880 Name.starts_with(
"fma.vfnmadd.") ||
3881 Name.starts_with(
"fma.vfnmsub."))) {
3882 bool NegMul =
Name[6] ==
'n';
3883 bool NegAcc = NegMul ?
Name[8] ==
's' :
Name[7] ==
's';
3884 bool IsScalar = NegMul ?
Name[12] ==
's' :
Name[11] ==
's';
3895 if (NegMul && !IsScalar)
3897 if (NegMul && IsScalar)
3910 }
else if (IsX86 &&
Name.starts_with(
"fma4.vfmadd.s")) {
3925 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmadd.s") ||
3926 Name.starts_with(
"avx512.maskz.vfmadd.s") ||
3927 Name.starts_with(
"avx512.mask3.vfmadd.s") ||
3928 Name.starts_with(
"avx512.mask3.vfmsub.s") ||
3929 Name.starts_with(
"avx512.mask3.vfnmsub.s"))) {
3930 bool IsMask3 =
Name[11] ==
'3';
3931 bool IsMaskZ =
Name[11] ==
'z';
3933 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3934 bool NegMul =
Name[2] ==
'n';
3935 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3941 if (NegMul && (IsMask3 || IsMaskZ))
3943 if (NegMul && !(IsMask3 || IsMaskZ))
3953 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4) {
3957 if (
Name.back() ==
'd')
3958 IID = Intrinsic::x86_avx512_vfmadd_f64;
3960 IID = Intrinsic::x86_avx512_vfmadd_f32;
3975 if (NegAcc && IsMask3)
3982 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmadd.p") ||
3983 Name.starts_with(
"avx512.mask.vfnmadd.p") ||
3984 Name.starts_with(
"avx512.mask.vfnmsub.p") ||
3985 Name.starts_with(
"avx512.mask3.vfmadd.p") ||
3986 Name.starts_with(
"avx512.mask3.vfmsub.p") ||
3987 Name.starts_with(
"avx512.mask3.vfnmsub.p") ||
3988 Name.starts_with(
"avx512.maskz.vfmadd.p"))) {
3989 bool IsMask3 =
Name[11] ==
'3';
3990 bool IsMaskZ =
Name[11] ==
'z';
3992 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3993 bool NegMul =
Name[2] ==
'n';
3994 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
4000 if (NegMul && (IsMask3 || IsMaskZ))
4002 if (NegMul && !(IsMask3 || IsMaskZ))
4009 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4)) {
4013 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4015 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4018 { A, B, C, CI->getArgOperand(4) });
4031 }
else if (IsX86 &&
Name.starts_with(
"fma.vfmsubadd.p")) {
4035 if (VecWidth == 128 && EltWidth == 32)
4036 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4037 else if (VecWidth == 256 && EltWidth == 32)
4038 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4039 else if (VecWidth == 128 && EltWidth == 64)
4040 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4041 else if (VecWidth == 256 && EltWidth == 64)
4042 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4051 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vfmaddsub.p") ||
4052 Name.starts_with(
"avx512.mask3.vfmaddsub.p") ||
4053 Name.starts_with(
"avx512.maskz.vfmaddsub.p") ||
4054 Name.starts_with(
"avx512.mask3.vfmsubadd.p"))) {
4055 bool IsMask3 =
Name[11] ==
'3';
4056 bool IsMaskZ =
Name[11] ==
'z';
4058 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4059 bool IsSubAdd =
Name[3] ==
's';
4064 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4066 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4076 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
4091 for (
int i = 0; i != NumElts; ++i)
4092 Idxs[i] = i + (i % 2) * NumElts;
4102 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.pternlog.") ||
4103 Name.starts_with(
"avx512.maskz.pternlog."))) {
4104 bool ZeroMask =
Name[11] ==
'z';
4108 if (VecWidth == 128 && EltWidth == 32)
4109 IID = Intrinsic::x86_avx512_pternlog_d_128;
4110 else if (VecWidth == 256 && EltWidth == 32)
4111 IID = Intrinsic::x86_avx512_pternlog_d_256;
4112 else if (VecWidth == 512 && EltWidth == 32)
4113 IID = Intrinsic::x86_avx512_pternlog_d_512;
4114 else if (VecWidth == 128 && EltWidth == 64)
4115 IID = Intrinsic::x86_avx512_pternlog_q_128;
4116 else if (VecWidth == 256 && EltWidth == 64)
4117 IID = Intrinsic::x86_avx512_pternlog_q_256;
4118 else if (VecWidth == 512 && EltWidth == 64)
4119 IID = Intrinsic::x86_avx512_pternlog_q_512;
4130 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpmadd52") ||
4131 Name.starts_with(
"avx512.maskz.vpmadd52"))) {
4132 bool ZeroMask =
Name[11] ==
'z';
4136 if (VecWidth == 128 && !
High)
4137 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4138 else if (VecWidth == 256 && !
High)
4139 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4140 else if (VecWidth == 512 && !
High)
4141 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4142 else if (VecWidth == 128 &&
High)
4143 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4144 else if (VecWidth == 256 &&
High)
4145 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4146 else if (VecWidth == 512 &&
High)
4147 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4158 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpermi2var.") ||
4159 Name.starts_with(
"avx512.mask.vpermt2var.") ||
4160 Name.starts_with(
"avx512.maskz.vpermt2var."))) {
4161 bool ZeroMask =
Name[11] ==
'z';
4162 bool IndexForm =
Name[17] ==
'i';
4164 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpdpbusd.") ||
4165 Name.starts_with(
"avx512.maskz.vpdpbusd.") ||
4166 Name.starts_with(
"avx512.mask.vpdpbusds.") ||
4167 Name.starts_with(
"avx512.maskz.vpdpbusds."))) {
4168 bool ZeroMask =
Name[11] ==
'z';
4169 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
4172 if (VecWidth == 128 && !IsSaturating)
4173 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4174 else if (VecWidth == 256 && !IsSaturating)
4175 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4176 else if (VecWidth == 512 && !IsSaturating)
4177 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4178 else if (VecWidth == 128 && IsSaturating)
4179 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4180 else if (VecWidth == 256 && IsSaturating)
4181 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4182 else if (VecWidth == 512 && IsSaturating)
4183 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4194 }
else if (IsX86 && (
Name.starts_with(
"avx512.mask.vpdpwssd.") ||
4195 Name.starts_with(
"avx512.maskz.vpdpwssd.") ||
4196 Name.starts_with(
"avx512.mask.vpdpwssds.") ||
4197 Name.starts_with(
"avx512.maskz.vpdpwssds."))) {
4198 bool ZeroMask =
Name[11] ==
'z';
4199 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
4202 if (VecWidth == 128 && !IsSaturating)
4203 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4204 else if (VecWidth == 256 && !IsSaturating)
4205 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4206 else if (VecWidth == 512 && !IsSaturating)
4207 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4208 else if (VecWidth == 128 && IsSaturating)
4209 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4210 else if (VecWidth == 256 && IsSaturating)
4211 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4212 else if (VecWidth == 512 && IsSaturating)
4213 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4224 }
else if (IsX86 && (
Name ==
"addcarryx.u32" ||
Name ==
"addcarryx.u64" ||
4225 Name ==
"addcarry.u32" ||
Name ==
"addcarry.u64" ||
4226 Name ==
"subborrow.u32" ||
Name ==
"subborrow.u64")) {
4228 if (
Name[0] ==
'a' &&
Name.back() ==
'2')
4229 IID = Intrinsic::x86_addcarry_32;
4230 else if (
Name[0] ==
'a' &&
Name.back() ==
'4')
4231 IID = Intrinsic::x86_addcarry_64;
4232 else if (
Name[0] ==
's' &&
Name.back() ==
'2')
4233 IID = Intrinsic::x86_subborrow_32;
4234 else if (
Name[0] ==
's' &&
Name.back() ==
'4')
4235 IID = Intrinsic::x86_subborrow_64;
4257 }
else if (IsX86 &&
Name.starts_with(
"avx512.mask.") &&
4260 }
else if (IsNVVM && (
Name ==
"abs.i" ||
Name ==
"abs.ll")) {
4266 }
else if (IsNVVM && (
Name.starts_with(
"atomic.load.add.f32.p") ||
4267 Name.starts_with(
"atomic.load.add.f64.p"))) {
4271 AtomicOrdering::SequentiallyConsistent);
4272 }
else if (IsNVVM &&
Name.consume_front(
"max.") &&
4281 }
else if (IsNVVM &&
Name.consume_front(
"min.") &&
4290 }
else if (IsNVVM &&
Name ==
"clz.ll") {
4296 {Arg, Builder.getFalse()},
"ctlz");
4298 }
else if (IsNVVM &&
Name ==
"popc.ll") {
4307 }
else if (IsNVVM) {
4308 if (
Name ==
"h2f") {
4311 F->getParent(), Intrinsic::convert_from_fp16,
4312 {Builder.getFloatTy()}),
4317 !
F->getReturnType()->getScalarType()->isBFloatTy()) {
4331 if (
F->getReturnType()->isIntegerTy())
4337 }
else if (IsAMDGCN) {
4349 FallthroughToDefaultUpgrade =
true;
4357 if (!FallthroughToDefaultUpgrade) {
4365 const auto &DefaultCase = [&]() ->
void {
4370 "Unknown function for CallBase upgrade and isn't just a name change");
4376 if (
auto *OldST = dyn_cast<StructType>(CI->
getType())) {
4378 "Return type must have changed");
4379 assert(OldST->getNumElements() ==
4380 cast<StructType>(NewFn->
getReturnType())->getNumElements() &&
4381 "Must have same number of elements");
4386 for (
unsigned Idx = 0;
Idx < OldST->getNumElements(); ++
Idx) {
4407 case Intrinsic::arm_neon_vst1:
4408 case Intrinsic::arm_neon_vst2:
4409 case Intrinsic::arm_neon_vst3:
4410 case Intrinsic::arm_neon_vst4:
4411 case Intrinsic::arm_neon_vst2lane:
4412 case Intrinsic::arm_neon_vst3lane:
4413 case Intrinsic::arm_neon_vst4lane: {
4418 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
4419 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
4420 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
4424 cast<ConstantInt>(Args[3])->getZExtValue());
4428 case Intrinsic::aarch64_sve_ld3_sret:
4429 case Intrinsic::aarch64_sve_ld4_sret:
4430 case Intrinsic::aarch64_sve_ld2_sret: {
4438 auto *
RetTy = cast<ScalableVectorType>(
F->getReturnType());
4439 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4443 for (
unsigned I = 0;
I <
N;
I++) {
4448 NewCall = dyn_cast<CallInst>(Ret);
4452 case Intrinsic::coro_end: {
4459 case Intrinsic::vector_extract: {
4462 if (!
Name.starts_with(
"aarch64.sve.tuple.get")) {
4466 auto *
RetTy = cast<ScalableVectorType>(
F->getReturnType());
4467 unsigned MinElts =
RetTy->getMinNumElements();
4468 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4474 case Intrinsic::vector_insert: {
4477 if (!
Name.starts_with(
"aarch64.sve.tuple")) {
4481 if (
Name.starts_with(
"aarch64.sve.tuple.set")) {
4482 unsigned I = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
4490 if (
Name.starts_with(
"aarch64.sve.tuple.create")) {
4496 assert(
N > 1 &&
"Create is expected to be between 2-4");
4497 auto *
RetTy = cast<ScalableVectorType>(
F->getReturnType());
4499 unsigned MinElts =
RetTy->getMinNumElements() /
N;
4500 for (
unsigned I = 0;
I <
N;
I++) {
4505 NewCall = dyn_cast<CallInst>(Ret);
4510 case Intrinsic::arm_neon_bfdot:
4511 case Intrinsic::arm_neon_bfmmla:
4512 case Intrinsic::arm_neon_bfmlalb:
4513 case Intrinsic::arm_neon_bfmlalt:
4514 case Intrinsic::aarch64_neon_bfdot:
4515 case Intrinsic::aarch64_neon_bfmmla:
4516 case Intrinsic::aarch64_neon_bfmlalb:
4517 case Intrinsic::aarch64_neon_bfmlalt: {
4520 "Mismatch between function args and call args");
4521 size_t OperandWidth =
4523 assert((OperandWidth == 64 || OperandWidth == 128) &&
4524 "Unexpected operand width");
4526 auto Iter = CI->
args().begin();
4527 Args.push_back(*Iter++);
4534 case Intrinsic::bitreverse:
4538 case Intrinsic::ctlz:
4539 case Intrinsic::cttz:
4541 "Mismatch between function args and call args");
4546 case Intrinsic::objectsize: {
4547 Value *NullIsUnknownSize =
4556 case Intrinsic::ctpop:
4560 case Intrinsic::convert_from_fp16:
4564 case Intrinsic::dbg_value: {
4568 if (
Name.starts_with(
"dbg.addr")) {
4570 cast<MetadataAsValue>(CI->
getArgOperand(2))->getMetadata());
4582 if (
Offset->isZeroValue()) {
4592 case Intrinsic::ptr_annotation:
4609 case Intrinsic::var_annotation:
4625 case Intrinsic::riscv_aes32dsi:
4626 case Intrinsic::riscv_aes32dsmi:
4627 case Intrinsic::riscv_aes32esi:
4628 case Intrinsic::riscv_aes32esmi:
4629 case Intrinsic::riscv_sm4ks:
4630 case Intrinsic::riscv_sm4ed: {
4645 cast<ConstantInt>(Arg2)->getZExtValue());
4647 NewCall = Builder.
CreateCall(NewFn, {Arg0, Arg1, Arg2});
4648 Value *Res = NewCall;
4656 case Intrinsic::riscv_sha256sig0:
4657 case Intrinsic::riscv_sha256sig1:
4658 case Intrinsic::riscv_sha256sum0:
4659 case Intrinsic::riscv_sha256sum1:
4660 case Intrinsic::riscv_sm3p0:
4661 case Intrinsic::riscv_sm3p1: {
4679 case Intrinsic::x86_xop_vfrcz_ss:
4680 case Intrinsic::x86_xop_vfrcz_sd:
4684 case Intrinsic::x86_xop_vpermil2pd:
4685 case Intrinsic::x86_xop_vpermil2ps:
4686 case Intrinsic::x86_xop_vpermil2pd_256:
4687 case Intrinsic::x86_xop_vpermil2ps_256: {
4690 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
4696 case Intrinsic::x86_sse41_ptestc:
4697 case Intrinsic::x86_sse41_ptestz:
4698 case Intrinsic::x86_sse41_ptestnzc: {
4715 NewCall = Builder.
CreateCall(NewFn, {BC0, BC1});
4719 case Intrinsic::x86_rdtscp: {
4741 case Intrinsic::x86_sse41_insertps:
4742 case Intrinsic::x86_sse41_dppd:
4743 case Intrinsic::x86_sse41_dpps:
4744 case Intrinsic::x86_sse41_mpsadbw:
4745 case Intrinsic::x86_avx_dp_ps_256:
4746 case Intrinsic::x86_avx2_mpsadbw: {
4757 case Intrinsic::x86_avx512_mask_cmp_pd_128:
4758 case Intrinsic::x86_avx512_mask_cmp_pd_256:
4759 case Intrinsic::x86_avx512_mask_cmp_pd_512:
4760 case Intrinsic::x86_avx512_mask_cmp_ps_128:
4761 case Intrinsic::x86_avx512_mask_cmp_ps_256:
4762 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
4765 cast<FixedVectorType>(Args[0]->
getType())->getNumElements();
4777 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
4778 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
4779 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
4780 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
4781 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
4782 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
4784 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
4786 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
4799 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
4800 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
4801 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
4804 cast<FixedVectorType>(CI->
getType())->getNumElements() * 2;
4814 case Intrinsic::thread_pointer: {
4819 case Intrinsic::memcpy:
4820 case Intrinsic::memmove:
4821 case Intrinsic::memset: {
4841 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
4842 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
4844 auto *MemCI = cast<MemIntrinsic>(NewCall);
4847 MemCI->setDestAlignment(
Align->getMaybeAlignValue());
4849 if (
auto *MTI = dyn_cast<MemTransferInst>(MemCI))
4850 MTI->setSourceAlignment(
Align->getMaybeAlignValue());
4854 assert(NewCall &&
"Should have either set this variable or returned through "
4855 "the default case");
4862 assert(
F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
4871 if (
CallBase *CB = dyn_cast<CallBase>(U))
4875 F->eraseFromParent();
4881 if (NumOperands == 0)
4885 if (isa<MDNode>(MD.
getOperand(0)) && NumOperands >= 3)
4889 if (NumOperands == 3) {
4893 Metadata *Elts2[] = {ScalarType, ScalarType,
4907 if (Opc != Instruction::BitCast)
4911 Type *SrcTy = V->getType();
4928 if (Opc != Instruction::BitCast)
4931 Type *SrcTy =
C->getType();
4955 bool BrokenDebugInfo =
false;
4958 if (!BrokenDebugInfo)
4964 M.getContext().diagnose(Diag);
4971 M.getContext().diagnose(DiagVersion);
4979 bool Changed =
false;
4980 const char *MarkerKey =
"clang.arc.retainAutoreleasedReturnValueMarker";
4981 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4982 if (ModRetainReleaseMarker) {
4985 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(0));
4988 ID->getString().split(ValueComp,
"#");
4989 if (ValueComp.
size() == 2) {
4990 std::string NewValue = ValueComp[0].str() +
";" + ValueComp[1].str();
4994 M.eraseNamedMetadata(ModRetainReleaseMarker);
5005 auto UpgradeToIntrinsic = [&](
const char *OldFunc,
5007 Function *Fn = M.getFunction(OldFunc);
5015 CallInst *CI = dyn_cast<CallInst>(U);
5030 bool InvalidCast =
false;
5032 for (
unsigned I = 0, E = CI->
arg_size();
I != E; ++
I) {
5037 if (I < NewFuncTy->getNumParams()) {
5047 Args.push_back(Arg);
5072 UpgradeToIntrinsic(
"clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
5080 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
5081 {
"objc_autorelease", llvm::Intrinsic::objc_autorelease},
5082 {
"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
5083 {
"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
5084 {
"objc_autoreleaseReturnValue",
5085 llvm::Intrinsic::objc_autoreleaseReturnValue},
5086 {
"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
5087 {
"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
5088 {
"objc_initWeak", llvm::Intrinsic::objc_initWeak},
5089 {
"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
5090 {
"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
5091 {
"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
5092 {
"objc_release", llvm::Intrinsic::objc_release},
5093 {
"objc_retain", llvm::Intrinsic::objc_retain},
5094 {
"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
5095 {
"objc_retainAutoreleaseReturnValue",
5096 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
5097 {
"objc_retainAutoreleasedReturnValue",
5098 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
5099 {
"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
5100 {
"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
5101 {
"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
5102 {
"objc_unsafeClaimAutoreleasedReturnValue",
5103 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
5104 {
"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
5105 {
"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
5106 {
"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
5107 {
"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
5108 {
"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
5109 {
"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
5110 {
"objc_arc_annotation_topdown_bbstart",
5111 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
5112 {
"objc_arc_annotation_topdown_bbend",
5113 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
5114 {
"objc_arc_annotation_bottomup_bbstart",
5115 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
5116 {
"objc_arc_annotation_bottomup_bbend",
5117 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
5119 for (
auto &
I : RuntimeFuncs)
5120 UpgradeToIntrinsic(
I.first,
I.second);
5124 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
5128 bool HasObjCFlag =
false, HasClassProperties =
false, Changed =
false;
5129 bool HasSwiftVersionFlag =
false;
5130 uint8_t SwiftMajorVersion, SwiftMinorVersion;
5139 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(1));
5151 if (
ID->getString() ==
"Objective-C Image Info Version")
5153 if (
ID->getString() ==
"Objective-C Class Properties")
5154 HasClassProperties =
true;
5156 if (
ID->getString() ==
"PIC Level") {
5157 if (
auto *Behavior =
5158 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5159 uint64_t V = Behavior->getLimitedValue();
5165 if (
ID->getString() ==
"PIE Level")
5166 if (
auto *Behavior =
5167 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0)))
5173 if (
ID->getString() ==
"branch-target-enforcement" ||
5174 ID->getString().starts_with(
"sign-return-address")) {
5175 if (
auto *Behavior =
5176 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
5181 Op->getOperand(1),
Op->getOperand(2)};
5191 if (
ID->getString() ==
"Objective-C Image Info Section") {
5192 if (
auto *
Value = dyn_cast_or_null<MDString>(
Op->getOperand(2))) {
5194 Value->getString().split(ValueComp,
" ");
5195 if (ValueComp.
size() != 1) {
5196 std::string NewValue;
5197 for (
auto &S : ValueComp)
5198 NewValue += S.str();
5199 Metadata *Ops[3] = {
Op->getOperand(0),
Op->getOperand(1),
5209 if (
ID->getString() ==
"Objective-C Garbage Collection") {
5210 auto Md = dyn_cast<ConstantAsMetadata>(
Op->getOperand(2));
5212 assert(Md->getValue() &&
"Expected non-empty metadata");
5213 auto Type = Md->getValue()->getType();
5216 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
5217 if ((Val & 0xff) != Val) {
5218 HasSwiftVersionFlag =
true;
5219 SwiftABIVersion = (Val & 0xff00) >> 8;
5220 SwiftMajorVersion = (Val & 0xff000000) >> 24;
5221 SwiftMinorVersion = (Val & 0xff0000) >> 16;
5232 if (
ID->getString() ==
"amdgpu_code_object_version") {
5235 MDString::get(M.getContext(),
"amdhsa_code_object_version"),
5247 if (HasObjCFlag && !HasClassProperties) {
5253 if (HasSwiftVersionFlag) {
5257 ConstantInt::get(Int8Ty, SwiftMajorVersion));
5259 ConstantInt::get(Int8Ty, SwiftMinorVersion));
5267 auto TrimSpaces = [](
StringRef Section) -> std::string {
5269 Section.split(Components,
',');
5274 for (
auto Component : Components)
5275 OS <<
',' << Component.trim();
5277 return std::string(
OS.str().substr(1));
5280 for (
auto &GV : M.globals()) {
5281 if (!GV.hasSection())
5286 if (!Section.starts_with(
"__DATA, __objc_catlist"))
5291 GV.setSection(TrimSpaces(Section));
5307struct StrictFPUpgradeVisitor :
public InstVisitor<StrictFPUpgradeVisitor> {
5308 StrictFPUpgradeVisitor() =
default;
5311 if (!Call.isStrictFP())
5313 if (isa<ConstrainedFPIntrinsic>(&Call))
5317 Call.removeFnAttr(Attribute::StrictFP);
5318 Call.addFnAttr(Attribute::NoBuiltin);
5326 if (!
F.isDeclaration() && !
F.hasFnAttribute(Attribute::StrictFP)) {
5327 StrictFPUpgradeVisitor SFPV;
5333 for (
auto &Arg :
F.args())
5338 if (
Attribute A =
F.getFnAttribute(
"implicit-section-name");
5339 A.isValid() &&
A.isStringAttribute()) {
5340 F.setSection(
A.getValueAsString());
5341 F.removeFnAttr(
"implicit-section-name");
5346 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5349 if (
T->getNumOperands() < 1)
5351 auto *S = dyn_cast_or_null<MDString>(
T->getOperand(0));
5354 return S->getString().starts_with(
"llvm.vectorizer.");
5358 StringRef OldPrefix =
"llvm.vectorizer.";
5361 if (OldTag ==
"llvm.vectorizer.unroll")
5370 auto *
T = dyn_cast_or_null<MDTuple>(MD);
5373 if (
T->getNumOperands() < 1)
5375 auto *OldTag = dyn_cast_or_null<MDString>(
T->getOperand(0));
5378 if (!OldTag->getString().starts_with(
"llvm.vectorizer."))
5385 for (
unsigned I = 1, E =
T->getNumOperands();
I != E; ++
I)
5392 auto *
T = dyn_cast<MDTuple>(&
N);
5411 if (((
T.isAMDGPU() && !
T.isAMDGCN()) ||
5412 (
T.isSPIR() || (
T.isSPIRV() && !
T.isSPIRVLogical()))) &&
5413 !
DL.contains(
"-G") && !
DL.starts_with(
"G")) {
5414 return DL.
empty() ? std::string(
"G1") : (
DL +
"-G1").str();
5417 if (
T.isLoongArch64() ||
T.isRISCV64()) {
5419 auto I =
DL.find(
"-n64-");
5421 return (
DL.take_front(
I) +
"-n32:64-" +
DL.drop_front(
I + 5)).str();
5425 std::string Res =
DL.str();
5429 if (!
DL.contains(
"-G") && !
DL.starts_with(
"G"))
5430 Res.append(Res.empty() ?
"G1" :
"-G1");
5435 if (!
DL.contains(
"-ni") && !
DL.starts_with(
"ni"))
5436 Res.append(
"-ni:7:8:9");
5438 if (
DL.ends_with(
"ni:7"))
5440 if (
DL.ends_with(
"ni:7:8"))
5445 if (!
DL.contains(
"-p7") && !
DL.starts_with(
"p7"))
5446 Res.append(
"-p7:160:256:256:32");
5447 if (!
DL.contains(
"-p8") && !
DL.starts_with(
"p8"))
5448 Res.append(
"-p8:128:128");
5449 if (!
DL.contains(
"-p9") && !
DL.starts_with(
"p9"))
5450 Res.append(
"-p9:192:256:256:32");
5456 if (
T.isAArch64()) {
5458 if (!
DL.
empty() && !
DL.contains(
"-Fn32"))
5459 Res.append(
"-Fn32");
5468 std::string AddrSpaces =
"-p270:32:32-p271:32:32-p272:64:64";
5471 Regex R(
"(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
5472 if (R.match(Res, &
Groups))
5482 if (!
T.isOSIAMCU()) {
5483 std::string I128 =
"-i128:128";
5486 Regex R(
"^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
5487 if (R.match(Res, &
Groups))
5495 if (
T.isWindowsMSVCEnvironment() && !
T.isArch64Bit()) {
5497 auto I =
Ref.find(
"-f80:32-");
5499 Res = (
Ref.take_front(
I) +
"-f80:128-" +
Ref.drop_front(
I + 8)).str();
5507 Attribute A =
B.getAttribute(
"no-frame-pointer-elim");
5510 FramePointer =
A.getValueAsString() ==
"true" ?
"all" :
"none";
5511 B.removeAttribute(
"no-frame-pointer-elim");
5513 if (
B.contains(
"no-frame-pointer-elim-non-leaf")) {
5515 if (FramePointer !=
"all")
5516 FramePointer =
"non-leaf";
5517 B.removeAttribute(
"no-frame-pointer-elim-non-leaf");
5519 if (!FramePointer.
empty())
5520 B.addAttribute(
"frame-pointer", FramePointer);
5522 A =
B.getAttribute(
"null-pointer-is-valid");
5525 bool NullPointerIsValid =
A.getValueAsString() ==
"true";
5526 B.removeAttribute(
"null-pointer-is-valid");
5527 if (NullPointerIsValid)
5528 B.addAttribute(Attribute::NullPointerIsValid);
5538 return OBD.
getTag() ==
"clang.arc.attachedcall" &&
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static Metadata * upgradeLoopArgument(Metadata *MD)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static MDType * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Class to represent array types.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the parameter attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static ConstantAggregateZero * get(Type *Ty)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
LinkageTypes getLinkage() const
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateFSub(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateFDiv(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
CallInst * CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec, Value *Idx, const Twine &Name="")
Create a call to the vector.insert intrinsic.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
Value * CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Type * getBFloatTy()
Fetch the type representing a 16-bit brain floating point value.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
RetTy visitCallBase(CallBase &I)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
This is an important class for using LLVM in a threaded context.
SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
LLVMContext & getContext() const
static MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
@ Min
Takes the min of the two values, which are required to be integers.
@ Max
Takes the max of the two values, which are required to be integers.
bool IsNewDbgInfoFormat
Is this Module using intrinsics to record the position of debugging information, or non-intrinsic rec...
void setOperand(unsigned I, MDNode *New)
MDNode * getOperand(unsigned i) const
unsigned getNumOperands() const
A container for an operand bundle being viewed as a set of values rather than a set of uses.
ArrayRef< InputTy > inputs() const
Class to represent pointers.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
ArrayRef< int > getShuffleMask() const
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
empty - Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
size - Get the string size.
static constexpr size_t npos
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getBFloatTy(LLVMContext &C)
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
self_iterator getIterator()
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
AttributeMask typeIncompatible(Type *Ty, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)
Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.
std::optional< Function * > remangleIntrinsicFunction(Function *F)
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
This is an optimization pass for GlobalISel generic memory operations.
void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
unsigned getDebugMetadataVersionFromModule(const Module &M)
Return Debug Info Metadata Version by checking module flags.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
@ Dynamic
Denotes mode unknown at compile time.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.