28 #include "llvm/IR/IntrinsicsAArch64.h"
29 #include "llvm/IR/IntrinsicsARM.h"
30 #include "llvm/IR/IntrinsicsX86.h"
47 Type *Arg0Type =
F->getFunctionType()->getParamType(0);
62 Type *LastArgType =
F->getFunctionType()->getParamType(
63 F->getFunctionType()->getNumParams() - 1);
78 if (
F->getReturnType()->isVectorTy())
92 if (
Name ==
"addcarryx.u32" ||
93 Name ==
"addcarryx.u64" ||
94 Name ==
"addcarry.u32" ||
95 Name ==
"addcarry.u64" ||
96 Name ==
"subborrow.u32" ||
97 Name ==
"subborrow.u64" ||
98 Name.startswith(
"sse2.padds.") ||
99 Name.startswith(
"sse2.psubs.") ||
100 Name.startswith(
"sse2.paddus.") ||
101 Name.startswith(
"sse2.psubus.") ||
102 Name.startswith(
"avx2.padds.") ||
103 Name.startswith(
"avx2.psubs.") ||
104 Name.startswith(
"avx2.paddus.") ||
105 Name.startswith(
"avx2.psubus.") ||
106 Name.startswith(
"avx512.padds.") ||
107 Name.startswith(
"avx512.psubs.") ||
108 Name.startswith(
"avx512.mask.padds.") ||
109 Name.startswith(
"avx512.mask.psubs.") ||
110 Name.startswith(
"avx512.mask.paddus.") ||
111 Name.startswith(
"avx512.mask.psubus.") ||
112 Name==
"ssse3.pabs.b.128" ||
113 Name==
"ssse3.pabs.w.128" ||
114 Name==
"ssse3.pabs.d.128" ||
115 Name.startswith(
"fma4.vfmadd.s") ||
116 Name.startswith(
"fma.vfmadd.") ||
117 Name.startswith(
"fma.vfmsub.") ||
118 Name.startswith(
"fma.vfmsubadd.") ||
119 Name.startswith(
"fma.vfnmadd.") ||
120 Name.startswith(
"fma.vfnmsub.") ||
121 Name.startswith(
"avx512.mask.vfmadd.") ||
122 Name.startswith(
"avx512.mask.vfnmadd.") ||
123 Name.startswith(
"avx512.mask.vfnmsub.") ||
124 Name.startswith(
"avx512.mask3.vfmadd.") ||
125 Name.startswith(
"avx512.maskz.vfmadd.") ||
126 Name.startswith(
"avx512.mask3.vfmsub.") ||
127 Name.startswith(
"avx512.mask3.vfnmsub.") ||
128 Name.startswith(
"avx512.mask.vfmaddsub.") ||
129 Name.startswith(
"avx512.maskz.vfmaddsub.") ||
130 Name.startswith(
"avx512.mask3.vfmaddsub.") ||
131 Name.startswith(
"avx512.mask3.vfmsubadd.") ||
132 Name.startswith(
"avx512.mask.shuf.i") ||
133 Name.startswith(
"avx512.mask.shuf.f") ||
134 Name.startswith(
"avx512.kunpck") ||
135 Name.startswith(
"avx2.pabs.") ||
136 Name.startswith(
"avx512.mask.pabs.") ||
137 Name.startswith(
"avx512.broadcastm") ||
138 Name ==
"sse.sqrt.ss" ||
139 Name ==
"sse2.sqrt.sd" ||
140 Name.startswith(
"avx512.mask.sqrt.p") ||
141 Name.startswith(
"avx.sqrt.p") ||
142 Name.startswith(
"sse2.sqrt.p") ||
143 Name.startswith(
"sse.sqrt.p") ||
144 Name.startswith(
"avx512.mask.pbroadcast") ||
145 Name.startswith(
"sse2.pcmpeq.") ||
146 Name.startswith(
"sse2.pcmpgt.") ||
147 Name.startswith(
"avx2.pcmpeq.") ||
148 Name.startswith(
"avx2.pcmpgt.") ||
149 Name.startswith(
"avx512.mask.pcmpeq.") ||
150 Name.startswith(
"avx512.mask.pcmpgt.") ||
151 Name.startswith(
"avx.vperm2f128.") ||
152 Name ==
"avx2.vperm2i128" ||
153 Name ==
"sse.add.ss" ||
154 Name ==
"sse2.add.sd" ||
155 Name ==
"sse.sub.ss" ||
156 Name ==
"sse2.sub.sd" ||
157 Name ==
"sse.mul.ss" ||
158 Name ==
"sse2.mul.sd" ||
159 Name ==
"sse.div.ss" ||
160 Name ==
"sse2.div.sd" ||
161 Name ==
"sse41.pmaxsb" ||
162 Name ==
"sse2.pmaxs.w" ||
163 Name ==
"sse41.pmaxsd" ||
164 Name ==
"sse2.pmaxu.b" ||
165 Name ==
"sse41.pmaxuw" ||
166 Name ==
"sse41.pmaxud" ||
167 Name ==
"sse41.pminsb" ||
168 Name ==
"sse2.pmins.w" ||
169 Name ==
"sse41.pminsd" ||
170 Name ==
"sse2.pminu.b" ||
171 Name ==
"sse41.pminuw" ||
172 Name ==
"sse41.pminud" ||
173 Name ==
"avx512.kand.w" ||
174 Name ==
"avx512.kandn.w" ||
175 Name ==
"avx512.knot.w" ||
176 Name ==
"avx512.kor.w" ||
177 Name ==
"avx512.kxor.w" ||
178 Name ==
"avx512.kxnor.w" ||
179 Name ==
"avx512.kortestc.w" ||
180 Name ==
"avx512.kortestz.w" ||
181 Name.startswith(
"avx512.mask.pshuf.b.") ||
182 Name.startswith(
"avx2.pmax") ||
183 Name.startswith(
"avx2.pmin") ||
184 Name.startswith(
"avx512.mask.pmax") ||
185 Name.startswith(
"avx512.mask.pmin") ||
186 Name.startswith(
"avx2.vbroadcast") ||
187 Name.startswith(
"avx2.pbroadcast") ||
188 Name.startswith(
"avx.vpermil.") ||
189 Name.startswith(
"sse2.pshuf") ||
190 Name.startswith(
"avx512.pbroadcast") ||
191 Name.startswith(
"avx512.mask.broadcast.s") ||
192 Name.startswith(
"avx512.mask.movddup") ||
193 Name.startswith(
"avx512.mask.movshdup") ||
194 Name.startswith(
"avx512.mask.movsldup") ||
195 Name.startswith(
"avx512.mask.pshuf.d.") ||
196 Name.startswith(
"avx512.mask.pshufl.w.") ||
197 Name.startswith(
"avx512.mask.pshufh.w.") ||
198 Name.startswith(
"avx512.mask.shuf.p") ||
199 Name.startswith(
"avx512.mask.vpermil.p") ||
200 Name.startswith(
"avx512.mask.perm.df.") ||
201 Name.startswith(
"avx512.mask.perm.di.") ||
202 Name.startswith(
"avx512.mask.punpckl") ||
203 Name.startswith(
"avx512.mask.punpckh") ||
204 Name.startswith(
"avx512.mask.unpckl.") ||
205 Name.startswith(
"avx512.mask.unpckh.") ||
206 Name.startswith(
"avx512.mask.pand.") ||
207 Name.startswith(
"avx512.mask.pandn.") ||
208 Name.startswith(
"avx512.mask.por.") ||
209 Name.startswith(
"avx512.mask.pxor.") ||
210 Name.startswith(
"avx512.mask.and.") ||
211 Name.startswith(
"avx512.mask.andn.") ||
212 Name.startswith(
"avx512.mask.or.") ||
213 Name.startswith(
"avx512.mask.xor.") ||
214 Name.startswith(
"avx512.mask.padd.") ||
215 Name.startswith(
"avx512.mask.psub.") ||
216 Name.startswith(
"avx512.mask.pmull.") ||
217 Name.startswith(
"avx512.mask.cvtdq2pd.") ||
218 Name.startswith(
"avx512.mask.cvtudq2pd.") ||
219 Name.startswith(
"avx512.mask.cvtudq2ps.") ||
220 Name.startswith(
"avx512.mask.cvtqq2pd.") ||
221 Name.startswith(
"avx512.mask.cvtuqq2pd.") ||
222 Name.startswith(
"avx512.mask.cvtdq2ps.") ||
223 Name ==
"avx512.mask.vcvtph2ps.128" ||
224 Name ==
"avx512.mask.vcvtph2ps.256" ||
225 Name ==
"avx512.mask.cvtqq2ps.256" ||
226 Name ==
"avx512.mask.cvtqq2ps.512" ||
227 Name ==
"avx512.mask.cvtuqq2ps.256" ||
228 Name ==
"avx512.mask.cvtuqq2ps.512" ||
229 Name ==
"avx512.mask.cvtpd2dq.256" ||
230 Name ==
"avx512.mask.cvtpd2ps.256" ||
231 Name ==
"avx512.mask.cvttpd2dq.256" ||
232 Name ==
"avx512.mask.cvttps2dq.128" ||
233 Name ==
"avx512.mask.cvttps2dq.256" ||
234 Name ==
"avx512.mask.cvtps2pd.128" ||
235 Name ==
"avx512.mask.cvtps2pd.256" ||
236 Name ==
"avx512.cvtusi2sd" ||
237 Name.startswith(
"avx512.mask.permvar.") ||
238 Name ==
"sse2.pmulu.dq" ||
239 Name ==
"sse41.pmuldq" ||
240 Name ==
"avx2.pmulu.dq" ||
241 Name ==
"avx2.pmul.dq" ||
242 Name ==
"avx512.pmulu.dq.512" ||
243 Name ==
"avx512.pmul.dq.512" ||
244 Name.startswith(
"avx512.mask.pmul.dq.") ||
245 Name.startswith(
"avx512.mask.pmulu.dq.") ||
246 Name.startswith(
"avx512.mask.pmul.hr.sw.") ||
247 Name.startswith(
"avx512.mask.pmulh.w.") ||
248 Name.startswith(
"avx512.mask.pmulhu.w.") ||
249 Name.startswith(
"avx512.mask.pmaddw.d.") ||
250 Name.startswith(
"avx512.mask.pmaddubs.w.") ||
251 Name.startswith(
"avx512.mask.packsswb.") ||
252 Name.startswith(
"avx512.mask.packssdw.") ||
253 Name.startswith(
"avx512.mask.packuswb.") ||
254 Name.startswith(
"avx512.mask.packusdw.") ||
255 Name.startswith(
"avx512.mask.cmp.b") ||
256 Name.startswith(
"avx512.mask.cmp.d") ||
257 Name.startswith(
"avx512.mask.cmp.q") ||
258 Name.startswith(
"avx512.mask.cmp.w") ||
259 Name.startswith(
"avx512.cmp.p") ||
260 Name.startswith(
"avx512.mask.ucmp.") ||
261 Name.startswith(
"avx512.cvtb2mask.") ||
262 Name.startswith(
"avx512.cvtw2mask.") ||
263 Name.startswith(
"avx512.cvtd2mask.") ||
264 Name.startswith(
"avx512.cvtq2mask.") ||
265 Name.startswith(
"avx512.mask.vpermilvar.") ||
266 Name.startswith(
"avx512.mask.psll.d") ||
267 Name.startswith(
"avx512.mask.psll.q") ||
268 Name.startswith(
"avx512.mask.psll.w") ||
269 Name.startswith(
"avx512.mask.psra.d") ||
270 Name.startswith(
"avx512.mask.psra.q") ||
271 Name.startswith(
"avx512.mask.psra.w") ||
272 Name.startswith(
"avx512.mask.psrl.d") ||
273 Name.startswith(
"avx512.mask.psrl.q") ||
274 Name.startswith(
"avx512.mask.psrl.w") ||
275 Name.startswith(
"avx512.mask.pslli") ||
276 Name.startswith(
"avx512.mask.psrai") ||
277 Name.startswith(
"avx512.mask.psrli") ||
278 Name.startswith(
"avx512.mask.psllv") ||
279 Name.startswith(
"avx512.mask.psrav") ||
280 Name.startswith(
"avx512.mask.psrlv") ||
281 Name.startswith(
"sse41.pmovsx") ||
282 Name.startswith(
"sse41.pmovzx") ||
283 Name.startswith(
"avx2.pmovsx") ||
284 Name.startswith(
"avx2.pmovzx") ||
285 Name.startswith(
"avx512.mask.pmovsx") ||
286 Name.startswith(
"avx512.mask.pmovzx") ||
287 Name.startswith(
"avx512.mask.lzcnt.") ||
288 Name.startswith(
"avx512.mask.pternlog.") ||
289 Name.startswith(
"avx512.maskz.pternlog.") ||
290 Name.startswith(
"avx512.mask.vpmadd52") ||
291 Name.startswith(
"avx512.maskz.vpmadd52") ||
292 Name.startswith(
"avx512.mask.vpermi2var.") ||
293 Name.startswith(
"avx512.mask.vpermt2var.") ||
294 Name.startswith(
"avx512.maskz.vpermt2var.") ||
295 Name.startswith(
"avx512.mask.vpdpbusd.") ||
296 Name.startswith(
"avx512.maskz.vpdpbusd.") ||
297 Name.startswith(
"avx512.mask.vpdpbusds.") ||
298 Name.startswith(
"avx512.maskz.vpdpbusds.") ||
299 Name.startswith(
"avx512.mask.vpdpwssd.") ||
300 Name.startswith(
"avx512.maskz.vpdpwssd.") ||
301 Name.startswith(
"avx512.mask.vpdpwssds.") ||
302 Name.startswith(
"avx512.maskz.vpdpwssds.") ||
303 Name.startswith(
"avx512.mask.dbpsadbw.") ||
304 Name.startswith(
"avx512.mask.vpshld.") ||
305 Name.startswith(
"avx512.mask.vpshrd.") ||
306 Name.startswith(
"avx512.mask.vpshldv.") ||
307 Name.startswith(
"avx512.mask.vpshrdv.") ||
308 Name.startswith(
"avx512.maskz.vpshldv.") ||
309 Name.startswith(
"avx512.maskz.vpshrdv.") ||
310 Name.startswith(
"avx512.vpshld.") ||
311 Name.startswith(
"avx512.vpshrd.") ||
312 Name.startswith(
"avx512.mask.add.p") ||
313 Name.startswith(
"avx512.mask.sub.p") ||
314 Name.startswith(
"avx512.mask.mul.p") ||
315 Name.startswith(
"avx512.mask.div.p") ||
316 Name.startswith(
"avx512.mask.max.p") ||
317 Name.startswith(
"avx512.mask.min.p") ||
318 Name.startswith(
"avx512.mask.fpclass.p") ||
319 Name.startswith(
"avx512.mask.vpshufbitqmb.") ||
320 Name.startswith(
"avx512.mask.pmultishift.qb.") ||
321 Name.startswith(
"avx512.mask.conflict.") ||
322 Name ==
"avx512.mask.pmov.qd.256" ||
323 Name ==
"avx512.mask.pmov.qd.512" ||
324 Name ==
"avx512.mask.pmov.wb.256" ||
325 Name ==
"avx512.mask.pmov.wb.512" ||
326 Name ==
"sse.cvtsi2ss" ||
327 Name ==
"sse.cvtsi642ss" ||
328 Name ==
"sse2.cvtsi2sd" ||
329 Name ==
"sse2.cvtsi642sd" ||
330 Name ==
"sse2.cvtss2sd" ||
331 Name ==
"sse2.cvtdq2pd" ||
332 Name ==
"sse2.cvtdq2ps" ||
333 Name ==
"sse2.cvtps2pd" ||
334 Name ==
"avx.cvtdq2.pd.256" ||
335 Name ==
"avx.cvtdq2.ps.256" ||
336 Name ==
"avx.cvt.ps2.pd.256" ||
337 Name.startswith(
"vcvtph2ps.") ||
338 Name.startswith(
"avx.vinsertf128.") ||
339 Name ==
"avx2.vinserti128" ||
340 Name.startswith(
"avx512.mask.insert") ||
341 Name.startswith(
"avx.vextractf128.") ||
342 Name ==
"avx2.vextracti128" ||
343 Name.startswith(
"avx512.mask.vextract") ||
344 Name.startswith(
"sse4a.movnt.") ||
345 Name.startswith(
"avx.movnt.") ||
346 Name.startswith(
"avx512.storent.") ||
347 Name ==
"sse41.movntdqa" ||
348 Name ==
"avx2.movntdqa" ||
349 Name ==
"avx512.movntdqa" ||
350 Name ==
"sse2.storel.dq" ||
351 Name.startswith(
"sse.storeu.") ||
352 Name.startswith(
"sse2.storeu.") ||
353 Name.startswith(
"avx.storeu.") ||
354 Name.startswith(
"avx512.mask.storeu.") ||
355 Name.startswith(
"avx512.mask.store.p") ||
356 Name.startswith(
"avx512.mask.store.b.") ||
357 Name.startswith(
"avx512.mask.store.w.") ||
358 Name.startswith(
"avx512.mask.store.d.") ||
359 Name.startswith(
"avx512.mask.store.q.") ||
360 Name ==
"avx512.mask.store.ss" ||
361 Name.startswith(
"avx512.mask.loadu.") ||
362 Name.startswith(
"avx512.mask.load.") ||
363 Name.startswith(
"avx512.mask.expand.load.") ||
364 Name.startswith(
"avx512.mask.compress.store.") ||
365 Name.startswith(
"avx512.mask.expand.b") ||
366 Name.startswith(
"avx512.mask.expand.w") ||
367 Name.startswith(
"avx512.mask.expand.d") ||
368 Name.startswith(
"avx512.mask.expand.q") ||
369 Name.startswith(
"avx512.mask.expand.p") ||
370 Name.startswith(
"avx512.mask.compress.b") ||
371 Name.startswith(
"avx512.mask.compress.w") ||
372 Name.startswith(
"avx512.mask.compress.d") ||
373 Name.startswith(
"avx512.mask.compress.q") ||
374 Name.startswith(
"avx512.mask.compress.p") ||
375 Name ==
"sse42.crc32.64.8" ||
376 Name.startswith(
"avx.vbroadcast.s") ||
377 Name.startswith(
"avx512.vbroadcast.s") ||
378 Name.startswith(
"avx512.mask.palignr.") ||
379 Name.startswith(
"avx512.mask.valign.") ||
380 Name.startswith(
"sse2.psll.dq") ||
381 Name.startswith(
"sse2.psrl.dq") ||
382 Name.startswith(
"avx2.psll.dq") ||
383 Name.startswith(
"avx2.psrl.dq") ||
384 Name.startswith(
"avx512.psll.dq") ||
385 Name.startswith(
"avx512.psrl.dq") ||
386 Name ==
"sse41.pblendw" ||
387 Name.startswith(
"sse41.blendp") ||
388 Name.startswith(
"avx.blend.p") ||
389 Name ==
"avx2.pblendw" ||
390 Name.startswith(
"avx2.pblendd.") ||
391 Name.startswith(
"avx.vbroadcastf128") ||
392 Name ==
"avx2.vbroadcasti128" ||
393 Name.startswith(
"avx512.mask.broadcastf32x4.") ||
394 Name.startswith(
"avx512.mask.broadcastf64x2.") ||
395 Name.startswith(
"avx512.mask.broadcastf32x8.") ||
396 Name.startswith(
"avx512.mask.broadcastf64x4.") ||
397 Name.startswith(
"avx512.mask.broadcasti32x4.") ||
398 Name.startswith(
"avx512.mask.broadcasti64x2.") ||
399 Name.startswith(
"avx512.mask.broadcasti32x8.") ||
400 Name.startswith(
"avx512.mask.broadcasti64x4.") ||
401 Name ==
"xop.vpcmov" ||
402 Name ==
"xop.vpcmov.256" ||
403 Name.startswith(
"avx512.mask.move.s") ||
404 Name.startswith(
"avx512.cvtmask2") ||
405 Name.startswith(
"xop.vpcom") ||
406 Name.startswith(
"xop.vprot") ||
407 Name.startswith(
"avx512.prol") ||
408 Name.startswith(
"avx512.pror") ||
409 Name.startswith(
"avx512.mask.prorv.") ||
410 Name.startswith(
"avx512.mask.pror.") ||
411 Name.startswith(
"avx512.mask.prolv.") ||
412 Name.startswith(
"avx512.mask.prol.") ||
413 Name.startswith(
"avx512.ptestm") ||
414 Name.startswith(
"avx512.ptestnm") ||
415 Name.startswith(
"avx512.mask.pavg"))
424 if (!
Name.startswith(
"x86."))
434 if (
Name ==
"rdtscp") {
436 if (
F->getFunctionType()->getNumParams() == 0)
441 Intrinsic::x86_rdtscp);
446 if (
Name.startswith(
"sse41.ptest")) {
447 if (
Name.substr(11) ==
"c")
449 if (
Name.substr(11) ==
"z")
451 if (
Name.substr(11) ==
"nzc")
456 if (
Name ==
"sse41.insertps")
459 if (
Name ==
"sse41.dppd")
462 if (
Name ==
"sse41.dpps")
465 if (
Name ==
"sse41.mpsadbw")
468 if (
Name ==
"avx.dp.ps.256")
471 if (
Name ==
"avx2.mpsadbw")
474 if (
Name ==
"avx512.mask.cmp.pd.128")
477 if (
Name ==
"avx512.mask.cmp.pd.256")
480 if (
Name ==
"avx512.mask.cmp.pd.512")
483 if (
Name ==
"avx512.mask.cmp.ps.128")
486 if (
Name ==
"avx512.mask.cmp.ps.256")
489 if (
Name ==
"avx512.mask.cmp.ps.512")
494 if (
Name.startswith(
"xop.vfrcz.ss") &&
F->arg_size() == 2) {
497 Intrinsic::x86_xop_vfrcz_ss);
500 if (
Name.startswith(
"xop.vfrcz.sd") &&
F->arg_size() == 2) {
503 Intrinsic::x86_xop_vfrcz_sd);
507 if (
Name.startswith(
"xop.vpermil2")) {
508 auto Idx =
F->getFunctionType()->getParamType(2);
509 if (Idx->isFPOrFPVectorTy()) {
511 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
512 unsigned EltSize = Idx->getScalarSizeInBits();
514 if (EltSize == 64 && IdxSize == 128)
515 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
516 else if (EltSize == 32 && IdxSize == 128)
517 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
518 else if (EltSize == 64 && IdxSize == 256)
519 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
521 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
527 if (
Name ==
"seh.recoverfp") {
536 assert(
F &&
"Illegal to upgrade a non-existent Function.");
540 if (
Name.size() <= 8 || !
Name.startswith(
"llvm."))
547 if (
Name.startswith(
"arm.rbit") ||
Name.startswith(
"aarch64.rbit")) {
549 F->arg_begin()->getType());
552 if (
Name.startswith(
"arm.neon.vclz")) {
554 F->arg_begin()->getType(),
562 "llvm.ctlz." +
Name.substr(14),
F->getParent());
565 if (
Name.startswith(
"arm.neon.vcnt")) {
567 F->arg_begin()->getType());
570 static const Regex vldRegex(
"^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
572 auto fArgs =
F->getFunctionType()->params();
578 "llvm." +
Name +
".p0i8",
F->getParent());
581 static const Regex vstRegex(
"^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
583 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
584 Intrinsic::arm_neon_vst2,
585 Intrinsic::arm_neon_vst3,
586 Intrinsic::arm_neon_vst4};
589 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
590 Intrinsic::arm_neon_vst4lane
593 auto fArgs =
F->getFunctionType()->params();
594 Type *Tys[] = {fArgs[0], fArgs[1]};
597 StoreInts[fArgs.size() - 3], Tys);
600 StoreLaneInts[fArgs.size() - 5], Tys);
603 if (
Name ==
"aarch64.thread.pointer" ||
Name ==
"arm.thread.pointer") {
607 if (
Name.startswith(
"arm.neon.vqadds.")) {
609 F->arg_begin()->getType());
612 if (
Name.startswith(
"arm.neon.vqaddu.")) {
614 F->arg_begin()->getType());
617 if (
Name.startswith(
"arm.neon.vqsubs.")) {
619 F->arg_begin()->getType());
622 if (
Name.startswith(
"arm.neon.vqsubu.")) {
624 F->arg_begin()->getType());
627 if (
Name.startswith(
"aarch64.neon.addp")) {
628 if (
F->arg_size() != 2)
630 VectorType *Ty = dyn_cast<VectorType>(
F->getReturnType());
633 Intrinsic::aarch64_neon_faddp, Ty);
640 if ((
Name.startswith(
"arm.neon.bfdot.") ||
641 Name.startswith(
"aarch64.neon.bfdot.")) &&
642 Name.endswith(
"i8")) {
645 .
Cases(
"arm.neon.bfdot.v2f32.v8i8",
646 "arm.neon.bfdot.v4f32.v16i8",
647 Intrinsic::arm_neon_bfdot)
648 .
Cases(
"aarch64.neon.bfdot.v2f32.v8i8",
649 "aarch64.neon.bfdot.v4f32.v16i8",
650 Intrinsic::aarch64_neon_bfdot)
655 size_t OperandWidth =
F->getReturnType()->getPrimitiveSizeInBits();
656 assert((OperandWidth == 64 || OperandWidth == 128) &&
657 "Unexpected operand width");
659 std::array<Type *, 2> Tys {{
669 if ((
Name.startswith(
"arm.neon.bfm") ||
670 Name.startswith(
"aarch64.neon.bfm")) &&
671 Name.endswith(
".v4f32.v16i8")) {
674 .
Case(
"arm.neon.bfmmla.v4f32.v16i8",
675 Intrinsic::arm_neon_bfmmla)
676 .
Case(
"arm.neon.bfmlalb.v4f32.v16i8",
677 Intrinsic::arm_neon_bfmlalb)
678 .
Case(
"arm.neon.bfmlalt.v4f32.v16i8",
679 Intrinsic::arm_neon_bfmlalt)
680 .
Case(
"aarch64.neon.bfmmla.v4f32.v16i8",
681 Intrinsic::aarch64_neon_bfmmla)
682 .
Case(
"aarch64.neon.bfmlalb.v4f32.v16i8",
683 Intrinsic::aarch64_neon_bfmlalb)
684 .
Case(
"aarch64.neon.bfmlalt.v4f32.v16i8",
685 Intrinsic::aarch64_neon_bfmlalt)
690 std::array<Type *, 0> Tys;
698 if (
Name.startswith(
"ctlz.") &&
F->arg_size() == 1) {
701 F->arg_begin()->getType());
704 if (
Name.startswith(
"cttz.") &&
F->arg_size() == 1) {
707 F->arg_begin()->getType());
713 if (
Name ==
"dbg.value" &&
F->arg_size() == 4) {
722 static const Regex R(
"^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
726 .
Case(
"add", Intrinsic::vector_reduce_add)
727 .
Case(
"mul", Intrinsic::vector_reduce_mul)
728 .
Case(
"and", Intrinsic::vector_reduce_and)
729 .
Case(
"or", Intrinsic::vector_reduce_or)
730 .
Case(
"xor", Intrinsic::vector_reduce_xor)
731 .
Case(
"smax", Intrinsic::vector_reduce_smax)
732 .
Case(
"smin", Intrinsic::vector_reduce_smin)
733 .
Case(
"umax", Intrinsic::vector_reduce_umax)
734 .
Case(
"umin", Intrinsic::vector_reduce_umin)
735 .
Case(
"fmax", Intrinsic::vector_reduce_fmax)
736 .
Case(
"fmin", Intrinsic::vector_reduce_fmin)
740 auto Args =
F->getFunctionType()->params();
746 "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
751 ID = Intrinsic::vector_reduce_fadd;
753 ID = Intrinsic::vector_reduce_fmul;
756 auto Args =
F->getFunctionType()->params();
766 bool IsLifetimeStart =
Name.startswith(
"lifetime.start");
767 if (IsLifetimeStart ||
Name.startswith(
"invariant.start")) {
769 Intrinsic::lifetime_start : Intrinsic::invariant_start;
770 auto Args =
F->getFunctionType()->params();
779 bool IsLifetimeEnd =
Name.startswith(
"lifetime.end");
780 if (IsLifetimeEnd ||
Name.startswith(
"invariant.end")) {
782 Intrinsic::lifetime_end : Intrinsic::invariant_end;
784 auto Args =
F->getFunctionType()->params();
785 Type* ObjectPtr[1] = {
Args[IsLifetimeEnd ? 1 : 2]};
792 if (
Name.startswith(
"invariant.group.barrier")) {
794 auto Args =
F->getFunctionType()->params();
798 Intrinsic::launder_invariant_group, ObjectPtr);
806 if (
Name.startswith(
"masked.load.")) {
807 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType() };
811 Intrinsic::masked_load,
816 if (
Name.startswith(
"masked.store.")) {
817 auto Args =
F->getFunctionType()->params();
822 Intrinsic::masked_store,
829 if (
Name.startswith(
"masked.gather.")) {
830 Type *Tys[] = {
F->getReturnType(),
F->arg_begin()->getType()};
834 Intrinsic::masked_gather, Tys);
838 if (
Name.startswith(
"masked.scatter.")) {
839 auto Args =
F->getFunctionType()->params();
844 Intrinsic::masked_scatter, Tys);
851 if (
Name.startswith(
"memcpy.") &&
F->arg_size() == 5) {
859 if (
Name.startswith(
"memmove.") &&
F->arg_size() == 5) {
867 if (
Name.startswith(
"memset.") &&
F->arg_size() == 5) {
870 const auto *FT =
F->getFunctionType();
871 Type *ParamTypes[2] = {
882 if (
Name.startswith(
"nvvm.")) {
887 .
Cases(
"brev32",
"brev64", Intrinsic::bitreverse)
888 .
Case(
"clz.i", Intrinsic::ctlz)
889 .
Case(
"popc.i", Intrinsic::ctpop)
893 {F->getReturnType()});
902 .
Cases(
"abs.i",
"abs.ll",
true)
903 .
Cases(
"clz.ll",
"popc.ll",
"h2f",
true)
904 .
Cases(
"max.i",
"max.ll",
"max.ui",
"max.ull",
true)
905 .
Cases(
"min.i",
"min.ll",
"min.ui",
"min.ull",
true)
919 if (
Name.startswith(
"objectsize.")) {
920 Type *Tys[2] = {
F->getReturnType(),
F->arg_begin()->getType() };
921 if (
F->arg_size() == 2 ||
F->arg_size() == 3 ||
932 if (
Name ==
"prefetch") {
934 Type *Tys[] = {
F->arg_begin()->getType()};
941 }
else if (
Name.startswith(
"ptr.annotation.") &&
F->arg_size() == 4) {
944 Intrinsic::ptr_annotation,
945 F->arg_begin()->getType());
951 if (
Name ==
"stackprotectorcheck") {
958 if (
Name ==
"var.annotation" &&
F->arg_size() == 4) {
961 Intrinsic::var_annotation);
973 if (Result !=
None) {
974 NewFn = Result.getValue();
988 assert(
F != NewFn &&
"Intrinsic function upgraded to the same function");
1000 GV->
getName() ==
"llvm.global_dtors")) ||
1015 unsigned N =
Init->getNumOperands();
1016 std::vector<Constant *> NewCtors(
N);
1017 for (
unsigned i = 0;
i !=
N; ++
i) {
1018 auto Ctor = cast<Constant>(
Init->getOperand(
i));
1020 EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1033 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1034 unsigned NumElts = ResultTy->getNumElements() * 8;
1048 for (
unsigned l = 0;
l != NumElts;
l += 16)
1049 for (
unsigned i = 0;
i != 16; ++
i) {
1050 unsigned Idx = NumElts +
i -
Shift;
1052 Idx -= NumElts - 16;
1053 Idxs[
l +
i] = Idx +
l;
1060 return Builder.CreateBitCast(Res, ResultTy,
"cast");
1067 auto *ResultTy = cast<FixedVectorType>(
Op->getType());
1068 unsigned NumElts = ResultTy->getNumElements() * 8;
1082 for (
unsigned l = 0;
l != NumElts;
l += 16)
1083 for (
unsigned i = 0;
i != 16; ++
i) {
1084 unsigned Idx =
i +
Shift;
1086 Idx += NumElts - 16;
1087 Idxs[
l +
i] = Idx +
l;
1094 return Builder.CreateBitCast(Res, ResultTy,
"cast");
1101 Builder.getInt1Ty(), cast<IntegerType>(
Mask->getType())->getBitWidth());
1108 for (
unsigned i = 0;
i != NumElts; ++
i)
1120 if (
const auto *
C = dyn_cast<Constant>(
Mask))
1121 if (
C->isAllOnesValue())
1125 cast<FixedVectorType>(Op0->
getType())->getNumElements());
1132 if (
const auto *
C = dyn_cast<Constant>(
Mask))
1133 if (
C->isAllOnesValue())
1137 Mask->getType()->getIntegerBitWidth());
1150 unsigned ShiftVal = cast<llvm::ConstantInt>(
Shift)->getZExtValue();
1152 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1153 assert((IsVALIGN || NumElts % 16 == 0) &&
"Illegal NumElts for PALIGNR!");
1154 assert((!IsVALIGN || NumElts <= 16) &&
"NumElts too large for VALIGN!");
1159 ShiftVal &= (NumElts - 1);
1168 if (ShiftVal > 16) {
1176 for (
unsigned l = 0;
l < NumElts;
l += 16) {
1177 for (
unsigned i = 0;
i != 16; ++
i) {
1178 unsigned Idx = ShiftVal +
i;
1179 if (!IsVALIGN && Idx >= 16)
1180 Idx += NumElts - 16;
1181 Indices[
l +
i] = Idx +
l;
1193 bool ZeroMask,
bool IndexForm) {
1199 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1200 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1201 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1202 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1203 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1204 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1205 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1206 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1207 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1208 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1209 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1210 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1211 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1212 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1213 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1214 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1215 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1216 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1217 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1218 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1219 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1220 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1221 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1222 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1223 else if (VecWidth == 128 && EltWidth == 16)
1224 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1225 else if (VecWidth == 256 && EltWidth == 16)
1226 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1227 else if (VecWidth == 512 && EltWidth == 16)
1228 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1229 else if (VecWidth == 128 && EltWidth == 8)
1230 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1231 else if (VecWidth == 256 && EltWidth == 8)
1232 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1233 else if (VecWidth == 512 && EltWidth == 8)
1234 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1270 bool IsRotateRight) {
1279 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1281 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
1284 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1286 Value *Res =
Builder.CreateCall(Intrin, {Src, Src, Amt});
1336 bool IsShiftRight,
bool ZeroMask) {
1349 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1351 Amt =
Builder.CreateVectorSplat(NumElts, Amt);
1354 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1356 Value *Res =
Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1373 Ptr =
Builder.CreateBitCast(Ptr,
1375 const Align Alignment =
1377 ?
Align(
Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)
1381 if (
const auto *
C = dyn_cast<Constant>(
Mask))
1382 if (
C->isAllOnesValue())
1383 return Builder.CreateAlignedStore(
Data, Ptr, Alignment);
1386 unsigned NumElts = cast<FixedVectorType>(
Data->getType())->getNumElements();
1397 const Align Alignment =
1404 if (
const auto *
C = dyn_cast<Constant>(
Mask))
1405 if (
C->isAllOnesValue())
1406 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1410 cast<FixedVectorType>(Passthru->
getType())->getNumElements();
1412 return Builder.CreateMaskedLoad(Ptr, Alignment,
Mask, Passthru);
1435 LHS =
Builder.CreateShl(LHS, ShiftAmt);
1436 LHS =
Builder.CreateAShr(LHS, ShiftAmt);
1437 RHS =
Builder.CreateShl(RHS, ShiftAmt);
1438 RHS =
Builder.CreateAShr(RHS, ShiftAmt);
1457 unsigned NumElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
1459 const auto *
C = dyn_cast<Constant>(
Mask);
1460 if (!
C || !
C->isAllOnesValue())
1466 for (
unsigned i = 0;
i != NumElts; ++
i)
1468 for (
unsigned i = NumElts;
i != 8; ++
i)
1469 Indices[
i] = NumElts +
i % NumElts;
1470 Vec =
Builder.CreateShuffleVector(Vec,
1478 unsigned CC,
bool Signed) {
1480 unsigned NumElts = cast<FixedVectorType>(Op0->
getType())->getNumElements();
1486 }
else if (CC == 7) {
1525 Value* Extract1 =
Builder.CreateExtractElement(
B, (uint64_t)0);
1526 Value* Extract2 =
Builder.CreateExtractElement(Src, (uint64_t)0);
1535 unsigned NumElts = cast<FixedVectorType>(CI.
getType())->getNumElements();
1537 return Builder.CreateSExt(
Mask, ReturnOp,
"vpmovm2");
1548 if (
Name.startswith(
"max.p")) {
1549 if (VecWidth == 128 && EltWidth == 32)
1550 IID = Intrinsic::x86_sse_max_ps;
1551 else if (VecWidth == 128 && EltWidth == 64)
1552 IID = Intrinsic::x86_sse2_max_pd;
1553 else if (VecWidth == 256 && EltWidth == 32)
1554 IID = Intrinsic::x86_avx_max_ps_256;
1555 else if (VecWidth == 256 && EltWidth == 64)
1556 IID = Intrinsic::x86_avx_max_pd_256;
1559 }
else if (
Name.startswith(
"min.p")) {
1560 if (VecWidth == 128 && EltWidth == 32)
1561 IID = Intrinsic::x86_sse_min_ps;
1562 else if (VecWidth == 128 && EltWidth == 64)
1563 IID = Intrinsic::x86_sse2_min_pd;
1564 else if (VecWidth == 256 && EltWidth == 32)
1565 IID = Intrinsic::x86_avx_min_ps_256;
1566 else if (VecWidth == 256 && EltWidth == 64)
1567 IID = Intrinsic::x86_avx_min_pd_256;
1570 }
else if (
Name.startswith(
"pshuf.b.")) {
1571 if (VecWidth == 128)
1572 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1573 else if (VecWidth == 256)
1574 IID = Intrinsic::x86_avx2_pshuf_b;
1575 else if (VecWidth == 512)
1576 IID = Intrinsic::x86_avx512_pshuf_b_512;
1579 }
else if (
Name.startswith(
"pmul.hr.sw.")) {
1580 if (VecWidth == 128)
1581 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1582 else if (VecWidth == 256)
1583 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1584 else if (VecWidth == 512)
1585 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1588 }
else if (
Name.startswith(
"pmulh.w.")) {
1589 if (VecWidth == 128)
1590 IID = Intrinsic::x86_sse2_pmulh_w;
1591 else if (VecWidth == 256)
1592 IID = Intrinsic::x86_avx2_pmulh_w;
1593 else if (VecWidth == 512)
1594 IID = Intrinsic::x86_avx512_pmulh_w_512;
1597 }
else if (
Name.startswith(
"pmulhu.w.")) {
1598 if (VecWidth == 128)
1599 IID = Intrinsic::x86_sse2_pmulhu_w;
1600 else if (VecWidth == 256)
1601 IID = Intrinsic::x86_avx2_pmulhu_w;
1602 else if (VecWidth == 512)
1603 IID = Intrinsic::x86_avx512_pmulhu_w_512;
1606 }
else if (
Name.startswith(
"pmaddw.d.")) {
1607 if (VecWidth == 128)
1608 IID = Intrinsic::x86_sse2_pmadd_wd;
1609 else if (VecWidth == 256)
1610 IID = Intrinsic::x86_avx2_pmadd_wd;
1611 else if (VecWidth == 512)
1612 IID = Intrinsic::x86_avx512_pmaddw_d_512;
1615 }
else if (
Name.startswith(
"pmaddubs.w.")) {
1616 if (VecWidth == 128)
1617 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1618 else if (VecWidth == 256)
1619 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1620 else if (VecWidth == 512)
1621 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1624 }
else if (
Name.startswith(
"packsswb.")) {
1625 if (VecWidth == 128)
1626 IID = Intrinsic::x86_sse2_packsswb_128;
1627 else if (VecWidth == 256)
1628 IID = Intrinsic::x86_avx2_packsswb;
1629 else if (VecWidth == 512)
1630 IID = Intrinsic::x86_avx512_packsswb_512;
1633 }
else if (
Name.startswith(
"packssdw.")) {
1634 if (VecWidth == 128)
1635 IID = Intrinsic::x86_sse2_packssdw_128;
1636 else if (VecWidth == 256)
1637 IID = Intrinsic::x86_avx2_packssdw;
1638 else if (VecWidth == 512)
1639 IID = Intrinsic::x86_avx512_packssdw_512;
1642 }
else if (
Name.startswith(
"packuswb.")) {
1643 if (VecWidth == 128)
1644 IID = Intrinsic::x86_sse2_packuswb_128;
1645 else if (VecWidth == 256)
1646 IID = Intrinsic::x86_avx2_packuswb;
1647 else if (VecWidth == 512)
1648 IID = Intrinsic::x86_avx512_packuswb_512;
1651 }
else if (
Name.startswith(
"packusdw.")) {
1652 if (VecWidth == 128)
1653 IID = Intrinsic::x86_sse41_packusdw;
1654 else if (VecWidth == 256)
1655 IID = Intrinsic::x86_avx2_packusdw;
1656 else if (VecWidth == 512)
1657 IID = Intrinsic::x86_avx512_packusdw_512;
1660 }
else if (
Name.startswith(
"vpermilvar.")) {
1661 if (VecWidth == 128 && EltWidth == 32)
1662 IID = Intrinsic::x86_avx_vpermilvar_ps;
1663 else if (VecWidth == 128 && EltWidth == 64)
1664 IID = Intrinsic::x86_avx_vpermilvar_pd;
1665 else if (VecWidth == 256 && EltWidth == 32)
1666 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1667 else if (VecWidth == 256 && EltWidth == 64)
1668 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1669 else if (VecWidth == 512 && EltWidth == 32)
1670 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1671 else if (VecWidth == 512 && EltWidth == 64)
1672 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1675 }
else if (
Name ==
"cvtpd2dq.256") {
1676 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1677 }
else if (
Name ==
"cvtpd2ps.256") {
1678 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1679 }
else if (
Name ==
"cvttpd2dq.256") {
1680 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1681 }
else if (
Name ==
"cvttps2dq.128") {
1682 IID = Intrinsic::x86_sse2_cvttps2dq;
1683 }
else if (
Name ==
"cvttps2dq.256") {
1684 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1685 }
else if (
Name.startswith(
"permvar.")) {
1687 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1688 IID = Intrinsic::x86_avx2_permps;
1689 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1690 IID = Intrinsic::x86_avx2_permd;
1691 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1692 IID = Intrinsic::x86_avx512_permvar_df_256;
1693 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1694 IID = Intrinsic::x86_avx512_permvar_di_256;
1695 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1696 IID = Intrinsic::x86_avx512_permvar_sf_512;
1697 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1698 IID = Intrinsic::x86_avx512_permvar_si_512;
1699 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1700 IID = Intrinsic::x86_avx512_permvar_df_512;
1701 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1702 IID = Intrinsic::x86_avx512_permvar_di_512;
1703 else if (VecWidth == 128 && EltWidth == 16)
1704 IID = Intrinsic::x86_avx512_permvar_hi_128;
1705 else if (VecWidth == 256 && EltWidth == 16)
1706 IID = Intrinsic::x86_avx512_permvar_hi_256;
1707 else if (VecWidth == 512 && EltWidth == 16)
1708 IID = Intrinsic::x86_avx512_permvar_hi_512;
1709 else if (VecWidth == 128 && EltWidth == 8)
1710 IID = Intrinsic::x86_avx512_permvar_qi_128;
1711 else if (VecWidth == 256 && EltWidth == 8)
1712 IID = Intrinsic::x86_avx512_permvar_qi_256;
1713 else if (VecWidth == 512 && EltWidth == 8)
1714 IID = Intrinsic::x86_avx512_permvar_qi_512;
1717 }
else if (
Name.startswith(
"dbpsadbw.")) {
1718 if (VecWidth == 128)
1719 IID = Intrinsic::x86_avx512_dbpsadbw_128;
1720 else if (VecWidth == 256)
1721 IID = Intrinsic::x86_avx512_dbpsadbw_256;
1722 else if (VecWidth == 512)
1723 IID = Intrinsic::x86_avx512_dbpsadbw_512;
1726 }
else if (
Name.startswith(
"pmultishift.qb.")) {
1727 if (VecWidth == 128)
1728 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1729 else if (VecWidth == 256)
1730 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1731 else if (VecWidth == 512)
1732 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1735 }
else if (
Name.startswith(
"conflict.")) {
1736 if (
Name[9] ==
'd' && VecWidth == 128)
1737 IID = Intrinsic::x86_avx512_conflict_d_128;
1738 else if (
Name[9] ==
'd' && VecWidth == 256)
1739 IID = Intrinsic::x86_avx512_conflict_d_256;
1740 else if (
Name[9] ==
'd' && VecWidth == 512)
1741 IID = Intrinsic::x86_avx512_conflict_d_512;
1742 else if (
Name[9] ==
'q' && VecWidth == 128)
1743 IID = Intrinsic::x86_avx512_conflict_q_128;
1744 else if (
Name[9] ==
'q' && VecWidth == 256)
1745 IID = Intrinsic::x86_avx512_conflict_q_256;
1746 else if (
Name[9] ==
'q' && VecWidth == 512)
1747 IID = Intrinsic::x86_avx512_conflict_q_512;
1750 }
else if (
Name.startswith(
"pavg.")) {
1751 if (
Name[5] ==
'b' && VecWidth == 128)
1752 IID = Intrinsic::x86_sse2_pavg_b;
1753 else if (
Name[5] ==
'b' && VecWidth == 256)
1754 IID = Intrinsic::x86_avx2_pavg_b;
1755 else if (
Name[5] ==
'b' && VecWidth == 512)
1756 IID = Intrinsic::x86_avx512_pavg_b_512;
1757 else if (
Name[5] ==
'w' && VecWidth == 128)
1758 IID = Intrinsic::x86_sse2_pavg_w;
1759 else if (
Name[5] ==
'w' && VecWidth == 256)
1760 IID = Intrinsic::x86_avx2_pavg_w;
1761 else if (
Name[5] ==
'w' && VecWidth == 512)
1762 IID = Intrinsic::x86_avx512_pavg_w_512;
1784 if (AsmStr->find(
"mov\tfp") == 0 &&
1785 AsmStr->find(
"objc_retainAutoreleaseReturnValue") != std::string::npos &&
1786 (Pos = AsmStr->find(
"# marker")) != std::string::npos) {
1787 AsmStr->replace(Pos, 1,
";");
1799 assert(
F &&
"Intrinsic call is not direct?");
1805 assert(
Name.startswith(
"llvm.") &&
"Intrinsic doesn't start with 'llvm.'");
1808 bool IsX86 =
Name.startswith(
"x86.");
1811 bool IsNVVM =
Name.startswith(
"nvvm.");
1815 if (IsX86 &&
Name.startswith(
"sse4a.movnt.")) {
1827 Type *SrcEltTy = cast<VectorType>(Arg1->
getType())->getElementType();
1831 Builder.CreateExtractElement(Arg1, (uint64_t)0,
"extractelement");
1834 SI->setMetadata(
M->getMDKindID(
"nontemporal"), Node);
1841 if (IsX86 && (
Name.startswith(
"avx.movnt.") ||
1842 Name.startswith(
"avx512.storent."))) {
1859 SI->setMetadata(
M->getMDKindID(
"nontemporal"), Node);
1866 if (IsX86 &&
Name ==
"sse2.storel.dq") {
1871 Value *BC0 =
Builder.CreateBitCast(Arg1, NewVecTy,
"cast");
1872 Value *Elt =
Builder.CreateExtractElement(BC0, (uint64_t)0);
1883 if (IsX86 && (
Name.startswith(
"sse.storeu.") ||
1884 Name.startswith(
"sse2.storeu.") ||
1885 Name.startswith(
"avx.storeu."))) {
1889 Arg0 =
Builder.CreateBitCast(Arg0,
1899 if (IsX86 &&
Name ==
"avx512.mask.store.ss") {
1909 if (IsX86 && (
Name.startswith(
"avx512.mask.store"))) {
1911 bool Aligned =
Name[17] !=
'u';
1922 if (IsX86 && (
Name.startswith(
"sse2.pcmp") ||
1923 Name.startswith(
"avx2.pcmp"))) {
1925 bool CmpEq =
Name[9] ==
'e';
1929 }
else if (IsX86 && (
Name.startswith(
"avx512.broadcastm"))) {
1936 Rep =
Builder.CreateVectorSplat(NumElts, Rep);
1937 }
else if (IsX86 && (
Name ==
"sse.sqrt.ss" ||
1938 Name ==
"sse2.sqrt.sd")) {
1940 Value *Elt0 =
Builder.CreateExtractElement(Vec, (uint64_t)0);
1942 Intrinsic::sqrt, Elt0->
getType());
1944 Rep =
Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
1945 }
else if (IsX86 && (
Name.startswith(
"avx.sqrt.p") ||
1946 Name.startswith(
"sse2.sqrt.p") ||
1947 Name.startswith(
"sse.sqrt.p"))) {
1951 {CI->getArgOperand(0)});
1952 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.sqrt.p"))) {
1955 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
1957 : Intrinsic::x86_avx512_sqrt_pd_512;
1966 {CI->getArgOperand(0)});
1970 }
else if (IsX86 && (
Name.startswith(
"avx512.ptestm") ||
1971 Name.startswith(
"avx512.ptestnm"))) {
1975 Rep =
Builder.CreateAnd(Op0, Op1);
1980 Rep =
Builder.CreateICmp(Pred, Rep, Zero);
1982 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.pbroadcast"))){
1988 }
else if (IsX86 && (
Name.startswith(
"avx512.kunpck"))) {
1993 for (
unsigned i = 0;
i != NumElts; ++
i)
1998 LHS =
Builder.CreateShuffleVector(LHS, LHS,
2000 RHS =
Builder.CreateShuffleVector(RHS, RHS,
2004 Rep =
Builder.CreateShuffleVector(RHS, LHS,
2007 }
else if (IsX86 &&
Name ==
"avx512.kand.w") {
2010 Rep =
Builder.CreateAnd(LHS, RHS);
2012 }
else if (IsX86 &&
Name ==
"avx512.kandn.w") {
2016 Rep =
Builder.CreateAnd(LHS, RHS);
2018 }
else if (IsX86 &&
Name ==
"avx512.kor.w") {
2021 Rep =
Builder.CreateOr(LHS, RHS);
2023 }
else if (IsX86 &&
Name ==
"avx512.kxor.w") {
2026 Rep =
Builder.CreateXor(LHS, RHS);
2028 }
else if (IsX86 &&
Name ==
"avx512.kxnor.w") {
2032 Rep =
Builder.CreateXor(LHS, RHS);
2034 }
else if (IsX86 &&
Name ==
"avx512.knot.w") {
2039 (
Name ==
"avx512.kortestz.w" ||
Name ==
"avx512.kortestc.w")) {
2042 Rep =
Builder.CreateOr(LHS, RHS);
2045 if (
Name[14] ==
'c')
2049 Rep =
Builder.CreateICmpEQ(Rep,
C);
2051 }
else if (IsX86 && (
Name ==
"sse.add.ss" ||
Name ==
"sse2.add.sd" ||
2052 Name ==
"sse.sub.ss" ||
Name ==
"sse2.sub.sd" ||
2053 Name ==
"sse.mul.ss" ||
Name ==
"sse2.mul.sd" ||
2054 Name ==
"sse.div.ss" ||
Name ==
"sse2.div.sd")) {
2061 if (
Name.contains(
".add."))
2062 EltOp =
Builder.CreateFAdd(Elt0, Elt1);
2063 else if (
Name.contains(
".sub."))
2064 EltOp =
Builder.CreateFSub(Elt0, Elt1);
2065 else if (
Name.contains(
".mul."))
2066 EltOp =
Builder.CreateFMul(Elt0, Elt1);
2068 EltOp =
Builder.CreateFDiv(Elt0, Elt1);
2071 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.pcmp")) {
2073 bool CmpEq =
Name[16] ==
'e';
2075 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.vpshufbitqmb.")) {
2081 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
break;
2082 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
break;
2083 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
break;
2087 { CI->getOperand(0), CI->getArgOperand(1) });
2089 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.fpclass.p")) {
2094 if (VecWidth == 128 && EltWidth == 32)
2095 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2096 else if (VecWidth == 256 && EltWidth == 32)
2097 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2098 else if (VecWidth == 512 && EltWidth == 32)
2099 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2100 else if (VecWidth == 128 && EltWidth == 64)
2101 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2102 else if (VecWidth == 256 && EltWidth == 64)
2103 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2104 else if (VecWidth == 512 && EltWidth == 64)
2105 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2110 { CI->getOperand(0), CI->getArgOperand(1) });
2112 }
else if (IsX86 &&
Name.startswith(
"avx512.cmp.p")) {
2119 if (VecWidth == 128 && EltWidth == 32)
2120 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2121 else if (VecWidth == 256 && EltWidth == 32)
2122 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2123 else if (VecWidth == 512 && EltWidth == 32)
2124 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2125 else if (VecWidth == 128 && EltWidth == 64)
2126 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2127 else if (VecWidth == 256 && EltWidth == 64)
2128 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2129 else if (VecWidth == 512 && EltWidth == 64)
2130 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2135 if (VecWidth == 512)
2141 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.cmp.")) {
2143 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2145 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.ucmp.")) {
2146 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2148 }
else if (IsX86 && (
Name.startswith(
"avx512.cvtb2mask.") ||
2149 Name.startswith(
"avx512.cvtw2mask.") ||
2150 Name.startswith(
"avx512.cvtd2mask.") ||
2151 Name.startswith(
"avx512.cvtq2mask."))) {
2156 }
else if(IsX86 && (
Name ==
"ssse3.pabs.b.128" ||
2157 Name ==
"ssse3.pabs.w.128" ||
2158 Name ==
"ssse3.pabs.d.128" ||
2159 Name.startswith(
"avx2.pabs") ||
2160 Name.startswith(
"avx512.mask.pabs"))) {
2162 }
else if (IsX86 && (
Name ==
"sse41.pmaxsb" ||
2163 Name ==
"sse2.pmaxs.w" ||
2164 Name ==
"sse41.pmaxsd" ||
2165 Name.startswith(
"avx2.pmaxs") ||
2166 Name.startswith(
"avx512.mask.pmaxs"))) {
2168 }
else if (IsX86 && (
Name ==
"sse2.pmaxu.b" ||
2169 Name ==
"sse41.pmaxuw" ||
2170 Name ==
"sse41.pmaxud" ||
2171 Name.startswith(
"avx2.pmaxu") ||
2172 Name.startswith(
"avx512.mask.pmaxu"))) {
2174 }
else if (IsX86 && (
Name ==
"sse41.pminsb" ||
2175 Name ==
"sse2.pmins.w" ||
2176 Name ==
"sse41.pminsd" ||
2177 Name.startswith(
"avx2.pmins") ||
2178 Name.startswith(
"avx512.mask.pmins"))) {
2180 }
else if (IsX86 && (
Name ==
"sse2.pminu.b" ||
2181 Name ==
"sse41.pminuw" ||
2182 Name ==
"sse41.pminud" ||
2183 Name.startswith(
"avx2.pminu") ||
2184 Name.startswith(
"avx512.mask.pminu"))) {
2186 }
else if (IsX86 && (
Name ==
"sse2.pmulu.dq" ||
2187 Name ==
"avx2.pmulu.dq" ||
2188 Name ==
"avx512.pmulu.dq.512" ||
2189 Name.startswith(
"avx512.mask.pmulu.dq."))) {
2191 }
else if (IsX86 && (
Name ==
"sse41.pmuldq" ||
2192 Name ==
"avx2.pmul.dq" ||
2193 Name ==
"avx512.pmul.dq.512" ||
2194 Name.startswith(
"avx512.mask.pmul.dq."))) {
2196 }
else if (IsX86 && (
Name ==
"sse.cvtsi2ss" ||
2197 Name ==
"sse2.cvtsi2sd" ||
2198 Name ==
"sse.cvtsi642ss" ||
2199 Name ==
"sse2.cvtsi642sd")) {
2202 cast<VectorType>(CI->
getType())->getElementType());
2204 }
else if (IsX86 &&
Name ==
"avx512.cvtusi2sd") {
2207 cast<VectorType>(CI->
getType())->getElementType());
2209 }
else if (IsX86 &&
Name ==
"sse2.cvtss2sd") {
2212 Rep, cast<VectorType>(CI->
getType())->getElementType());
2214 }
else if (IsX86 && (
Name ==
"sse2.cvtdq2pd" ||
2215 Name ==
"sse2.cvtdq2ps" ||
2216 Name ==
"avx.cvtdq2.pd.256" ||
2217 Name ==
"avx.cvtdq2.ps.256" ||
2218 Name.startswith(
"avx512.mask.cvtdq2pd.") ||
2219 Name.startswith(
"avx512.mask.cvtudq2pd.") ||
2220 Name.startswith(
"avx512.mask.cvtdq2ps.") ||
2221 Name.startswith(
"avx512.mask.cvtudq2ps.") ||
2222 Name.startswith(
"avx512.mask.cvtqq2pd.") ||
2223 Name.startswith(
"avx512.mask.cvtuqq2pd.") ||
2224 Name ==
"avx512.mask.cvtqq2ps.256" ||
2225 Name ==
"avx512.mask.cvtqq2ps.512" ||
2226 Name ==
"avx512.mask.cvtuqq2ps.256" ||
2227 Name ==
"avx512.mask.cvtuqq2ps.512" ||
2228 Name ==
"sse2.cvtps2pd" ||
2229 Name ==
"avx.cvt.ps2.pd.256" ||
2230 Name ==
"avx512.mask.cvtps2pd.128" ||
2231 Name ==
"avx512.mask.cvtps2pd.256")) {
2232 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2234 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2236 unsigned NumDstElts = DstTy->getNumElements();
2237 if (NumDstElts < SrcTy->getNumElements()) {
2238 assert(NumDstElts == 2 &&
"Unexpected vector size");
2242 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2245 Rep =
Builder.CreateFPExt(Rep, DstTy,
"cvtps2pd");
2248 cast<ConstantInt>(CI->
getArgOperand(3))->getZExtValue() != 4)) {
2249 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2250 : Intrinsic::x86_avx512_sitofp_round;
2255 Rep = IsUnsigned ?
Builder.CreateUIToFP(Rep, DstTy,
"cvt")
2256 :
Builder.CreateSIToFP(Rep, DstTy,
"cvt");
2262 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vcvtph2ps.") ||
2263 Name.startswith(
"vcvtph2ps."))) {
2264 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2266 auto *SrcTy = cast<FixedVectorType>(Rep->
getType());
2267 unsigned NumDstElts = DstTy->getNumElements();
2268 if (NumDstElts != SrcTy->getNumElements()) {
2269 assert(NumDstElts == 4 &&
"Unexpected vector size");
2274 Rep =
Builder.CreateFPExt(Rep, DstTy,
"cvtph2ps");
2278 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.loadu."))) {
2282 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.load."))) {
2286 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.expand.load.")) {
2287 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2288 Type *PtrTy = ResultTy->getElementType();
2295 ResultTy->getNumElements());
2298 Intrinsic::masked_expandload,
2301 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.compress.store.")) {
2303 Type *PtrTy = ResultTy->getElementType();
2311 cast<FixedVectorType>(ResultTy)->getNumElements());
2314 Intrinsic::masked_compressstore,
2317 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.compress.") ||
2318 Name.startswith(
"avx512.mask.expand."))) {
2319 auto *ResultTy = cast<FixedVectorType>(CI->
getType());
2322 ResultTy->getNumElements());
2324 bool IsCompress =
Name[12] ==
'c';
2325 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2326 : Intrinsic::x86_avx512_mask_expand;
2330 }
else if (IsX86 &&
Name.startswith(
"xop.vpcom")) {
2332 if (
Name.endswith(
"ub") ||
Name.endswith(
"uw") ||
Name.endswith(
"ud") ||
2333 Name.endswith(
"uq"))
2335 else if (
Name.endswith(
"b") ||
Name.endswith(
"w") ||
Name.endswith(
"d") ||
2343 Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2346 if (
Name.startswith(
"lt"))
2348 else if (
Name.startswith(
"le"))
2350 else if (
Name.startswith(
"gt"))
2352 else if (
Name.startswith(
"ge"))
2354 else if (
Name.startswith(
"eq"))
2356 else if (
Name.startswith(
"ne"))
2358 else if (
Name.startswith(
"false"))
2360 else if (
Name.startswith(
"true"))
2367 }
else if (IsX86 &&
Name.startswith(
"xop.vpcmov")) {
2372 Rep =
Builder.CreateOr(Sel0, Sel1);
2373 }
else if (IsX86 && (
Name.startswith(
"xop.vprot") ||
2374 Name.startswith(
"avx512.prol") ||
2375 Name.startswith(
"avx512.mask.prol"))) {
2377 }
else if (IsX86 && (
Name.startswith(
"avx512.pror") ||
2378 Name.startswith(
"avx512.mask.pror"))) {
2380 }
else if (IsX86 && (
Name.startswith(
"avx512.vpshld.") ||
2381 Name.startswith(
"avx512.mask.vpshld") ||
2382 Name.startswith(
"avx512.maskz.vpshld"))) {
2383 bool ZeroMask =
Name[11] ==
'z';
2385 }
else if (IsX86 && (
Name.startswith(
"avx512.vpshrd.") ||
2386 Name.startswith(
"avx512.mask.vpshrd") ||
2387 Name.startswith(
"avx512.maskz.vpshrd"))) {
2388 bool ZeroMask =
Name[11] ==
'z';
2390 }
else if (IsX86 &&
Name ==
"sse42.crc32.64.8") {
2392 Intrinsic::x86_sse42_crc32_32_8);
2396 }
else if (IsX86 && (
Name.startswith(
"avx.vbroadcast.s") ||
2397 Name.startswith(
"avx512.vbroadcast.s"))) {
2399 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2400 Type *EltTy = VecTy->getElementType();
2401 unsigned EltNum = VecTy->getNumElements();
2407 for (
unsigned I = 0;
I < EltNum; ++
I)
2410 }
else if (IsX86 && (
Name.startswith(
"sse41.pmovsx") ||
2411 Name.startswith(
"sse41.pmovzx") ||
2412 Name.startswith(
"avx2.pmovsx") ||
2413 Name.startswith(
"avx2.pmovzx") ||
2414 Name.startswith(
"avx512.mask.pmovsx") ||
2415 Name.startswith(
"avx512.mask.pmovzx"))) {
2416 auto *DstTy = cast<FixedVectorType>(CI->
getType());
2417 unsigned NumDstElts = DstTy->getNumElements();
2421 for (
unsigned i = 0;
i != NumDstElts; ++
i)
2428 Rep = DoSext ?
Builder.CreateSExt(SV, DstTy)
2429 :
Builder.CreateZExt(SV, DstTy);
2434 }
else if (
Name ==
"avx512.mask.pmov.qd.256" ||
2435 Name ==
"avx512.mask.pmov.qd.512" ||
2436 Name ==
"avx512.mask.pmov.wb.256" ||
2437 Name ==
"avx512.mask.pmov.wb.512") {
2442 }
else if (IsX86 && (
Name.startswith(
"avx.vbroadcastf128") ||
2443 Name ==
"avx2.vbroadcasti128")) {
2445 Type *EltTy = cast<VectorType>(CI->
getType())->getElementType();
2451 if (NumSrcElts == 2)
2454 Rep =
Builder.CreateShuffleVector(
2456 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.shuf.i") ||
2457 Name.startswith(
"avx512.mask.shuf.f"))) {
2458 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2462 unsigned ControlBitsMask = NumLanes - 1;
2463 unsigned NumControlBits = NumLanes / 2;
2466 for (
unsigned l = 0;
l != NumLanes; ++
l) {
2467 unsigned LaneMask = (Imm >> (
l * NumControlBits)) & ControlBitsMask;
2469 if (
l >= NumLanes / 2)
2470 LaneMask += NumLanes;
2471 for (
unsigned i = 0;
i != NumElementsInLane; ++
i)
2472 ShuffleMask.push_back(LaneMask * NumElementsInLane +
i);
2478 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.broadcastf") ||
2479 Name.startswith(
"avx512.mask.broadcasti"))) {
2480 unsigned NumSrcElts =
2483 unsigned NumDstElts =
2484 cast<FixedVectorType>(CI->
getType())->getNumElements();
2487 for (
unsigned i = 0;
i != NumDstElts; ++
i)
2488 ShuffleMask[
i] =
i % NumSrcElts;
2495 }
else if (IsX86 && (
Name.startswith(
"avx2.pbroadcast") ||
2496 Name.startswith(
"avx2.vbroadcast") ||
2497 Name.startswith(
"avx512.pbroadcast") ||
2498 Name.startswith(
"avx512.mask.broadcast.s"))) {
2510 }
else if (IsX86 && (
Name.startswith(
"sse2.padds.") ||
2511 Name.startswith(
"avx2.padds.") ||
2512 Name.startswith(
"avx512.padds.") ||
2513 Name.startswith(
"avx512.mask.padds."))) {
2515 }
else if (IsX86 && (
Name.startswith(
"sse2.psubs.") ||
2516 Name.startswith(
"avx2.psubs.") ||
2517 Name.startswith(
"avx512.psubs.") ||
2518 Name.startswith(
"avx512.mask.psubs."))) {
2520 }
else if (IsX86 && (
Name.startswith(
"sse2.paddus.") ||
2521 Name.startswith(
"avx2.paddus.") ||
2522 Name.startswith(
"avx512.mask.paddus."))) {
2524 }
else if (IsX86 && (
Name.startswith(
"sse2.psubus.") ||
2525 Name.startswith(
"avx2.psubus.") ||
2526 Name.startswith(
"avx512.mask.psubus."))) {
2528 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.palignr.")) {
2535 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.valign.")) {
2542 }
else if (IsX86 && (
Name ==
"sse2.psll.dq" ||
2543 Name ==
"avx2.psll.dq")) {
2548 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq" ||
2549 Name ==
"avx2.psrl.dq")) {
2554 }
else if (IsX86 && (
Name ==
"sse2.psll.dq.bs" ||
2555 Name ==
"avx2.psll.dq.bs" ||
2556 Name ==
"avx512.psll.dq.512")) {
2560 }
else if (IsX86 && (
Name ==
"sse2.psrl.dq.bs" ||
2561 Name ==
"avx2.psrl.dq.bs" ||
2562 Name ==
"avx512.psrl.dq.512")) {
2566 }
else if (IsX86 && (
Name ==
"sse41.pblendw" ||
2567 Name.startswith(
"sse41.blendp") ||
2568 Name.startswith(
"avx.blend.p") ||
2569 Name ==
"avx2.pblendw" ||
2570 Name.startswith(
"avx2.pblendd."))) {
2573 unsigned Imm = cast <ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2574 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2575 unsigned NumElts = VecTy->getNumElements();
2578 for (
unsigned i = 0;
i != NumElts; ++
i)
2579 Idxs[
i] = ((Imm >> (
i%8)) & 1) ?
i + NumElts :
i;
2581 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
2582 }
else if (IsX86 && (
Name.startswith(
"avx.vinsertf128.") ||
2583 Name ==
"avx2.vinserti128" ||
2584 Name.startswith(
"avx512.mask.insert"))) {
2587 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2588 unsigned DstNumElts =
2589 cast<FixedVectorType>(CI->
getType())->getNumElements();
2590 unsigned SrcNumElts =
2591 cast<FixedVectorType>(Op1->
getType())->getNumElements();
2592 unsigned Scale = DstNumElts / SrcNumElts;
2599 for (
unsigned i = 0;
i != SrcNumElts; ++
i)
2601 for (
unsigned i = SrcNumElts;
i != DstNumElts; ++
i)
2602 Idxs[
i] = SrcNumElts;
2603 Rep =
Builder.CreateShuffleVector(Op1, Idxs);
2617 for (
unsigned i = 0;
i != DstNumElts; ++
i)
2620 for (
unsigned i = 0;
i != SrcNumElts; ++
i)
2621 Idxs[
i + Imm * SrcNumElts] =
i + DstNumElts;
2622 Rep =
Builder.CreateShuffleVector(Op0, Rep, Idxs);
2628 }
else if (IsX86 && (
Name.startswith(
"avx.vextractf128.") ||
2629 Name ==
"avx2.vextracti128" ||
2630 Name.startswith(
"avx512.mask.vextract"))) {
2632 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2633 unsigned DstNumElts =
2634 cast<FixedVectorType>(CI->
getType())->getNumElements();
2635 unsigned SrcNumElts =
2636 cast<FixedVectorType>(Op0->
getType())->getNumElements();
2637 unsigned Scale = SrcNumElts / DstNumElts;
2644 for (
unsigned i = 0;
i != DstNumElts; ++
i) {
2645 Idxs[
i] =
i + (Imm * DstNumElts);
2647 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2653 }
else if (!IsX86 &&
Name ==
"stackprotectorcheck") {
2655 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.perm.df.") ||
2656 Name.startswith(
"avx512.mask.perm.di."))) {
2658 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2659 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2660 unsigned NumElts = VecTy->getNumElements();
2663 for (
unsigned i = 0;
i != NumElts; ++
i)
2664 Idxs[
i] = (
i & ~0
x3) + ((Imm >> (2 * (
i & 0x3))) & 3);
2666 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2671 }
else if (IsX86 && (
Name.startswith(
"avx.vperm2f128.") ||
2672 Name ==
"avx2.vperm2i128")) {
2681 uint8_t Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2683 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2684 unsigned HalfSize = NumElts / 2;
2696 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
2697 for (
unsigned i = 0;
i < HalfSize; ++
i)
2698 ShuffleMask[
i] = StartIndex +
i;
2701 StartIndex = (Imm & 0x10) ? HalfSize : 0;
2702 for (
unsigned i = 0;
i < HalfSize; ++
i)
2703 ShuffleMask[
i + HalfSize] = NumElts + StartIndex +
i;
2705 Rep =
Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2707 }
else if (IsX86 && (
Name.startswith(
"avx.vpermil.") ||
2708 Name ==
"sse2.pshuf.d" ||
2709 Name.startswith(
"avx512.mask.vpermil.p") ||
2710 Name.startswith(
"avx512.mask.pshuf.d."))) {
2712 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2713 auto *VecTy = cast<FixedVectorType>(CI->
getType());
2714 unsigned NumElts = VecTy->getNumElements();
2716 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
2717 unsigned IdxMask = ((1 << IdxSize) - 1);
2723 for (
unsigned i = 0;
i != NumElts; ++
i)
2724 Idxs[
i] = ((Imm >> ((
i * IdxSize) % 8)) & IdxMask) | (
i & ~IdxMask);
2726 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2731 }
else if (IsX86 && (
Name ==
"sse2.pshufl.w" ||
2732 Name.startswith(
"avx512.mask.pshufl.w."))) {
2734 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2735 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2738 for (
unsigned l = 0;
l != NumElts;
l += 8) {
2739 for (
unsigned i = 0;
i != 4; ++
i)
2740 Idxs[
i +
l] = ((Imm >> (2 *
i)) & 0
x3) +
l;
2741 for (
unsigned i = 4;
i != 8; ++
i)
2742 Idxs[
i +
l] =
i +
l;
2745 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2750 }
else if (IsX86 && (
Name ==
"sse2.pshufh.w" ||
2751 Name.startswith(
"avx512.mask.pshufh.w."))) {
2753 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(1))->getZExtValue();
2754 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2757 for (
unsigned l = 0;
l != NumElts;
l += 8) {
2758 for (
unsigned i = 0;
i != 4; ++
i)
2759 Idxs[
i +
l] =
i +
l;
2760 for (
unsigned i = 0;
i != 4; ++
i)
2761 Idxs[
i +
l + 4] = ((Imm >> (2 *
i)) & 0
x3) + 4 +
l;
2764 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2769 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.shuf.p")) {
2772 unsigned Imm = cast<ConstantInt>(CI->
getArgOperand(2))->getZExtValue();
2773 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2776 unsigned HalfLaneElts = NumLaneElts / 2;
2779 for (
unsigned i = 0;
i != NumElts; ++
i) {
2781 Idxs[
i] =
i - (
i % NumLaneElts);
2783 if ((
i % NumLaneElts) >= HalfLaneElts)
2787 Idxs[
i] += (Imm >> ((
i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
2790 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
2794 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.movddup") ||
2795 Name.startswith(
"avx512.mask.movshdup") ||
2796 Name.startswith(
"avx512.mask.movsldup"))) {
2798 unsigned NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2802 if (
Name.startswith(
"avx512.mask.movshdup."))
2806 for (
unsigned l = 0;
l != NumElts;
l += NumLaneElts)
2807 for (
unsigned i = 0;
i != NumLaneElts;
i += 2) {
2812 Rep =
Builder.CreateShuffleVector(Op0, Op0, Idxs);
2816 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.punpckl") ||
2817 Name.startswith(
"avx512.mask.unpckl."))) {
2820 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2824 for (
int l = 0;
l != NumElts;
l += NumLaneElts)
2825 for (
int i = 0;
i != NumLaneElts; ++
i)
2826 Idxs[
i +
l] =
l + (
i / 2) + NumElts * (
i % 2);
2828 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
2832 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.punpckh") ||
2833 Name.startswith(
"avx512.mask.unpckh."))) {
2836 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
2840 for (
int l = 0;
l != NumElts;
l += NumLaneElts)
2841 for (
int i = 0;
i != NumLaneElts; ++
i)
2842 Idxs[
i +
l] = (NumLaneElts / 2) +
l + (
i / 2) + NumElts * (
i % 2);
2844 Rep =
Builder.CreateShuffleVector(Op0, Op1, Idxs);
2848 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.and.") ||
2849 Name.startswith(
"avx512.mask.pand."))) {
2854 Rep =
Builder.CreateBitCast(Rep, FTy);
2857 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.andn.") ||
2858 Name.startswith(
"avx512.mask.pandn."))) {
2864 Rep =
Builder.CreateBitCast(Rep, FTy);
2867 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.or.") ||
2868 Name.startswith(
"avx512.mask.por."))) {
2873 Rep =
Builder.CreateBitCast(Rep, FTy);
2876 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.xor.") ||
2877 Name.startswith(
"avx512.mask.pxor."))) {
2882 Rep =
Builder.CreateBitCast(Rep, FTy);
2885 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.padd.")) {
2889 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psub.")) {
2893 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.pmull.")) {
2897 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.add.p")) {
2898 if (
Name.endswith(
".512")) {
2900 if (
Name[17] ==
's')
2901 IID = Intrinsic::x86_avx512_add_ps_512;
2903 IID = Intrinsic::x86_avx512_add_pd_512;
2906 { CI->getArgOperand(0), CI->getArgOperand(1),
2907 CI->getArgOperand(4) });
2913 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.div.p")) {
2914 if (
Name.endswith(
".512")) {
2916 if (
Name[17] ==
's')
2917 IID = Intrinsic::x86_avx512_div_ps_512;
2919 IID = Intrinsic::x86_avx512_div_pd_512;
2922 { CI->getArgOperand(0), CI->getArgOperand(1),
2923 CI->getArgOperand(4) });
2929 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.mul.p")) {
2930 if (
Name.endswith(
".512")) {
2932 if (
Name[17] ==
's')
2933 IID = Intrinsic::x86_avx512_mul_ps_512;
2935 IID = Intrinsic::x86_avx512_mul_pd_512;
2938 { CI->getArgOperand(0), CI->getArgOperand(1),
2939 CI->getArgOperand(4) });
2945 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.sub.p")) {
2946 if (
Name.endswith(
".512")) {
2948 if (
Name[17] ==
's')
2949 IID = Intrinsic::x86_avx512_sub_ps_512;
2951 IID = Intrinsic::x86_avx512_sub_pd_512;
2954 { CI->getArgOperand(0), CI->getArgOperand(1),
2955 CI->getArgOperand(4) });
2961 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.max.p") ||
2962 Name.startswith(
"avx512.mask.min.p")) &&
2963 Name.drop_front(18) ==
".512") {
2964 bool IsDouble =
Name[17] ==
'd';
2965 bool IsMin =
Name[13] ==
'i';
2967 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
2968 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
2973 { CI->getArgOperand(0), CI->getArgOperand(1),
2974 CI->getArgOperand(4) });
2977 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.lzcnt.")) {
2981 { CI->getArgOperand(0), Builder.getInt1(false) });
2984 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psll")) {
2985 bool IsImmediate =
Name[16] ==
'i' ||
2986 (
Name.size() > 18 &&
Name[18] ==
'i');
2987 bool IsVariable =
Name[16] ==
'v';
2994 if (IsVariable &&
Name[17] !=
'.') {
2995 if (
Size ==
'd' &&
Name[17] ==
'2')
2996 IID = Intrinsic::x86_avx2_psllv_q;
2997 else if (
Size ==
'd' &&
Name[17] ==
'4')
2998 IID = Intrinsic::x86_avx2_psllv_q_256;
2999 else if (
Size ==
's' &&
Name[17] ==
'4')
3000 IID = Intrinsic::x86_avx2_psllv_d;
3001 else if (
Size ==
's' &&
Name[17] ==
'8')
3002 IID = Intrinsic::x86_avx2_psllv_d_256;
3003 else if (
Size ==
'h' &&
Name[17] ==
'8')
3004 IID = Intrinsic::x86_avx512_psllv_w_128;
3005 else if (
Size ==
'h' &&
Name[17] ==
'1')
3006 IID = Intrinsic::x86_avx512_psllv_w_256;
3007 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3008 IID = Intrinsic::x86_avx512_psllv_w_512;
3011 }
else if (
Name.endswith(
".128")) {
3013 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3014 : Intrinsic::x86_sse2_psll_d;
3015 else if (
Size ==
'q')
3016 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3017 : Intrinsic::x86_sse2_psll_q;
3018 else if (
Size ==
'w')
3019 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3020 : Intrinsic::x86_sse2_psll_w;
3023 }
else if (
Name.endswith(
".256")) {
3025 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3026 : Intrinsic::x86_avx2_psll_d;
3027 else if (
Size ==
'q')
3028 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3029 : Intrinsic::x86_avx2_psll_q;
3030 else if (
Size ==
'w')
3031 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3032 : Intrinsic::x86_avx2_psll_w;
3037 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3038 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3039 Intrinsic::x86_avx512_psll_d_512;
3040 else if (
Size ==
'q')
3041 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3042 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3043 Intrinsic::x86_avx512_psll_q_512;
3044 else if (
Size ==
'w')
3045 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3046 : Intrinsic::x86_avx512_psll_w_512;
3052 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psrl")) {
3053 bool IsImmediate =
Name[16] ==
'i' ||
3054 (
Name.size() > 18 &&
Name[18] ==
'i');
3055 bool IsVariable =
Name[16] ==
'v';
3062 if (IsVariable &&
Name[17] !=
'.') {
3063 if (
Size ==
'd' &&
Name[17] ==
'2')
3064 IID = Intrinsic::x86_avx2_psrlv_q;
3065 else if (
Size ==
'd' &&
Name[17] ==
'4')
3066 IID = Intrinsic::x86_avx2_psrlv_q_256;
3067 else if (
Size ==
's' &&
Name[17] ==
'4')
3068 IID = Intrinsic::x86_avx2_psrlv_d;
3069 else if (
Size ==
's' &&
Name[17] ==
'8')
3070 IID = Intrinsic::x86_avx2_psrlv_d_256;
3071 else if (
Size ==
'h' &&
Name[17] ==
'8')
3072 IID = Intrinsic::x86_avx512_psrlv_w_128;
3073 else if (
Size ==
'h' &&
Name[17] ==
'1')
3074 IID = Intrinsic::x86_avx512_psrlv_w_256;
3075 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3076 IID = Intrinsic::x86_avx512_psrlv_w_512;
3079 }
else if (
Name.endswith(
".128")) {
3081 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3082 : Intrinsic::x86_sse2_psrl_d;
3083 else if (
Size ==
'q')
3084 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3085 : Intrinsic::x86_sse2_psrl_q;
3086 else if (
Size ==
'w')
3087 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3088 : Intrinsic::x86_sse2_psrl_w;
3091 }
else if (
Name.endswith(
".256")) {
3093 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3094 : Intrinsic::x86_avx2_psrl_d;
3095 else if (
Size ==
'q')
3096 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3097 : Intrinsic::x86_avx2_psrl_q;
3098 else if (
Size ==
'w')
3099 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3100 : Intrinsic::x86_avx2_psrl_w;
3105 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3106 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3107 Intrinsic::x86_avx512_psrl_d_512;
3108 else if (
Size ==
'q')
3109 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3110 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3111 Intrinsic::x86_avx512_psrl_q_512;
3112 else if (
Size ==
'w')
3113 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3114 : Intrinsic::x86_avx512_psrl_w_512;
3120 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.psra")) {
3121 bool IsImmediate =
Name[16] ==
'i' ||
3122 (
Name.size() > 18 &&
Name[18] ==
'i');
3123 bool IsVariable =
Name[16] ==
'v';
3130 if (IsVariable &&
Name[17] !=
'.') {
3131 if (
Size ==
's' &&
Name[17] ==
'4')
3132 IID = Intrinsic::x86_avx2_psrav_d;
3133 else if (
Size ==
's' &&
Name[17] ==
'8')
3134 IID = Intrinsic::x86_avx2_psrav_d_256;
3135 else if (
Size ==
'h' &&
Name[17] ==
'8')
3136 IID = Intrinsic::x86_avx512_psrav_w_128;
3137 else if (
Size ==
'h' &&
Name[17] ==
'1')
3138 IID = Intrinsic::x86_avx512_psrav_w_256;
3139 else if (
Name[17] ==
'3' &&
Name[18] ==
'2')
3140 IID = Intrinsic::x86_avx512_psrav_w_512;
3143 }
else if (
Name.endswith(
".128")) {
3145 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3146 : Intrinsic::x86_sse2_psra_d;
3147 else if (
Size ==
'q')
3148 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3149 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3150 Intrinsic::x86_avx512_psra_q_128;
3151 else if (
Size ==
'w')
3152 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3153 : Intrinsic::x86_sse2_psra_w;
3156 }
else if (
Name.endswith(
".256")) {
3158 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3159 : Intrinsic::x86_avx2_psra_d;
3160 else if (
Size ==
'q')
3161 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3162 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3163 Intrinsic::x86_avx512_psra_q_256;
3164 else if (
Size ==
'w')
3165 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3166 : Intrinsic::x86_avx2_psra_w;
3171 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3172 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3173 Intrinsic::x86_avx512_psra_d_512;
3174 else if (
Size ==
'q')
3175 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3176 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3177 Intrinsic::x86_avx512_psra_q_512;
3178 else if (
Size ==
'w')
3179 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3180 : Intrinsic::x86_avx512_psra_w_512;
3186 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.move.s")) {
3188 }
else if (IsX86 &&
Name.startswith(
"avx512.cvtmask2")) {
3190 }
else if (IsX86 &&
Name.endswith(
".movntdqa")) {
3205 }
else if (IsX86 && (
Name.startswith(
"fma.vfmadd.") ||
3206 Name.startswith(
"fma.vfmsub.") ||
3207 Name.startswith(
"fma.vfnmadd.") ||
3208 Name.startswith(
"fma.vfnmsub."))) {
3209 bool NegMul =
Name[6] ==
'n';
3210 bool NegAcc = NegMul ?
Name[8] ==
's' :
Name[7] ==
's';
3211 bool IsScalar = NegMul ?
Name[12] ==
's' :
Name[11] ==
's';
3217 Ops[0] =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3218 Ops[1] =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3219 Ops[2] =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3222 if (NegMul && !IsScalar)
3223 Ops[0] =
Builder.CreateFNeg(Ops[0]);
3224 if (NegMul && IsScalar)
3225 Ops[1] =
Builder.CreateFNeg(Ops[1]);
3227 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3237 }
else if (IsX86 &&
Name.startswith(
"fma4.vfmadd.s")) {
3241 Ops[0] =
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3242 Ops[1] =
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3243 Ops[2] =
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3252 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vfmadd.s") ||
3253 Name.startswith(
"avx512.maskz.vfmadd.s") ||
3254 Name.startswith(
"avx512.mask3.vfmadd.s") ||
3255 Name.startswith(
"avx512.mask3.vfmsub.s") ||
3256 Name.startswith(
"avx512.mask3.vfnmsub.s"))) {
3257 bool IsMask3 =
Name[11] ==
'3';
3258 bool IsMaskZ =
Name[11] ==
'z';
3260 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3261 bool NegMul =
Name[2] ==
'n';
3262 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3268 if (NegMul && (IsMask3 || IsMaskZ))
3270 if (NegMul && !(IsMask3 || IsMaskZ))
3275 A =
Builder.CreateExtractElement(A, (uint64_t)0);
3276 B =
Builder.CreateExtractElement(
B, (uint64_t)0);
3277 C =
Builder.CreateExtractElement(
C, (uint64_t)0);
3280 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4) {
3284 if (
Name.back() ==
'd')
3285 IID = Intrinsic::x86_avx512_vfmadd_f64;
3287 IID = Intrinsic::x86_avx512_vfmadd_f32;
3302 if (NegAcc && IsMask3)
3310 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vfmadd.p") ||
3311 Name.startswith(
"avx512.mask.vfnmadd.p") ||
3312 Name.startswith(
"avx512.mask.vfnmsub.p") ||
3313 Name.startswith(
"avx512.mask3.vfmadd.p") ||
3314 Name.startswith(
"avx512.mask3.vfmsub.p") ||
3315 Name.startswith(
"avx512.mask3.vfnmsub.p") ||
3316 Name.startswith(
"avx512.maskz.vfmadd.p"))) {
3317 bool IsMask3 =
Name[11] ==
'3';
3318 bool IsMaskZ =
Name[11] ==
'z';
3320 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3321 bool NegMul =
Name[2] ==
'n';
3322 bool NegAcc = NegMul ?
Name[4] ==
's' :
Name[3] ==
's';
3328 if (NegMul && (IsMask3 || IsMaskZ))
3330 if (NegMul && !(IsMask3 || IsMaskZ))
3337 cast<ConstantInt>(CI->
getArgOperand(4))->getZExtValue() != 4)) {
3341 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3343 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3346 { A, B, C, CI->getArgOperand(4) });
3359 }
else if (IsX86 &&
Name.startswith(
"fma.vfmsubadd.p")) {
3363 if (VecWidth == 128 && EltWidth == 32)
3364 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3365 else if (VecWidth == 256 && EltWidth == 32)
3366 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3367 else if (VecWidth == 128 && EltWidth == 64)
3368 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3369 else if (VecWidth == 256 && EltWidth == 64)
3370 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3376 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3379 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vfmaddsub.p") ||
3380 Name.startswith(
"avx512.mask3.vfmaddsub.p") ||
3381 Name.startswith(
"avx512.maskz.vfmaddsub.p") ||
3382 Name.startswith(
"avx512.mask3.vfmsubadd.p"))) {
3383 bool IsMask3 =
Name[11] ==
'3';
3384 bool IsMaskZ =
Name[11] ==
'z';
3386 Name =
Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3387 bool IsSubAdd =
Name[3] ==
's';
3392 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3394 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3399 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3404 int NumElts = cast<FixedVectorType>(CI->
getType())->getNumElements();
3412 Ops[2] =
Builder.CreateFNeg(Ops[2]);
3419 for (
int i = 0;
i != NumElts; ++
i)
3420 Idxs[
i] =
i + (
i % 2) * NumElts;
3422 Rep =
Builder.CreateShuffleVector(Even, Odd, Idxs);
3430 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.pternlog.") ||
3431 Name.startswith(
"avx512.maskz.pternlog."))) {
3432 bool ZeroMask =
Name[11] ==
'z';
3436 if (VecWidth == 128 && EltWidth == 32)
3437 IID = Intrinsic::x86_avx512_pternlog_d_128;
3438 else if (VecWidth == 256 && EltWidth == 32)
3439 IID = Intrinsic::x86_avx512_pternlog_d_256;
3440 else if (VecWidth == 512 && EltWidth == 32)
3441 IID = Intrinsic::x86_avx512_pternlog_d_512;
3442 else if (VecWidth == 128 && EltWidth == 64)
3443 IID = Intrinsic::x86_avx512_pternlog_q_128;
3444 else if (VecWidth == 256 && EltWidth == 64)
3445 IID = Intrinsic::x86_avx512_pternlog_q_256;
3446 else if (VecWidth == 512 && EltWidth == 64)
3447 IID = Intrinsic::x86_avx512_pternlog_q_512;
3458 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpmadd52") ||
3459 Name.startswith(
"avx512.maskz.vpmadd52"))) {
3460 bool ZeroMask =
Name[11] ==
'z';
3464 if (VecWidth == 128 && !
High)
3465 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3466 else if (VecWidth == 256 && !
High)
3467 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3468 else if (VecWidth == 512 && !
High)
3469 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3470 else if (VecWidth == 128 &&
High)
3471 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3472 else if (VecWidth == 256 &&
High)
3473 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3474 else if (VecWidth == 512 &&
High)
3475 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3486 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpermi2var.") ||
3487 Name.startswith(
"avx512.mask.vpermt2var.") ||
3488 Name.startswith(
"avx512.maskz.vpermt2var."))) {
3489 bool ZeroMask =
Name[11] ==
'z';
3490 bool IndexForm =
Name[17] ==
'i';
3492 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpdpbusd.") ||
3493 Name.startswith(
"avx512.maskz.vpdpbusd.") ||
3494 Name.startswith(
"avx512.mask.vpdpbusds.") ||
3495 Name.startswith(
"avx512.maskz.vpdpbusds."))) {
3496 bool ZeroMask =
Name[11] ==
'z';
3497 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
3500 if (VecWidth == 128 && !IsSaturating)
3501 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3502 else if (VecWidth == 256 && !IsSaturating)
3503 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3504 else if (VecWidth == 512 && !IsSaturating)
3505 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3506 else if (VecWidth == 128 && IsSaturating)
3507 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3508 else if (VecWidth == 256 && IsSaturating)
3509 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3510 else if (VecWidth == 512 && IsSaturating)
3511 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3522 }
else if (IsX86 && (
Name.startswith(
"avx512.mask.vpdpwssd.") ||
3523 Name.startswith(
"avx512.maskz.vpdpwssd.") ||
3524 Name.startswith(
"avx512.mask.vpdpwssds.") ||
3525 Name.startswith(
"avx512.maskz.vpdpwssds."))) {
3526 bool ZeroMask =
Name[11] ==
'z';
3527 bool IsSaturating =
Name[ZeroMask ? 21 : 20] ==
's';
3530 if (VecWidth == 128 && !IsSaturating)
3531 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3532 else if (VecWidth == 256 && !IsSaturating)
3533 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3534 else if (VecWidth == 512 && !IsSaturating)
3535 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3536 else if (VecWidth == 128 && IsSaturating)
3537 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3538 else if (VecWidth == 256 && IsSaturating)
3539 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3540 else if (VecWidth == 512 && IsSaturating)
3541 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3552 }
else if (IsX86 && (
Name ==
"addcarryx.u32" ||
Name ==
"addcarryx.u64" ||
3553 Name ==
"addcarry.u32" ||
Name ==
"addcarry.u64" ||
3554 Name ==
"subborrow.u32" ||
Name ==
"subborrow.u64")) {
3556 if (
Name[0] ==
'a' &&
Name.back() ==
'2')
3557 IID = Intrinsic::x86_addcarry_32;
3558 else if (
Name[0] ==
'a' &&
Name.back() ==
'4')
3559 IID = Intrinsic::x86_addcarry_64;
3560 else if (
Name[0] ==
's' &&
Name.back() ==
'2')
3561 IID = Intrinsic::x86_subborrow_32;
3562 else if (
Name[0] ==
's' &&
Name.back() ==
'4')
3563 IID = Intrinsic::x86_subborrow_64;
3585 }
else if (IsX86 &&
Name.startswith(
"avx512.mask.") &&
3588 }
else if (IsNVVM && (
Name ==
"abs.i" ||
Name ==
"abs.ll")) {
3593 Rep =
Builder.CreateSelect(Cmp,
Arg, Neg,
"abs");
3594 }
else if (IsNVVM && (
Name.startswith(
"atomic.load.add.f32.p") ||
3595 Name.startswith(
"atomic.load.add.f64.p"))) {
3600 }
else if (IsNVVM && (
Name ==
"max.i" ||
Name ==
"max.ll" ||
3601 Name ==
"max.ui" ||
Name ==
"max.ull")) {
3605 ?
Builder.CreateICmpUGE(Arg0, Arg1,
"max.cond")
3606 :
Builder.CreateICmpSGE(Arg0, Arg1,
"max.cond");
3607 Rep =
Builder.CreateSelect(Cmp, Arg0, Arg1,
"max");
3608 }
else if (IsNVVM && (
Name ==
"min.i" ||
Name ==
"min.ll" ||
3609 Name ==
"min.ui" ||
Name ==
"min.ull")) {
3613 ?
Builder.CreateICmpULE(Arg0, Arg1,
"min.cond")
3614 :
Builder.CreateICmpSLE(Arg0, Arg1,
"min.cond");
3615 Rep =
Builder.CreateSelect(Cmp, Arg0, Arg1,
"min");
3616 }
else if (IsNVVM &&
Name ==
"clz.ll") {
3622 {Arg, Builder.getFalse()},
"ctlz");
3623 Rep =
Builder.CreateTrunc(Ctlz,
Builder.getInt32Ty(),
"ctlz.trunc");
3624 }
else if (IsNVVM &&
Name ==
"popc.ll") {
3632 Rep =
Builder.CreateTrunc(Popc,
Builder.getInt32Ty(),
"ctpop.trunc");
3633 }
else if (IsNVVM &&
Name ==
"h2f") {
3635 F->getParent(), Intrinsic::convert_from_fp16,
3636 {Builder.getFloatTy()}),
3648 const auto &DefaultCase = [&NewFn, &CI]() ->
void {
3652 "Unknown function for CallInst upgrade and isn't just a name change");
3661 case Intrinsic::arm_neon_vld1:
3662 case Intrinsic::arm_neon_vld2:
3663 case Intrinsic::arm_neon_vld3:
3664 case Intrinsic::arm_neon_vld4:
3665 case Intrinsic::arm_neon_vld2lane:
3666 case Intrinsic::arm_neon_vld3lane:
3667 case Intrinsic::arm_neon_vld4lane:
3668 case Intrinsic::arm_neon_vst1:
3669 case Intrinsic::arm_neon_vst2:
3670 case Intrinsic::arm_neon_vst3:
3671 case Intrinsic::arm_neon_vst4:
3672 case Intrinsic::arm_neon_vst2lane:
3673 case Intrinsic::arm_neon_vst3lane:
3674 case Intrinsic::arm_neon_vst4lane: {
3681 case Intrinsic::arm_neon_bfdot:
3682 case Intrinsic::arm_neon_bfmmla:
3683 case Intrinsic::arm_neon_bfmlalb:
3684 case Intrinsic::arm_neon_bfmlalt:
3685 case Intrinsic::aarch64_neon_bfdot:
3686 case Intrinsic::aarch64_neon_bfmmla:
3687 case Intrinsic::aarch64_neon_bfmlalb:
3688 case Intrinsic::aarch64_neon_bfmlalt: {
3691 "Mismatch between function args and call args");
3692 size_t OperandWidth =
3694 assert((OperandWidth == 64 || OperandWidth == 128) &&
3695 "Unexpected operand width");
3698 Args.push_back(*Iter++);
3699 Args.push_back(
Builder.CreateBitCast(*Iter++, NewTy));
3700 Args.push_back(
Builder.CreateBitCast(*Iter++, NewTy));
3705 case Intrinsic::bitreverse:
3709 case Intrinsic::ctlz:
3710 case Intrinsic::cttz:
3712 "Mismatch between function args and call args");
3717 case Intrinsic::objectsize: {
3728 case Intrinsic::ctpop:
3732 case Intrinsic::convert_from_fp16:
3736 case Intrinsic::dbg_value:
3741 if (
Offset->isZeroValue()) {
3750 case Intrinsic::ptr_annotation:
3753 "Before LLVM 12.0 this intrinsic took four arguments");
3764 case Intrinsic::var_annotation:
3767 "Before LLVM 12.0 this intrinsic took four arguments");
3776 case Intrinsic::x86_xop_vfrcz_ss:
3777 case Intrinsic::x86_xop_vfrcz_sd:
3781 case Intrinsic::x86_xop_vpermil2pd:
3782 case Intrinsic::x86_xop_vpermil2ps:
3783 case Intrinsic::x86_xop_vpermil2pd_256:
3784 case Intrinsic::x86_xop_vpermil2ps_256: {
3794 case Intrinsic::x86_sse41_ptestc:
3795 case Intrinsic::x86_sse41_ptestz:
3796 case Intrinsic::x86_sse41_ptestnzc: {
3810 Value *BC0 =
Builder.CreateBitCast(Arg0, NewVecTy,
"cast");
3811 Value *BC1 =
Builder.CreateBitCast(Arg1, NewVecTy,
"cast");
3813 NewCall =
Builder.CreateCall(NewFn, {BC0, BC1});
3817 case Intrinsic::x86_rdtscp: {
3823 NewCall =
Builder.CreateCall(NewFn);
3839 case Intrinsic::x86_sse41_insertps:
3840 case Intrinsic::x86_sse41_dppd:
3841 case Intrinsic::x86_sse41_dpps:
3842 case Intrinsic::x86_sse41_mpsadbw:
3843 case Intrinsic::x86_avx_dp_ps_256:
3844 case Intrinsic::x86_avx2_mpsadbw: {
3856 case Intrinsic::x86_avx512_mask_cmp_pd_128:
3857 case Intrinsic::x86_avx512_mask_cmp_pd_256:
3858 case Intrinsic::x86_avx512_mask_cmp_pd_512:
3859 case Intrinsic::x86_avx512_mask_cmp_ps_128:
3860 case Intrinsic::x86_avx512_mask_cmp_ps_256:
3861 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
3865 cast<FixedVectorType>(
Args[0]->
getType())->getNumElements();
3877 case Intrinsic::thread_pointer: {
3878 NewCall =
Builder.CreateCall(NewFn, {});
3882 case Intrinsic::invariant_start:
3883 case Intrinsic::invariant_end:
3884 case Intrinsic::masked_load:
3885 case Intrinsic::masked_store:
3886 case Intrinsic::masked_gather:
3887 case Intrinsic::masked_scatter: {
3895 case Intrinsic::memmove:
3896 case Intrinsic::memset: {
3913 auto *MemCI = cast<MemIntrinsic>(NewCall);
3916 MemCI->setDestAlignment(
Align->getMaybeAlignValue());
3918 if (
auto *MTI = dyn_cast<MemTransferInst>(MemCI))
3919 MTI->setSourceAlignment(
Align->getMaybeAlignValue());
3923 assert(NewCall &&
"Should have either set this variable or returned through "
3924 "the default case");
3931 assert(
F &&
"Illegal attempt to upgrade a non-existent intrinsic.");
3940 if (
CallInst *CI = dyn_cast<CallInst>(U))
3944 F->eraseFromParent();
3958 Metadata *Elts2[] = {ScalarType, ScalarType,
3972 if (Opc != Instruction::BitCast)
3993 if (Opc != Instruction::BitCast)
3996 Type *SrcTy =
C->getType();
4017 bool BrokenDebugInfo =
false;
4020 if (!BrokenDebugInfo)
4026 M.getContext().diagnose(Diag);
4033 M.getContext().diagnose(DiagVersion);
4041 bool Changed =
false;
4043 NamedMDNode *ModRetainReleaseMarker =
M.getNamedMetadata(MarkerKey);
4044 if (ModRetainReleaseMarker) {
4047 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(0));
4050 ID->getString().split(ValueComp,
"#");
4051 if (ValueComp.size() == 2) {
4052 std::string NewValue = ValueComp[0].str() +
";" + ValueComp[1].str();
4056 M.eraseNamedMetadata(ModRetainReleaseMarker);
4067 auto UpgradeToIntrinsic = [&](
const char *OldFunc,
4077 CallInst *CI = dyn_cast<CallInst>(U);
4092 bool InvalidCast =
false;
4099 if (I < NewFuncTy->getNumParams()) {
4134 UpgradeToIntrinsic(
"clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
4142 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
4143 {
"objc_autorelease", llvm::Intrinsic::objc_autorelease},
4144 {
"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
4145 {
"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
4146 {
"objc_autoreleaseReturnValue",
4147 llvm::Intrinsic::objc_autoreleaseReturnValue},
4148 {
"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
4149 {
"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
4150 {
"objc_initWeak", llvm::Intrinsic::objc_initWeak},
4151 {
"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
4152 {
"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
4153 {
"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
4154 {
"objc_release", llvm::Intrinsic::objc_release},
4155 {
"objc_retain", llvm::Intrinsic::objc_retain},
4156 {
"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
4157 {
"objc_retainAutoreleaseReturnValue",
4158 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
4159 {
"objc_retainAutoreleasedReturnValue",
4160 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
4161 {
"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
4162 {
"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
4163 {
"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
4164 {
"objc_unsafeClaimAutoreleasedReturnValue",
4165 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
4166 {
"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
4167 {
"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
4168 {
"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
4169 {
"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
4170 {
"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
4171 {
"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
4172 {
"objc_arc_annotation_topdown_bbstart",
4173 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
4174 {
"objc_arc_annotation_topdown_bbend",
4175 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
4176 {
"objc_arc_annotation_bottomup_bbstart",
4177 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
4178 {
"objc_arc_annotation_bottomup_bbend",
4179 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
4181 for (
auto &
I : RuntimeFuncs)
4182 UpgradeToIntrinsic(
I.first,
I.second);
4190 bool HasObjCFlag =
false, HasClassProperties =
false, Changed =
false;
4191 bool HasSwiftVersionFlag =
false;
4192 uint8_t SwiftMajorVersion, SwiftMinorVersion;
4199 if (
Op->getNumOperands() != 3)
4201 MDString *
ID = dyn_cast_or_null<MDString>(
Op->getOperand(1));
4204 if (
ID->getString() ==
"Objective-C Image Info Version")
4206 if (
ID->getString() ==
"Objective-C Class Properties")
4207 HasClassProperties =
true;
4210 if (
ID->getString() ==
"PIC Level" ||
ID->getString() ==
"PIE Level") {
4211 if (
auto *Behavior =
4212 mdconst::dyn_extract_or_null<ConstantInt>(
Op->getOperand(0))) {
4227 if (
ID->getString() ==
"Objective-C Image Info Section") {
4228 if (
auto *
Value = dyn_cast_or_null<MDString>(
Op->getOperand(2))) {
4230 Value->getString().split(ValueComp,
" ");
4231 if (ValueComp.size() != 1) {
4232 std::string NewValue;
4233 for (
auto &
S : ValueComp)
4234 NewValue +=
S.str();
4235 Metadata *Ops[3] = {
Op->getOperand(0),
Op->getOperand(1),
4245 if (
ID->getString() ==
"Objective-C Garbage Collection") {
4246 auto Md = dyn_cast<ConstantAsMetadata>(
Op->getOperand(2));
4248 assert(Md->getValue() &&
"Expected non-empty metadata");
4249 auto Type = Md->getValue()->getType();