LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "IROptions.h"
17#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/StringRef.h"
23#include "llvm/IR/Attributes.h"
24#include "llvm/IR/CallingConv.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DebugInfo.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalValue.h"
31#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/InstVisitor.h"
34#include "llvm/IR/Instruction.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsAArch64.h"
38#include "llvm/IR/IntrinsicsAMDGPU.h"
39#include "llvm/IR/IntrinsicsARM.h"
40#include "llvm/IR/IntrinsicsNVPTX.h"
41#include "llvm/IR/IntrinsicsRISCV.h"
42#include "llvm/IR/IntrinsicsWebAssembly.h"
43#include "llvm/IR/IntrinsicsX86.h"
44#include "llvm/IR/LLVMContext.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/Module.h"
49#include "llvm/IR/Value.h"
50#include "llvm/IR/Verifier.h"
53#include "llvm/Support/Debug.h"
57#include "llvm/Support/Regex.h"
60#include <cstdint>
61#include <cstring>
62#include <numeric>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "auto-upgrade"
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name.starts_with("pmulh.w") || // Added in 24.0
216 Name.starts_with("pmulhu.w") || // Added in 24.0
217 Name == "pmul.dq" || // Added in 7.0
218 Name == "pmulu.dq" || // Added in 7.0
219 Name.starts_with("psll.dq") || // Added in 3.7
220 Name.starts_with("psrl.dq") || // Added in 3.7
221 Name.starts_with("psubs.") || // Added in 8.0
222 Name.starts_with("psubus.") || // Added in 8.0
223 Name.starts_with("vbroadcast") || // Added in 3.8
224 Name == "vbroadcasti128" || // Added in 3.7
225 Name == "vextracti128" || // Added in 3.7
226 Name == "vinserti128" || // Added in 3.7
227 Name == "vperm2i128"); // Added in 6.0
228
229 if (Name.consume_front("avx512.")) {
230 if (Name.consume_front("mask."))
231 // 'avx512.mask.*'
232 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
233 Name.starts_with("and.") || // Added in 3.9
234 Name.starts_with("andn.") || // Added in 3.9
235 Name.starts_with("broadcast.s") || // Added in 3.9
236 Name.starts_with("broadcastf32x4.") || // Added in 6.0
237 Name.starts_with("broadcastf32x8.") || // Added in 6.0
238 Name.starts_with("broadcastf64x2.") || // Added in 6.0
239 Name.starts_with("broadcastf64x4.") || // Added in 6.0
240 Name.starts_with("broadcasti32x4.") || // Added in 6.0
241 Name.starts_with("broadcasti32x8.") || // Added in 6.0
242 Name.starts_with("broadcasti64x2.") || // Added in 6.0
243 Name.starts_with("broadcasti64x4.") || // Added in 6.0
244 Name.starts_with("cmp.b") || // Added in 5.0
245 Name.starts_with("cmp.d") || // Added in 5.0
246 Name.starts_with("cmp.q") || // Added in 5.0
247 Name.starts_with("cmp.w") || // Added in 5.0
248 Name.starts_with("compress.b") || // Added in 9.0
249 Name.starts_with("compress.d") || // Added in 9.0
250 Name.starts_with("compress.p") || // Added in 9.0
251 Name.starts_with("compress.q") || // Added in 9.0
252 Name.starts_with("compress.store.") || // Added in 7.0
253 Name.starts_with("compress.w") || // Added in 9.0
254 Name.starts_with("conflict.") || // Added in 9.0
255 Name.starts_with("cvtdq2pd.") || // Added in 4.0
256 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
257 Name == "cvtpd2dq.256" || // Added in 7.0
258 Name == "cvtpd2ps.256" || // Added in 7.0
259 Name == "cvtps2pd.128" || // Added in 7.0
260 Name == "cvtps2pd.256" || // Added in 7.0
261 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
262 Name == "cvtqq2ps.256" || // Added in 9.0
263 Name == "cvtqq2ps.512" || // Added in 9.0
264 Name == "cvttpd2dq.256" || // Added in 7.0
265 Name == "cvttps2dq.128" || // Added in 7.0
266 Name == "cvttps2dq.256" || // Added in 7.0
267 Name.starts_with("cvtudq2pd.") || // Added in 4.0
268 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
269 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
270 Name == "cvtuqq2ps.256" || // Added in 9.0
271 Name == "cvtuqq2ps.512" || // Added in 9.0
272 Name.starts_with("dbpsadbw.") || // Added in 7.0
273 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
274 Name.starts_with("expand.b") || // Added in 9.0
275 Name.starts_with("expand.d") || // Added in 9.0
276 Name.starts_with("expand.load.") || // Added in 7.0
277 Name.starts_with("expand.p") || // Added in 9.0
278 Name.starts_with("expand.q") || // Added in 9.0
279 Name.starts_with("expand.w") || // Added in 9.0
280 Name.starts_with("fpclass.p") || // Added in 7.0
281 Name.starts_with("insert") || // Added in 4.0
282 Name.starts_with("load.") || // Added in 3.9
283 Name.starts_with("loadu.") || // Added in 3.9
284 Name.starts_with("lzcnt.") || // Added in 5.0
285 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
286 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
287 Name.starts_with("movddup") || // Added in 3.9
288 Name.starts_with("move.s") || // Added in 4.0
289 Name.starts_with("movshdup") || // Added in 3.9
290 Name.starts_with("movsldup") || // Added in 3.9
291 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
292 Name.starts_with("or.") || // Added in 3.9
293 Name.starts_with("pabs.") || // Added in 6.0
294 Name.starts_with("packssdw.") || // Added in 5.0
295 Name.starts_with("packsswb.") || // Added in 5.0
296 Name.starts_with("packusdw.") || // Added in 5.0
297 Name.starts_with("packuswb.") || // Added in 5.0
298 Name.starts_with("padd.") || // Added in 4.0
299 Name.starts_with("padds.") || // Added in 8.0
300 Name.starts_with("paddus.") || // Added in 8.0
301 Name.starts_with("palignr.") || // Added in 3.9
302 Name.starts_with("pand.") || // Added in 3.9
303 Name.starts_with("pandn.") || // Added in 3.9
304 Name.starts_with("pavg") || // Added in 6.0
305 Name.starts_with("pbroadcast") || // Added in 6.0
306 Name.starts_with("pcmpeq.") || // Added in 3.9
307 Name.starts_with("pcmpgt.") || // Added in 3.9
308 Name.starts_with("perm.df.") || // Added in 3.9
309 Name.starts_with("perm.di.") || // Added in 3.9
310 Name.starts_with("permvar.") || // Added in 7.0
311 Name.starts_with("pmaddubs.w.") || // Added in 7.0
312 Name.starts_with("pmaddw.d.") || // Added in 7.0
313 Name.starts_with("pmax") || // Added in 4.0
314 Name.starts_with("pmin") || // Added in 4.0
315 Name == "pmov.qd.256" || // Added in 9.0
316 Name == "pmov.qd.512" || // Added in 9.0
317 Name == "pmov.wb.256" || // Added in 9.0
318 Name == "pmov.wb.512" || // Added in 9.0
319 Name.starts_with("pmovsx") || // Added in 4.0
320 Name.starts_with("pmovzx") || // Added in 4.0
321 Name.starts_with("pmul.dq.") || // Added in 4.0
322 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
323 Name.starts_with("pmulh.w.") || // Added in 7.0
324 Name.starts_with("pmulhu.w.") || // Added in 7.0
325 Name.starts_with("pmull.") || // Added in 4.0
326 Name.starts_with("pmultishift.qb.") || // Added in 8.0
327 Name.starts_with("pmulu.dq.") || // Added in 4.0
328 Name.starts_with("por.") || // Added in 3.9
329 Name.starts_with("prol.") || // Added in 8.0
330 Name.starts_with("prolv.") || // Added in 8.0
331 Name.starts_with("pror.") || // Added in 8.0
332 Name.starts_with("prorv.") || // Added in 8.0
333 Name.starts_with("pshuf.b.") || // Added in 4.0
334 Name.starts_with("pshuf.d.") || // Added in 3.9
335 Name.starts_with("pshufh.w.") || // Added in 3.9
336 Name.starts_with("pshufl.w.") || // Added in 3.9
337 Name.starts_with("psll.d") || // Added in 4.0
338 Name.starts_with("psll.q") || // Added in 4.0
339 Name.starts_with("psll.w") || // Added in 4.0
340 Name.starts_with("pslli") || // Added in 4.0
341 Name.starts_with("psllv") || // Added in 4.0
342 Name.starts_with("psra.d") || // Added in 4.0
343 Name.starts_with("psra.q") || // Added in 4.0
344 Name.starts_with("psra.w") || // Added in 4.0
345 Name.starts_with("psrai") || // Added in 4.0
346 Name.starts_with("psrav") || // Added in 4.0
347 Name.starts_with("psrl.d") || // Added in 4.0
348 Name.starts_with("psrl.q") || // Added in 4.0
349 Name.starts_with("psrl.w") || // Added in 4.0
350 Name.starts_with("psrli") || // Added in 4.0
351 Name.starts_with("psrlv") || // Added in 4.0
352 Name.starts_with("psub.") || // Added in 4.0
353 Name.starts_with("psubs.") || // Added in 8.0
354 Name.starts_with("psubus.") || // Added in 8.0
355 Name.starts_with("pternlog.") || // Added in 7.0
356 Name.starts_with("punpckh") || // Added in 3.9
357 Name.starts_with("punpckl") || // Added in 3.9
358 Name.starts_with("pxor.") || // Added in 3.9
359 Name.starts_with("shuf.f") || // Added in 6.0
360 Name.starts_with("shuf.i") || // Added in 6.0
361 Name.starts_with("shuf.p") || // Added in 4.0
362 Name.starts_with("sqrt.p") || // Added in 7.0
363 Name.starts_with("store.b.") || // Added in 3.9
364 Name.starts_with("store.d.") || // Added in 3.9
365 Name.starts_with("store.p") || // Added in 3.9
366 Name.starts_with("store.q.") || // Added in 3.9
367 Name.starts_with("store.w.") || // Added in 3.9
368 Name == "store.ss" || // Added in 7.0
369 Name.starts_with("storeu.") || // Added in 3.9
370 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
371 Name.starts_with("ucmp.") || // Added in 5.0
372 Name.starts_with("unpckh.") || // Added in 3.9
373 Name.starts_with("unpckl.") || // Added in 3.9
374 Name.starts_with("valign.") || // Added in 4.0
375 Name == "vcvtph2ps.128" || // Added in 11.0
376 Name == "vcvtph2ps.256" || // Added in 11.0
377 Name.starts_with("vextract") || // Added in 4.0
378 Name.starts_with("vfmadd.") || // Added in 7.0
379 Name.starts_with("vfmaddsub.") || // Added in 7.0
380 Name.starts_with("vfnmadd.") || // Added in 7.0
381 Name.starts_with("vfnmsub.") || // Added in 7.0
382 Name.starts_with("vpdpbusd.") || // Added in 7.0
383 Name.starts_with("vpdpbusds.") || // Added in 7.0
384 Name.starts_with("vpdpwssd.") || // Added in 7.0
385 Name.starts_with("vpdpwssds.") || // Added in 7.0
386 Name.starts_with("vpermi2var.") || // Added in 7.0
387 Name.starts_with("vpermil.p") || // Added in 3.9
388 Name.starts_with("vpermilvar.") || // Added in 4.0
389 Name.starts_with("vpermt2var.") || // Added in 7.0
390 Name.starts_with("vpmadd52") || // Added in 7.0
391 Name.starts_with("vpshld.") || // Added in 7.0
392 Name.starts_with("vpshldv.") || // Added in 8.0
393 Name.starts_with("vpshrd.") || // Added in 7.0
394 Name.starts_with("vpshrdv.") || // Added in 8.0
395 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
396 Name.starts_with("xor.")); // Added in 3.9
397
398 if (Name.consume_front("mask3."))
399 // 'avx512.mask3.*'
400 return (Name.starts_with("vfmadd.") || // Added in 7.0
401 Name.starts_with("vfmaddsub.") || // Added in 7.0
402 Name.starts_with("vfmsub.") || // Added in 7.0
403 Name.starts_with("vfmsubadd.") || // Added in 7.0
404 Name.starts_with("vfnmsub.")); // Added in 7.0
405
406 if (Name.consume_front("maskz."))
407 // 'avx512.maskz.*'
408 return (Name.starts_with("pternlog.") || // Added in 7.0
409 Name.starts_with("vfmadd.") || // Added in 7.0
410 Name.starts_with("vfmaddsub.") || // Added in 7.0
411 Name.starts_with("vpdpbusd.") || // Added in 7.0
412 Name.starts_with("vpdpbusds.") || // Added in 7.0
413 Name.starts_with("vpdpwssd.") || // Added in 7.0
414 Name.starts_with("vpdpwssds.") || // Added in 7.0
415 Name.starts_with("vpermt2var.") || // Added in 7.0
416 Name.starts_with("vpmadd52") || // Added in 7.0
417 Name.starts_with("vpshldv.") || // Added in 8.0
418 Name.starts_with("vpshrdv.")); // Added in 8.0
419
420 // 'avx512.*'
421 return (Name == "movntdqa" || // Added in 5.0
422 Name == "pmul.dq.512" || // Added in 7.0
423 Name == "pmulu.dq.512" || // Added in 7.0
424 Name.starts_with("broadcastm") || // Added in 6.0
425 Name.starts_with("cmp.p") || // Added in 12.0
426 Name.starts_with("cvtb2mask.") || // Added in 7.0
427 Name.starts_with("cvtd2mask.") || // Added in 7.0
428 Name.starts_with("cvtmask2") || // Added in 5.0
429 Name.starts_with("cvtq2mask.") || // Added in 7.0
430 Name == "cvtusi2sd" || // Added in 7.0
431 Name.starts_with("cvtw2mask.") || // Added in 7.0
432 Name == "kand.w" || // Added in 7.0
433 Name == "kandn.w" || // Added in 7.0
434 Name == "knot.w" || // Added in 7.0
435 Name == "kor.w" || // Added in 7.0
436 Name == "kortestc.w" || // Added in 7.0
437 Name == "kortestz.w" || // Added in 7.0
438 Name.starts_with("kunpck") || // added in 6.0
439 Name == "kxnor.w" || // Added in 7.0
440 Name == "kxor.w" || // Added in 7.0
441 Name.starts_with("padds.") || // Added in 8.0
442 Name.starts_with("pbroadcast") || // Added in 3.9
443 Name.starts_with("pmulh.w") || // Added in 24.0
444 Name.starts_with("pmulhu.w") || // Added in 24.0
445 Name.starts_with("prol") || // Added in 8.0
446 Name.starts_with("pror") || // Added in 8.0
447 Name.starts_with("psll.dq") || // Added in 3.9
448 Name.starts_with("psrl.dq") || // Added in 3.9
449 Name.starts_with("psubs.") || // Added in 8.0
450 Name.starts_with("ptestm") || // Added in 6.0
451 Name.starts_with("ptestnm") || // Added in 6.0
452 Name.starts_with("storent.") || // Added in 3.9
453 Name.starts_with("vbroadcast.s") || // Added in 7.0
454 Name.starts_with("vpshld.") || // Added in 8.0
455 Name.starts_with("vpshrd.")); // Added in 8.0
456 }
457
458 if (Name.consume_front("fma."))
459 return (Name.starts_with("vfmadd.") || // Added in 7.0
460 Name.starts_with("vfmsub.") || // Added in 7.0
461 Name.starts_with("vfmsubadd.") || // Added in 7.0
462 Name.starts_with("vfnmadd.") || // Added in 7.0
463 Name.starts_with("vfnmsub.")); // Added in 7.0
464
465 if (Name.consume_front("fma4."))
466 return Name.starts_with("vfmadd.s"); // Added in 7.0
467
468 if (Name.consume_front("sse."))
469 return (Name == "add.ss" || // Added in 4.0
470 Name == "cvtsi2ss" || // Added in 7.0
471 Name == "cvtsi642ss" || // Added in 7.0
472 Name == "div.ss" || // Added in 4.0
473 Name == "mul.ss" || // Added in 4.0
474 Name.starts_with("sqrt.p") || // Added in 7.0
475 Name == "sqrt.ss" || // Added in 7.0
476 Name.starts_with("storeu.") || // Added in 3.9
477 Name == "sub.ss"); // Added in 4.0
478
479 if (Name.consume_front("sse2."))
480 return (Name == "add.sd" || // Added in 4.0
481 Name == "cvtdq2pd" || // Added in 3.9
482 Name == "cvtdq2ps" || // Added in 7.0
483 Name == "cvtps2pd" || // Added in 3.9
484 Name == "cvtsi2sd" || // Added in 7.0
485 Name == "cvtsi642sd" || // Added in 7.0
486 Name == "cvtss2sd" || // Added in 7.0
487 Name == "div.sd" || // Added in 4.0
488 Name == "mul.sd" || // Added in 4.0
489 Name.starts_with("padds.") || // Added in 8.0
490 Name.starts_with("paddus.") || // Added in 8.0
491 Name.starts_with("pcmpeq.") || // Added in 3.1
492 Name.starts_with("pcmpgt.") || // Added in 3.1
493 Name == "pmaxs.w" || // Added in 3.9
494 Name == "pmaxu.b" || // Added in 3.9
495 Name == "pmins.w" || // Added in 3.9
496 Name == "pminu.b" || // Added in 3.9
497 Name == "pmulh.w" || // Added in 24.0
498 Name == "pmulhu.w" || // Added in 24.0
499 Name == "pmulu.dq" || // Added in 7.0
500 Name.starts_with("pshuf") || // Added in 3.9
501 Name.starts_with("psll.dq") || // Added in 3.7
502 Name.starts_with("psrl.dq") || // Added in 3.7
503 Name.starts_with("psubs.") || // Added in 8.0
504 Name.starts_with("psubus.") || // Added in 8.0
505 Name.starts_with("sqrt.p") || // Added in 7.0
506 Name == "sqrt.sd" || // Added in 7.0
507 Name == "storel.dq" || // Added in 3.9
508 Name.starts_with("storeu.") || // Added in 3.9
509 Name == "sub.sd"); // Added in 4.0
510
511 if (Name.consume_front("sse41."))
512 return (Name.starts_with("blendp") || // Added in 3.7
513 Name == "movntdqa" || // Added in 5.0
514 Name == "pblendw" || // Added in 3.7
515 Name == "pmaxsb" || // Added in 3.9
516 Name == "pmaxsd" || // Added in 3.9
517 Name == "pmaxud" || // Added in 3.9
518 Name == "pmaxuw" || // Added in 3.9
519 Name == "pminsb" || // Added in 3.9
520 Name == "pminsd" || // Added in 3.9
521 Name == "pminud" || // Added in 3.9
522 Name == "pminuw" || // Added in 3.9
523 Name.starts_with("pmovsx") || // Added in 3.8
524 Name.starts_with("pmovzx") || // Added in 3.9
525 Name == "pmuldq"); // Added in 7.0
526
527 if (Name.consume_front("sse42."))
528 return Name == "crc32.64.8"; // Added in 3.4
529
530 if (Name.consume_front("sse4a."))
531 return Name.starts_with("movnt."); // Added in 3.9
532
533 if (Name.consume_front("ssse3."))
534 return (Name == "pabs.b.128" || // Added in 6.0
535 Name == "pabs.d.128" || // Added in 6.0
536 Name == "pabs.w.128"); // Added in 6.0
537
538 if (Name.consume_front("xop."))
539 return (Name == "vpcmov" || // Added in 3.8
540 Name == "vpcmov.256" || // Added in 5.0
541 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
542 Name.starts_with("vprot")); // Added in 8.0
543
544 if (Name.consume_front("bmi."))
545 return (Name.starts_with("pdep.") || // Added in 23.0
546 Name.starts_with("pext.")); // Added in 23.0
547
548 return (Name == "addcarry.u32" || // Added in 8.0
549 Name == "addcarry.u64" || // Added in 8.0
550 Name == "addcarryx.u32" || // Added in 8.0
551 Name == "addcarryx.u64" || // Added in 8.0
552 Name == "subborrow.u32" || // Added in 8.0
553 Name == "subborrow.u64" || // Added in 8.0
554 Name.starts_with("vcvtph2ps.")); // Added in 11.0
555}
556
558 Function *&NewFn) {
559 // Only handle intrinsics that start with "x86.".
560 if (!Name.consume_front("x86."))
561 return false;
562
563 if (shouldUpgradeX86Intrinsic(F, Name)) {
564 NewFn = nullptr;
565 return true;
566 }
567
568 if (Name == "rdtscp") { // Added in 8.0
569 // If this intrinsic has 0 operands, it's the new version.
570 if (F->getFunctionType()->getNumParams() == 0)
571 return false;
572
573 rename(F);
574 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
575 Intrinsic::x86_rdtscp);
576 return true;
577 }
578
579 Intrinsic::ID ID;
580
581 // SSE4.1 ptest functions may have an old signature.
582 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
584 .Case("c", Intrinsic::x86_sse41_ptestc)
585 .Case("z", Intrinsic::x86_sse41_ptestz)
586 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
588 if (ID != Intrinsic::not_intrinsic)
589 return upgradePTESTIntrinsic(F, ID, NewFn);
590
591 return false;
592 }
593
594 // Several blend and other instructions with masks used the wrong number of
595 // bits.
596
597 // Added in 3.6
599 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
600 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
601 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
602 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
603 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
604 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
606 if (ID != Intrinsic::not_intrinsic)
607 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
608
609 if (Name.consume_front("avx512.")) {
610 if (Name.consume_front("mask.cmp.")) {
611 // Added in 7.0
613 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
614 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
615 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
616 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
617 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
618 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
620 if (ID != Intrinsic::not_intrinsic)
621 return upgradeX86MaskedFPCompare(F, ID, NewFn);
622 } else if (Name.starts_with("vpdpbusd.") ||
623 Name.starts_with("vpdpbusds.")) {
624 // Added in 21.1
626 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
627 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
628 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
629 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
630 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
631 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
633 if (ID != Intrinsic::not_intrinsic)
634 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
635 } else if (Name.starts_with("vpdpwssd.") ||
636 Name.starts_with("vpdpwssds.")) {
637 // Added in 21.1
639 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
640 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
641 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
642 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
643 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
644 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
646 if (ID != Intrinsic::not_intrinsic)
647 return upgradeX86MultiplyAddWords(F, ID, NewFn);
648 }
649 return false; // No other 'x86.avx512.*'.
650 }
651
652 if (Name.consume_front("avx2.")) {
653 if (Name.consume_front("vpdpb")) {
654 // Added in 21.1
656 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
657 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
658 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
659 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
660 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
661 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
662 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
663 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
664 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
665 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
666 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
667 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
669 if (ID != Intrinsic::not_intrinsic)
670 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
671 } else if (Name.consume_front("vpdpw")) {
672 // Added in 21.1
674 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
675 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
676 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
677 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
678 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
679 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
680 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
681 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
682 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
683 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
684 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
685 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
687 if (ID != Intrinsic::not_intrinsic)
688 return upgradeX86MultiplyAddWords(F, ID, NewFn);
689 }
690 return false; // No other 'x86.avx2.*'
691 }
692
693 if (Name.consume_front("avx10.")) {
694 if (Name.consume_front("vpdpb")) {
695 // Added in 21.1
697 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
698 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
699 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
700 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
701 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
702 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
704 if (ID != Intrinsic::not_intrinsic)
705 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
706 } else if (Name.consume_front("vpdpw")) {
708 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
709 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
710 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
711 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
712 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
713 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
715 if (ID != Intrinsic::not_intrinsic)
716 return upgradeX86MultiplyAddWords(F, ID, NewFn);
717 }
718 return false; // No other 'x86.avx10.*'
719 }
720
721 if (Name.consume_front("avx512bf16.")) {
722 // Added in 9.0
724 .Case("cvtne2ps2bf16.128",
725 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
726 .Case("cvtne2ps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
728 .Case("cvtne2ps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
730 .Case("mask.cvtneps2bf16.128",
731 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
732 .Case("cvtneps2bf16.256",
733 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
734 .Case("cvtneps2bf16.512",
735 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
737 if (ID != Intrinsic::not_intrinsic)
738 return upgradeX86BF16Intrinsic(F, ID, NewFn);
739
740 // Added in 9.0
742 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
743 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
744 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
746 if (ID != Intrinsic::not_intrinsic)
747 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
748 return false; // No other 'x86.avx512bf16.*'.
749 }
750
751 if (Name.consume_front("xop.")) {
753 if (Name.starts_with("vpermil2")) { // Added in 3.9
754 // Upgrade any XOP PERMIL2 index operand still using a float/double
755 // vector.
756 auto Idx = F->getFunctionType()->getParamType(2);
757 if (Idx->isFPOrFPVectorTy()) {
758 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
759 unsigned EltSize = Idx->getScalarSizeInBits();
760 if (EltSize == 64 && IdxSize == 128)
761 ID = Intrinsic::x86_xop_vpermil2pd;
762 else if (EltSize == 32 && IdxSize == 128)
763 ID = Intrinsic::x86_xop_vpermil2ps;
764 else if (EltSize == 64 && IdxSize == 256)
765 ID = Intrinsic::x86_xop_vpermil2pd_256;
766 else
767 ID = Intrinsic::x86_xop_vpermil2ps_256;
768 }
769 } else if (F->arg_size() == 2)
770 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
772 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
773 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
775
776 if (ID != Intrinsic::not_intrinsic) {
777 rename(F);
778 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
779 return true;
780 }
781 return false; // No other 'x86.xop.*'
782 }
783
784 if (Name == "seh.recoverfp") {
785 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
786 Intrinsic::eh_recoverfp);
787 return true;
788 }
789
790 return false;
791}
792
793// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
794// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
796 StringRef Name,
797 Function *&NewFn) {
798 if (Name.starts_with("rbit")) {
799 // '(arm|aarch64).rbit'.
801 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
802 return true;
803 }
804
805 if (Name == "thread.pointer") {
806 // '(arm|aarch64).thread.pointer'.
808 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
809 return true;
810 }
811
812 bool Neon = Name.consume_front("neon.");
813 if (Neon) {
814 // '(arm|aarch64).neon.*'.
815 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
816 // v16i8 respectively.
817 if (Name.consume_front("bfdot.")) {
818 // (arm|aarch64).neon.bfdot.*'.
819 Intrinsic::ID ID =
821 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
822 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
823 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
825 if (ID != Intrinsic::not_intrinsic) {
826 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
827 assert((OperandWidth == 64 || OperandWidth == 128) &&
828 "Unexpected operand width");
829 LLVMContext &Ctx = F->getParent()->getContext();
830 std::array<Type *, 2> Tys{
831 {F->getReturnType(),
832 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
833 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
834 return true;
835 }
836 return false; // No other '(arm|aarch64).neon.bfdot.*'.
837 }
838
839 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
840 // anymore and accept v8bf16 instead of v16i8.
841 if (Name.consume_front("bfm")) {
842 // (arm|aarch64).neon.bfm*'.
843 if (Name.consume_back(".v4f32.v16i8")) {
844 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
845 Intrinsic::ID ID =
847 .Case("mla",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
850 .Case("lalb",
851 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
852 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
853 .Case("lalt",
854 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
855 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
857 if (ID != Intrinsic::not_intrinsic) {
858 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
859 return true;
860 }
861 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
862 }
863 return false; // No other '(arm|aarch64).neon.bfm*.
864 }
865 // Continue on to Aarch64 Neon or Arm Neon.
866 }
867 // Continue on to Arm or Aarch64.
868
869 if (IsArm) {
870 // 'arm.*'.
871 if (Neon) {
872 // 'arm.neon.*'.
874 .StartsWith("vclz.", Intrinsic::ctlz)
875 .StartsWith("vcnt.", Intrinsic::ctpop)
876 .StartsWith("vqadds.", Intrinsic::sadd_sat)
877 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
878 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
879 .StartsWith("vqsubu.", Intrinsic::usub_sat)
880 .StartsWith("vrinta.", Intrinsic::round)
881 .StartsWith("vrintn.", Intrinsic::roundeven)
882 .StartsWith("vrintm.", Intrinsic::floor)
883 .StartsWith("vrintp.", Intrinsic::ceil)
884 .StartsWith("vrintx.", Intrinsic::rint)
885 .StartsWith("vrintz.", Intrinsic::trunc)
887 if (ID != Intrinsic::not_intrinsic) {
888 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
889 F->arg_begin()->getType());
890 return true;
891 }
892
893 if (Name.consume_front("vst")) {
894 // 'arm.neon.vst*'.
895 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
897 if (vstRegex.match(Name, &Groups)) {
898 static const Intrinsic::ID StoreInts[] = {
899 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
900 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
901
902 static const Intrinsic::ID StoreLaneInts[] = {
903 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
904 Intrinsic::arm_neon_vst4lane};
905
906 auto fArgs = F->getFunctionType()->params();
907 Type *Tys[] = {fArgs[0], fArgs[1]};
908 if (Groups[1].size() == 1)
910 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
911 else
913 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
914 return true;
915 }
916 return false; // No other 'arm.neon.vst*'.
917 }
918
919 return false; // No other 'arm.neon.*'.
920 }
921
922 if (Name.consume_front("mve.")) {
923 // 'arm.mve.*'.
924 if (Name == "vctp64") {
925 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
926 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
927 // the function and deal with it below in UpgradeIntrinsicCall.
928 rename(F);
929 return true;
930 }
931 return false; // Not 'arm.mve.vctp64'.
932 }
933
934 if (Name.starts_with("vrintn.v")) {
936 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
937 return true;
938 }
939
940 // These too are changed to accept a v2i1 instead of the old v4i1.
941 if (Name.consume_back(".v4i1")) {
942 // 'arm.mve.*.v4i1'.
943 if (Name.consume_back(".predicated.v2i64.v4i32"))
944 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
945 return Name == "mull.int" || Name == "vqdmull";
946
947 if (Name.consume_back(".v2i64")) {
948 // 'arm.mve.*.v2i64.v4i1'
949 bool IsGather = Name.consume_front("vldr.gather.");
950 if (IsGather || Name.consume_front("vstr.scatter.")) {
951 if (Name.consume_front("base.")) {
952 // Optional 'wb.' prefix.
953 Name.consume_front("wb.");
954 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
955 // predicated.v2i64.v2i64.v4i1'.
956 return Name == "predicated.v2i64";
957 }
958
959 if (Name.consume_front("offset.predicated."))
960 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
961 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
962
963 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
964 return false;
965 }
966
967 return false; // No other 'arm.mve.*.v2i64.v4i1'.
968 }
969 return false; // No other 'arm.mve.*.v4i1'.
970 }
971 return false; // No other 'arm.mve.*'.
972 }
973
974 if (Name.consume_front("cde.vcx")) {
975 // 'arm.cde.vcx*'.
976 if (Name.consume_back(".predicated.v2i64.v4i1"))
977 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
978 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
979 Name == "3q" || Name == "3qa";
980
981 return false; // No other 'arm.cde.vcx*'.
982 }
983 } else {
984 // 'aarch64.*'.
985 if (Neon) {
986 // 'aarch64.neon.*'.
988 .StartsWith("frintn", Intrinsic::roundeven)
989 .StartsWith("rbit", Intrinsic::bitreverse)
991 if (ID != Intrinsic::not_intrinsic) {
992 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
993 F->arg_begin()->getType());
994 return true;
995 }
996
997 Intrinsic::ID MinMaxID =
998 StringSwitch<Intrinsic::ID>(Name.split('.').first)
999 .Case("smax", Intrinsic::smax)
1000 .Case("smin", Intrinsic::smin)
1001 .Case("umax", Intrinsic::umax)
1002 .Case("umin", Intrinsic::umin)
1004 if (MinMaxID != Intrinsic::not_intrinsic) {
1005 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1006 return false; // Invalid IR.
1007 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), MinMaxID,
1008 F->getReturnType());
1009 return true;
1010 }
1011
1012 if (Name.starts_with("addp")) {
1013 // 'aarch64.neon.addp*'.
1014 if (F->arg_size() != 2)
1015 return false; // Invalid IR.
1016 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
1017 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1019 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1020 return true;
1021 }
1022 }
1023
1024 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1025 if (Name.starts_with("bfcvt")) {
1026 NewFn = nullptr;
1027 return true;
1028 }
1029
1030 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1031 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1032 NewFn = nullptr;
1033 return true;
1034 }
1035
1036 return false; // No other 'aarch64.neon.*'.
1037 }
1038 if (Name.consume_front("sve.")) {
1039 // 'aarch64.sve.*'.
1040 if (Name.consume_front("bf")) {
1041 if (Name == "mmla") {
1042 Type *Tys[] = {F->getReturnType(),
1043 std::next(F->arg_begin())->getType()};
1045 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1046 return true;
1047 }
1048 if (Name.consume_back(".lane")) {
1049 // 'aarch64.sve.bf*.lane'.
1050 Intrinsic::ID ID =
1052 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1053 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1054 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1056 if (ID != Intrinsic::not_intrinsic) {
1057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1058 return true;
1059 }
1060 return false; // No other 'aarch64.sve.bf*.lane'.
1061 }
1062 return false; // No other 'aarch64.sve.bf*'.
1063 }
1064
1065 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1066 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1067 NewFn = nullptr;
1068 return true;
1069 }
1070
1071 if (Name.consume_front("convert.from.svbool")) {
1072 // 'aarch64.sve.convert.from.svbool'
1073 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1074 if (!TTy || TTy->getName() != "aarch64.svcount")
1075 return false;
1076
1077 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1078 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1079 return true;
1080 }
1081
1082 if (Name.consume_front("convert.to.svbool")) {
1083 // 'aarch64.sve.convert.to.svbool'
1084 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1085 if (!TTy || TTy->getName() != "aarch64.svcount")
1086 return false;
1087
1088 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1089 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1090 return true;
1091 }
1092
1093 if (Name.consume_front("addqv")) {
1094 // 'aarch64.sve.addqv'.
1095 if (!F->getReturnType()->isFPOrFPVectorTy())
1096 return false;
1097
1098 auto Args = F->getFunctionType()->params();
1099 Type *Tys[] = {F->getReturnType(), Args[1]};
1101 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1102 return true;
1103 }
1104
1105 if (Name.consume_front("ld")) {
1106 // 'aarch64.sve.ld*'.
1107 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1108 if (LdRegex.match(Name)) {
1109 Type *ScalarTy =
1110 cast<VectorType>(F->getReturnType())->getElementType();
1111 ElementCount EC =
1112 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1113 assert(F->arg_size() == 2 &&
1114 "Expected 2 arguments for ld* intrinsic.");
1115 Type *PtrTy = F->getArg(1)->getType();
1116 Type *Ty = VectorType::get(ScalarTy, EC);
1117 static const Intrinsic::ID LoadIDs[] = {
1118 Intrinsic::aarch64_sve_ld2_sret,
1119 Intrinsic::aarch64_sve_ld3_sret,
1120 Intrinsic::aarch64_sve_ld4_sret,
1121 };
1123 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1124 return true;
1125 }
1126 return false; // No other 'aarch64.sve.ld*'.
1127 }
1128
1129 if (Name.consume_front("tuple.")) {
1130 // 'aarch64.sve.tuple.*'.
1131 if (Name.starts_with("get")) {
1132 // 'aarch64.sve.tuple.get*'.
1133 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1135 F->getParent(), Intrinsic::vector_extract, Tys);
1136 return true;
1137 }
1138
1139 if (Name.starts_with("set")) {
1140 // 'aarch64.sve.tuple.set*'.
1141 auto Args = F->getFunctionType()->params();
1142 Type *Tys[] = {Args[0], Args[2], Args[1]};
1144 F->getParent(), Intrinsic::vector_insert, Tys);
1145 return true;
1146 }
1147
1148 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1149 if (CreateTupleRegex.match(Name)) {
1150 // 'aarch64.sve.tuple.create*'.
1151 auto Args = F->getFunctionType()->params();
1152 Type *Tys[] = {F->getReturnType(), Args[1]};
1154 F->getParent(), Intrinsic::vector_insert, Tys);
1155 return true;
1156 }
1157 return false; // No other 'aarch64.sve.tuple.*'.
1158 }
1159
1160 if (Name.starts_with("rev.nxv")) {
1161 // 'aarch64.sve.rev.<Ty>'
1163 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1164 return true;
1165 }
1166
1167 return false; // No other 'aarch64.sve.*'.
1168 }
1169 if (Name.consume_front("sme.")) {
1170 // 'aarch64.sme.*'.
1171 if (Name.consume_front("ftmopa.")) {
1172 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1173 // intrinsics to model their FPMR dependency.
1174 Intrinsic::ID ID =
1176 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1177 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1179 if (ID != Intrinsic::not_intrinsic) {
1180 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1181 return true;
1182 }
1183 return false; // No other 'aarch64.sme.ftmopa.*'.
1184 }
1185
1186 return false; // No other 'aarch64.sme.*'.
1187 }
1188 }
1189 return false; // No other 'arm.*', 'aarch64.*'.
1190}
1191
1192// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1193// declarations requiring an auto-upgrade. The same set applies to the
1194// cluster (g2s) and CTA (g2s_cta) variants.
1195#define NVVM_TMA_G2S_MODES(M) \
1196 M(tile_1d, "tile.1d") \
1197 M(tile_2d, "tile.2d") \
1198 M(tile_3d, "tile.3d") \
1199 M(tile_4d, "tile.4d") \
1200 M(tile_5d, "tile.5d") \
1201 M(tile_gather4_2d, "tile.gather4.2d") \
1202 M(im2col_3d, "im2col.3d") \
1203 M(im2col_4d, "im2col.4d") \
1204 M(im2col_5d, "im2col.5d") \
1205 M(im2col_w_3d, "im2col.w.3d") \
1206 M(im2col_w_4d, "im2col.w.4d") \
1207 M(im2col_w_5d, "im2col.w.5d") \
1208 M(im2col_w_128_3d, "im2col.w.128.3d") \
1209 M(im2col_w_128_4d, "im2col.w.128.4d") \
1210 M(im2col_w_128_5d, "im2col.w.128.5d")
1211
1212// Two legacy tails are:
1213//
1214// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1215// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1216//
1217// The current tail appends a trailing i32 %flag_valid_pattern, so both
1218// legacy tails are recognized by an i1 at parameter N-2.
1219static Intrinsic::ID
1221 SmallVectorImpl<Type *> &OvlTys) {
1222 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1224
1225#define G2S_ID(ID_SUFFIX, NAME) \
1226 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1227 // clang-format off
1231#undef G2S_ID
1232 // clang-format on
1233 if (ID == Intrinsic::not_intrinsic)
1234 return ID;
1235
1236 size_t NumParams = F->getFunctionType()->getNumParams();
1237
1238 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1239 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1240 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1242
1243 // The multicast mask is the parameter immediately before the i64
1244 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1245 ArrayRef<Type *> Params = F->getFunctionType()->params();
1246 size_t MaskIdx =
1247 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1248 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1249 "expected the i64 cache-hint after the multicast mask");
1250 Type *MaskTy = Params[MaskIdx];
1251 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1252 OvlTys.push_back(MaskTy);
1253
1254 return ID;
1255}
1256
1257// The legacy tail is:
1258//
1259// arg1, arg2, .. i64 %ch, i1 %flag_ch
1260//
1261// The current tail appends a trailing i32 %flag_valid_pattern, so the
1262// legacy tail is recognized by an i1 at parameter N-1.
1264 StringRef Name) {
1265 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1267
1268#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1269 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1270 // clang-format off
1274#undef G2S_CTA_ID
1275 // clang-format on
1276 if (ID == Intrinsic::not_intrinsic)
1277 return ID;
1278
1279 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1280 // with i32 %flag_valid_pattern, for which N-1 is i32.
1281 if (!F->getFunctionType()
1282 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1283 ->isIntegerTy(1))
1285
1286 return ID;
1287}
1288// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1289//
1290// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1291//
1292// The current intrinsic is overloaded on the multicast-mask type and takes a
1293// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1294// parameter N-1.
1295static Intrinsic::ID
1297 SmallVectorImpl<Type *> &OvlTys) {
1298 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1300
1301 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1302 // i32 %flag_valid_pattern, for which N-1 is i32.
1303 size_t NumParams = F->getFunctionType()->getNumParams();
1304 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1306
1307 // The multicast mask is parameter 4; legacy IR only uses i16.
1308 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1309 if (!MaskTy->isIntegerTy(16))
1311 OvlTys.push_back(MaskTy);
1312
1313 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1314}
1315
1316// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1317//
1318// ..., i64 %ch, i1 %flag_ch
1319//
1320// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1321// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1322// recognized by an i1 at parameter N-1, whereas the current tail ends
1323// with an i32.
1325 StringRef Name) {
1326 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1328
1329 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1330 // i32 %flag_valid_pattern, for which N-1 is i32.
1331 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1333
1334 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1335}
1336
1337// The legacy TMA reduction intrinsics encode the reduction operator in their
1338// name, while the current ones take it as an immediate argument. Map the
1339// operator part of a legacy name to the corresponding immediate value.
1340static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1342 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1343 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1344 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1345 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1346 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1347 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1348 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1349 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1350 .Default(std::nullopt);
1351}
1352
1354 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1356
1357 auto [RedOpName, ShapeName] = Name.split('.');
1358 if (!getNVPTXTMAReductionOp(RedOpName))
1360
1361 return StringSwitch<Intrinsic::ID>(ShapeName)
1362 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1363 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1364 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1365 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1366 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1367 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1368 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1369 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1371}
1372
1373static Intrinsic::ID
1375 if (!Name.consume_front("mbarrier.") ||
1376 !Name.consume_back(".space.cluster") || F->arg_size() != 2)
1378
1379 Intrinsic::ID IID = F->getIntrinsicID();
1380 if (IID == Intrinsic::not_intrinsic ||
1381 Intrinsic::getType(F->getContext(), IID) == F->getFunctionType())
1383
1384 return IID;
1385}
1386
1388 StringRef Name) {
1389 if (Name.consume_front("mapa.shared.cluster"))
1390 if (F->getReturnType()->getPointerAddressSpace() ==
1392 return Intrinsic::nvvm_mapa_shared_cluster;
1393
1394 if (Name.consume_front("cp.async.bulk.")) {
1395 Intrinsic::ID ID =
1397 .Case("shared.cta.to.cluster",
1398 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1400
1401 if (ID != Intrinsic::not_intrinsic)
1402 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1404 return ID;
1405 }
1406
1408}
1409
1410static Intrinsic::ID
1412 if (!Name.consume_front("tcgen05.commit."))
1414
1415 if (Name.consume_front("shared."))
1416 return StringSwitch<Intrinsic::ID>(Name)
1417 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1418 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1420
1421 if (Name.consume_front("mc.shared.")) {
1422 // Only upgrade older i16 mc variants.
1423 if (!F->getArg(1)->getType()->isIntegerTy(16))
1425
1426 return StringSwitch<Intrinsic::ID>(Name)
1427 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1428 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1430 }
1431
1433}
1434
1435static Intrinsic::ID
1437 if (F->arg_size() != 2)
1439
1440 return StringSwitch<Intrinsic::ID>(Name)
1441 .Case("tcgen05.alloc.shared.cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1442 .Case("tcgen05.alloc.shared.cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1444}
1445
1447 if (Name.consume_front("fmax."))
1448 return StringSwitch<Intrinsic::ID>(Name)
1449 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1450 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1451 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1452 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1453 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1454 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1455 .Case("ftz.nan.xorsign.abs.bf16",
1456 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1457 .Case("ftz.nan.xorsign.abs.bf16x2",
1458 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1459 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1460 .Case("ftz.xorsign.abs.bf16x2",
1461 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1462 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1463 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1464 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1465 .Case("nan.xorsign.abs.bf16x2",
1466 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1467 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1468 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1470
1471 if (Name.consume_front("fmin."))
1472 return StringSwitch<Intrinsic::ID>(Name)
1473 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1474 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1475 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1476 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1477 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1478 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1479 .Case("ftz.nan.xorsign.abs.bf16",
1480 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1481 .Case("ftz.nan.xorsign.abs.bf16x2",
1482 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1483 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1484 .Case("ftz.xorsign.abs.bf16x2",
1485 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1486 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1487 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1488 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1489 .Case("nan.xorsign.abs.bf16x2",
1490 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1491 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1492 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1494
1495 if (Name.consume_front("neg."))
1496 return StringSwitch<Intrinsic::ID>(Name)
1497 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1498 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1500
1502}
1503
1505 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1506 FunctionType *OldFnTy = F->getFunctionType();
1507 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1508 return OldTy->getScalarType()->isIntegerTy() &&
1509 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1510 };
1511
1512 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1513 return false;
1514
1515 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1516 return false;
1517
1518 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1519 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1520 return false;
1521
1522 return true;
1523}
1524
1525// Overloaded fadd/fmul/fma intrinsic IDs, indexed by
1526// [`.ftz`][none|`.sat`|`.relu`].
1527static constexpr Intrinsic::ID NVVMFAddIIDs[2][3] = {
1528 {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat, Intrinsic::not_intrinsic},
1529 {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
1531static constexpr Intrinsic::ID NVVMFMulIIDs[2][3] = {
1532 {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat, Intrinsic::not_intrinsic},
1533 {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
1535static constexpr Intrinsic::ID NVVMFMAIIDs[2][3] = {
1536 {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
1537 {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
1538 Intrinsic::nvvm_fma_ftz_relu}};
1539
1540// Returns true if the legacy nvvm.{add,mul,fma}.<rnd>{.ftz}{.sat|.relu}.<Type>
1541// intrinsic with these modifiers existed.
1542static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID (&IIDs)[2][3],
1543 StringRef Type, bool IsRN,
1544 bool IsFTZ, bool IsSat,
1545 bool IsRelu) {
1546 const bool IsFMA = &IIDs == &NVVMFMAIIDs;
1547 const bool IsFMul = &IIDs == &NVVMFMulIIDs;
1548 return StringSwitch<bool>(Type)
1549 .Case("f", !IsRelu && !(IsSat && IsFMul))
1550 .Case("d", !IsFTZ && !IsSat && !IsRelu)
1551 .Case("f16", IsRN && (IsFMA || IsSat))
1552 .Case("v2f16", IsRN && !IsFMA && IsSat)
1553 .Case("f16x2", IsRN && IsFMA)
1554 .Cases({"bf16", "bf16x2"}, IsRN && IsFMA && !IsFTZ && !IsSat)
1555 .Default(false);
1556}
1557
1558static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1560 auto [Modifiers, Type] = Name.rsplit('.');
1561
1562 std::optional<llvm::RoundingMode> RoundingMode =
1563 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1568 .Default(std::nullopt);
1569 if (!RoundingMode)
1570 return std::nullopt;
1571
1572 StringRef Rest = Modifiers.drop_front(2);
1573 const bool IsFTZ = Rest.consume_front(".ftz");
1574 const bool IsSat = Rest.consume_front(".sat");
1575 const bool IsRelu = !IsSat && Rest.consume_front(".relu");
1576 if (!Rest.empty())
1577 return std::nullopt;
1578
1579 const Intrinsic::ID IID = IIDs[IsFTZ][IsSat ? 1 : IsRelu ? 2 : 0];
1580 if (IID == Intrinsic::not_intrinsic)
1581 return std::nullopt;
1582
1583 // Reject combinations that never existed as legacy intrinsics.
1585 if (!isValidLegacyNVVMFPArithIntrinsic(IIDs, Type, IsRN, IsFTZ, IsSat,
1586 IsRelu))
1587 return std::nullopt;
1588
1589 return std::make_pair(IID, *RoundingMode);
1590}
1591
1593 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1595
1596 return Intrinsic::nvvm_mbarrier_init;
1597}
1598
1600 return Name.consume_front("local") || Name.consume_front("shared") ||
1601 Name.consume_front("global") || Name.consume_front("constant") ||
1602 Name.consume_front("param");
1603}
1604
1606 if (!Name.consume_front("vp."))
1607 return 0;
1608 return StringSwitch<unsigned>(Name)
1609 .StartsWith("select", Instruction::Select)
1610 .StartsWith("add", Instruction::Add)
1611 .StartsWith("sub", Instruction::Sub)
1612 .StartsWith("mul", Instruction::Mul)
1613 .StartsWith("ashr", Instruction::AShr)
1614 .StartsWith("lshr", Instruction::LShr)
1615 .StartsWith("shl", Instruction::Shl)
1616 .StartsWith("or", Instruction::Or)
1617 .StartsWith("and", Instruction::And)
1618 .StartsWith("xor", Instruction::Xor)
1619 .StartsWith("fadd", Instruction::FAdd)
1620 .StartsWith("fsub", Instruction::FSub)
1621 .StartsWith("fmuladd", 0)
1622 .StartsWith("fmul", Instruction::FMul)
1623 .StartsWith("fdiv", Instruction::FDiv)
1624 .StartsWith("frem", Instruction::FRem)
1625 .StartsWith("fneg", Instruction::FNeg)
1626 .StartsWith("trunc", Instruction::Trunc)
1627 .StartsWith("zext", Instruction::ZExt)
1628 .StartsWith("sext", Instruction::SExt)
1629 .StartsWith("fptrunc", Instruction::FPTrunc)
1630 .StartsWith("fpext", Instruction::FPExt)
1631 .StartsWith("fptoui", Instruction::FPToUI)
1632 .StartsWith("fptosi", Instruction::FPToSI)
1633 .StartsWith("uitofp", Instruction::UIToFP)
1634 .StartsWith("sitofp", Instruction::SIToFP)
1635 .StartsWith("ptrtoint", Instruction::PtrToInt)
1636 .StartsWith("inttoptr", Instruction::IntToPtr)
1637 .StartsWith("icmp", Instruction::ICmp)
1638 .StartsWith("fcmp", Instruction::FCmp)
1639 .Default(0);
1640}
1641
1643 if (!Name.consume_front("vp."))
1644 return 0;
1645 return StringSwitch<Intrinsic::ID>(Name)
1646 .StartsWith("abs", Intrinsic::abs)
1647 .StartsWith("smax", Intrinsic::smax)
1648 .StartsWith("smin", Intrinsic::smin)
1649 .StartsWith("umax", Intrinsic::umax)
1650 .StartsWith("umin", Intrinsic::umin)
1651 .StartsWith("copysign", Intrinsic::copysign)
1652 .StartsWith("minnum", Intrinsic::minnum)
1653 .StartsWith("maxnum", Intrinsic::maxnum)
1654 .StartsWith("minimum", Intrinsic::minimum)
1655 .StartsWith("maximum", Intrinsic::maximum)
1656 .StartsWith("fabs", Intrinsic::fabs)
1657 .StartsWith("sqrt", Intrinsic::sqrt)
1658 .StartsWith("fma", Intrinsic::fma)
1659 .StartsWith("fmuladd", Intrinsic::fmuladd)
1660 .StartsWith("ceil", Intrinsic::ceil)
1661 .StartsWith("floor", Intrinsic::floor)
1662 .StartsWith("rint", Intrinsic::rint)
1663 .StartsWith("nearbyint", Intrinsic::nearbyint)
1664 .StartsWith("roundeven", Intrinsic::roundeven)
1665 .StartsWith("roundtozero", Intrinsic::trunc)
1666 .StartsWith("round", Intrinsic::round)
1667 .StartsWith("lrint", Intrinsic::lrint)
1668 .StartsWith("llrint", Intrinsic::llrint)
1669 .StartsWith("bitreverse", Intrinsic::bitreverse)
1670 .StartsWith("bswap", Intrinsic::bswap)
1671 .StartsWith("ctpop", Intrinsic::ctpop)
1672 .StartsWith("ctlz", Intrinsic::ctlz)
1673 .StartsWith("cttz.elts", 0)
1674 .StartsWith("cttz", Intrinsic::cttz)
1675 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1676 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1677 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1678 .StartsWith("usub.sat", Intrinsic::usub_sat)
1679 .StartsWith("fshl", Intrinsic::fshl)
1680 .StartsWith("fshr", Intrinsic::fshr)
1681 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1682 .Default(0);
1683}
1684
1688
1690 const FunctionType *FuncTy) {
1691 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1692 if (Name.starts_with("to.fp16")) {
1693 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1694 HalfTy) &&
1695 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1696 FuncTy->getReturnType());
1697 }
1698
1699 if (Name.starts_with("from.fp16")) {
1700 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1701 HalfTy) &&
1702 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1703 FuncTy->getReturnType());
1704 }
1705
1706 return false;
1707}
1708
1709static unsigned
1711 SmallVectorImpl<Type *> &OverloadTys) {
1712 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1713 if (Defaults.empty())
1714 return 0;
1715
1716 unsigned FullArgCount = FirstDefault + Defaults.size();
1717
1718 // Only trailing default arguments can be missing.
1719 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1720 return 0;
1721
1722 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1723 if (!Intrinsic::isSignatureValid(IID, F->getFunctionType(), OverloadTys,
1724 NumMissingTrailingParams))
1725 return 0;
1726
1727 return FullArgCount;
1728}
1729
1731 Intrinsic::ID IID = F->getIntrinsicID();
1732 SmallVector<Type *, 4> OverloadTys;
1733
1734 unsigned FullArgCount =
1735 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1736 if (FullArgCount == 0)
1737 return false;
1738
1739 rename(F);
1740 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OverloadTys);
1741 assert(NewFn->arg_size() == FullArgCount &&
1742 "total number of default args does not match intrinsic signature");
1743 return true;
1744}
1745
1747 bool CanUpgradeDebugIntrinsicsToRecords) {
1748 assert(F && "Illegal to upgrade a non-existent Function.");
1749
1750 StringRef Name = F->getName();
1751
1752 // Quickly eliminate it, if it's not a candidate.
1753 if (!Name.consume_front("llvm.") || Name.empty())
1754 return false;
1755
1756 switch (Name[0]) {
1757 default: break;
1758 case 'a': {
1759 bool IsArm = Name.consume_front("arm.");
1760 if (IsArm || Name.consume_front("aarch64.")) {
1761 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1762 return true;
1763 break;
1764 }
1765
1766 if (Name.consume_front("amdgcn.")) {
1767 if (Name == "alignbit") {
1768 // Target specific intrinsic became redundant
1770 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1771 return true;
1772 }
1773
1774 if (Name.consume_front("atomic.")) {
1775 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1776 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1777 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1778 // and usub_sat so there's no new declaration.
1779 NewFn = nullptr;
1780 return true;
1781 }
1782 break; // No other 'amdgcn.atomic.*'
1783 }
1784
1785 if (Name.starts_with("addrspacecast.nonnull")) {
1786 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1787 // so there's no new declaration.
1788 NewFn = nullptr;
1789 return true;
1790 }
1791
1792 switch (F->getIntrinsicID()) {
1793 default:
1794 break;
1795 // Legacy asyncmark intrinsics without the stage operand.
1796 case Intrinsic::amdgcn_asyncmark:
1797 case Intrinsic::amdgcn_wait_asyncmark: {
1798 Intrinsic::ID ID = F->getIntrinsicID();
1799 if (F->arg_size() == (ID == Intrinsic::amdgcn_asyncmark ? 0u : 1u)) {
1800 rename(F);
1801 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1802 return true;
1803 }
1804 break;
1805 }
1806 // Legacy wmma iu intrinsics without the optional clamp operand.
1807 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1808 if (F->arg_size() == 7) {
1809 NewFn = nullptr;
1810 return true;
1811 }
1812 break;
1813 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1814 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1815 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1816 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1817 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1818 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1819 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1820 if (F->arg_size() == 8) {
1821 NewFn = nullptr;
1822 return true;
1823 }
1824 break;
1825 }
1826
1827 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1828 Name.consume_front("flat.atomic.")) {
1829 if (Name.starts_with("fadd") ||
1830 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1831 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1832 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1833 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1834 // declaration.
1835 NewFn = nullptr;
1836 return true;
1837 }
1838 }
1839
1840 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1841 NewFn = nullptr;
1842 return true;
1843 }
1844
1845 if (Name.starts_with("ldexp.")) {
1846 // Target specific intrinsic became redundant
1848 F->getParent(), Intrinsic::ldexp,
1849 {F->getReturnType(), F->getArg(1)->getType()});
1850 return true;
1851 }
1852 break; // No other 'amdgcn.*'
1853 }
1854
1855 break;
1856 }
1857 case 'c': {
1858 if (F->arg_size() == 1) {
1859 if (Name.consume_front("convert.")) {
1860 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1861 NewFn = nullptr;
1862 return true;
1863 }
1864 }
1865
1867 .StartsWith("ctlz.", Intrinsic::ctlz)
1868 .StartsWith("cttz.", Intrinsic::cttz)
1870 if (ID != Intrinsic::not_intrinsic) {
1871 rename(F);
1872 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1873 F->arg_begin()->getType());
1874 return true;
1875 }
1876 }
1877
1879 if (Name == "coro.end" &&
1880 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1881 CoroEndID = Intrinsic::coro_end;
1882 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1883 CoroEndID = Intrinsic::coro_end_async;
1884
1885 if (CoroEndID != Intrinsic::not_intrinsic) {
1886 rename(F);
1887 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1888 return true;
1889 }
1890
1891 break;
1892 }
1893 case 'd':
1894 if (Name.consume_front("dbg.")) {
1895 // Mark debug intrinsics for upgrade to new debug format.
1896 if (CanUpgradeDebugIntrinsicsToRecords) {
1897 if (Name == "addr" || Name == "value" || Name == "assign" ||
1898 Name == "declare" || Name == "label") {
1899 // There's no function to replace these with.
1900 NewFn = nullptr;
1901 // But we do want these to get upgraded.
1902 return true;
1903 }
1904 }
1905 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1906 // converted to DbgVariableRecords later.
1907 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1908 rename(F);
1909 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1910 Intrinsic::dbg_value);
1911 return true;
1912 }
1913 break; // No other 'dbg.*'.
1914 }
1915 break;
1916 case 'e':
1917 if (Name.consume_front("experimental.vector.")) {
1918 Intrinsic::ID ID =
1920 // Skip over extract.last.active, otherwise it will be 'upgraded'
1921 // to a regular vector extract which is a different operation.
1922 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1923 .StartsWith("extract.", Intrinsic::vector_extract)
1924 .StartsWith("insert.", Intrinsic::vector_insert)
1925 .StartsWith("reverse.", Intrinsic::vector_reverse)
1926 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1927 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1928 .StartsWith("partial.reduce.add",
1929 Intrinsic::vector_partial_reduce_add)
1931 if (ID != Intrinsic::not_intrinsic) {
1932 const auto *FT = F->getFunctionType();
1934 if (ID == Intrinsic::vector_extract ||
1935 ID == Intrinsic::vector_interleave2)
1936 // Extracting overloads the return type.
1937 Tys.push_back(FT->getReturnType());
1938 if (ID != Intrinsic::vector_interleave2)
1939 Tys.push_back(FT->getParamType(0));
1940 if (ID == Intrinsic::vector_insert ||
1941 ID == Intrinsic::vector_partial_reduce_add)
1942 // Inserting overloads the inserted type.
1943 Tys.push_back(FT->getParamType(1));
1944 rename(F);
1945 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1946 return true;
1947 }
1948
1949 if (Name.consume_front("reduce.")) {
1951 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1952 if (R.match(Name, &Groups))
1954 .Case("add", Intrinsic::vector_reduce_add)
1955 .Case("mul", Intrinsic::vector_reduce_mul)
1956 .Case("and", Intrinsic::vector_reduce_and)
1957 .Case("or", Intrinsic::vector_reduce_or)
1958 .Case("xor", Intrinsic::vector_reduce_xor)
1959 .Case("smax", Intrinsic::vector_reduce_smax)
1960 .Case("smin", Intrinsic::vector_reduce_smin)
1961 .Case("umax", Intrinsic::vector_reduce_umax)
1962 .Case("umin", Intrinsic::vector_reduce_umin)
1963 .Case("fmax", Intrinsic::vector_reduce_fmax)
1964 .Case("fmin", Intrinsic::vector_reduce_fmin)
1966
1967 bool V2 = false;
1968 if (ID == Intrinsic::not_intrinsic) {
1969 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1970 Groups.clear();
1971 V2 = true;
1972 if (R2.match(Name, &Groups))
1974 .Case("fadd", Intrinsic::vector_reduce_fadd)
1975 .Case("fmul", Intrinsic::vector_reduce_fmul)
1977 }
1978 if (ID != Intrinsic::not_intrinsic) {
1979 rename(F);
1980 auto Args = F->getFunctionType()->params();
1981 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1982 {Args[V2 ? 1 : 0]});
1983 return true;
1984 }
1985 break; // No other 'expermental.vector.reduce.*'.
1986 }
1987
1988 if (Name.consume_front("splice"))
1989 return true;
1990 break; // No other 'experimental.vector.*'.
1991 }
1992 if (Name.consume_front("experimental.stepvector.")) {
1993 Intrinsic::ID ID = Intrinsic::stepvector;
1994 rename(F);
1996 F->getParent(), ID, F->getFunctionType()->getReturnType());
1997 return true;
1998 }
1999 break; // No other 'e*'.
2000 case 'f':
2001 if (Name.starts_with("flt.rounds")) {
2002 rename(F);
2003 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2004 Intrinsic::get_rounding);
2005 return true;
2006 }
2007 break;
2008 case 'i':
2009 if (Name.starts_with("invariant.group.barrier")) {
2010 // Rename invariant.group.barrier to launder.invariant.group
2011 auto Args = F->getFunctionType()->params();
2012 Type* ObjectPtr[1] = {Args[0]};
2013 rename(F);
2015 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
2016 return true;
2017 }
2018 break;
2019 case 'l': {
2020 bool IsLifetimeStart = Name.consume_front("lifetime.start");
2021 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
2022 if (IsLifetimeStart || IsLifetimeEnd) {
2023 if (F->arg_size() == 2) {
2024 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
2025 : Intrinsic::lifetime_end;
2026 rename(F);
2027 // Old 2 argument form of these intrinsics have [Size, Ptr] as
2028 // arguments. Use the Ptr argument to create new declaration.
2029 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2030 F->getArg(1)->getType());
2031 return true;
2032 } else if (F->arg_size() == 1 && Name == ".i64") {
2033 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
2034 // Autoupgrade prior to
2035 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
2036 // intrinsic with no expected calls. To allow auto-upgrade process to
2037 // delete such invalid intrinsic declaration, set NewFn = nullptr
2038 // and return true here. If there are actual calls to this intrinsic
2039 // (which is not expected), they will be deleted in
2040 // UpgradeIntrinsicCall.
2041 NewFn = nullptr;
2042 return true;
2043 }
2044 }
2045 break;
2046 }
2047 case 'm': {
2048 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
2049 // alignment parameter to embedding the alignment as an attribute of
2050 // the pointer args.
2051 if (unsigned ID = StringSwitch<unsigned>(Name)
2052 .StartsWith("memcpy.", Intrinsic::memcpy)
2053 .StartsWith("memmove.", Intrinsic::memmove)
2054 .Default(0)) {
2055 if (F->arg_size() == 5) {
2056 rename(F);
2057 // Get the types of dest, src, and len
2058 ArrayRef<Type *> ParamTypes =
2059 F->getFunctionType()->params().slice(0, 3);
2060 NewFn =
2061 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
2062 return true;
2063 }
2064 }
2065 if (Name.starts_with("memset.") && F->arg_size() == 5) {
2066 rename(F);
2067 // Get the types of dest, and len
2068 const auto *FT = F->getFunctionType();
2069 Type *ParamTypes[2] = {
2070 FT->getParamType(0), // Dest
2071 FT->getParamType(2) // len
2072 };
2073 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2074 Intrinsic::memset, ParamTypes);
2075 return true;
2076 }
2077
2078 unsigned MaskedID =
2080 .StartsWith("masked.load", Intrinsic::masked_load)
2081 .StartsWith("masked.gather", Intrinsic::masked_gather)
2082 .StartsWith("masked.store", Intrinsic::masked_store)
2083 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2084 .Default(0);
2085 if (MaskedID && F->arg_size() == 4) {
2086 rename(F);
2087 if (MaskedID == Intrinsic::masked_load ||
2088 MaskedID == Intrinsic::masked_gather) {
2090 F->getParent(), MaskedID,
2091 {F->getReturnType(), F->getArg(0)->getType()});
2092 return true;
2093 }
2095 F->getParent(), MaskedID,
2096 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2097 return true;
2098 }
2099 break;
2100 }
2101 case 'n': {
2102 if (Name.consume_front("nvvm.")) {
2103 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2104 if (F->arg_size() == 1) {
2105 Intrinsic::ID IID =
2107 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2108 .Case("clz.i", Intrinsic::ctlz)
2109 .Case("popc.i", Intrinsic::ctpop)
2111 if (IID != Intrinsic::not_intrinsic) {
2112 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2113 {F->getReturnType()});
2114 return true;
2115 }
2116 } else if (F->arg_size() == 2) {
2117 Intrinsic::ID IID =
2119 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2120 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2121 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2122 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2123 .Cases({"mulhi.s", "mulhi.i", "mulhi.ll"}, Intrinsic::smulh)
2124 .Cases({"mulhi.us", "mulhi.ui", "mulhi.ull"}, Intrinsic::umulh)
2126 if (IID != Intrinsic::not_intrinsic) {
2127 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2128 {F->getReturnType()});
2129 return true;
2130 }
2131 }
2132
2133 // Check for nvvm intrinsics that need a return type adjustment.
2134 {
2136 if (IID != Intrinsic::not_intrinsic &&
2138 NewFn = nullptr;
2139 return true;
2140 }
2141 }
2142
2143 // Upgrade Distributed Shared Memory Intrinsics
2145 if (IID != Intrinsic::not_intrinsic) {
2146 rename(F);
2147 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2148 return true;
2149 }
2150
2151 // Upgrade TMA reduction intrinsics
2152 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2153 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2155 if (IID != Intrinsic::not_intrinsic) {
2156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2157 return true;
2158 }
2159
2160 // Upgrade the shared::cluster mbarrier expect_tx/complete_tx/arrive*
2161 // intrinsics to the forms carrying the 32-bit multicast CTA mask and
2162 // the is-multicast flag.
2164 if (IID != Intrinsic::not_intrinsic) {
2165 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2166 return true;
2167 }
2168
2169 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2171 if (IID != Intrinsic::not_intrinsic) {
2172 rename(F);
2174 F->getParent(), IID, F->getReturnType(),
2175 F->getFunctionType()->params());
2176 return true;
2177 }
2178
2179 // Upgrade tcgen05.alloc.shared.* to anyptr alloc intrinsic.
2181 if (IID != Intrinsic::not_intrinsic) {
2182 rename(F);
2183 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2184 {F->getArg(0)->getType()});
2185 return true;
2186 }
2187
2188 // Upgrade TMA copy G2S CTA intrinsics.
2190 if (IID != Intrinsic::not_intrinsic) {
2191 rename(F);
2192 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2193 return true;
2194 }
2195
2196 // Upgrade TMA copy G2S (cluster) intrinsics.
2198 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2199 if (IID != Intrinsic::not_intrinsic) {
2200 rename(F);
2201 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2202 return true;
2203 }
2204
2205 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2206 // (multicast-mask overloading + trailing flag_valid_pattern).
2207 SmallVector<Type *, 1> BulkG2SOvlTys;
2208 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2209 if (IID != Intrinsic::not_intrinsic) {
2210 rename(F);
2211 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2212 BulkG2SOvlTys);
2213 return true;
2214 }
2215
2216 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2217 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2219 if (IID != Intrinsic::not_intrinsic) {
2220 rename(F);
2221 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2222 return true;
2223 }
2224
2225 // Upgrade mbarrier.init intrinsics missing the layout operand.
2227 if (IID != Intrinsic::not_intrinsic) {
2228 rename(F);
2229 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2230 F->getArg(0)->getType());
2231 return true;
2232 }
2233
2234 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2235 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2236 //
2237 // TODO: We could add lohi.i2d.
2238 bool Expand = false;
2239 if (Name.consume_front("abs."))
2240 // nvvm.abs.{i,ii}
2241 Expand =
2242 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2243 else if (Name.consume_front("fabs."))
2244 // nvvm.fabs.{f,ftz.f,d}
2245 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2246 else if (Name.consume_front("add."))
2247 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2248 Expand = getNVVMFPArithUpgrade(Name, NVVMFAddIIDs).has_value();
2249 else if (Name.consume_front("mul."))
2250 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2251 Expand = getNVVMFPArithUpgrade(Name, NVVMFMulIIDs).has_value();
2252 else if (Name.consume_front("ex2.approx."))
2253 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2254 Expand =
2255 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2256 else if (Name.consume_front("fma."))
2257 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
2258 Expand = getNVVMFPArithUpgrade(Name, NVVMFMAIIDs).has_value();
2259 else if (Name.consume_front("atomic.load."))
2260 // nvvm.atomic.load.add.{f32,f64}.p
2261 // nvvm.atomic.load.{inc,dec}.32.p
2262 Expand = StringSwitch<bool>(Name)
2263 .StartsWith("add.f32.p", true)
2264 .StartsWith("add.f64.p", true)
2265 .StartsWith("inc.32.p", true)
2266 .StartsWith("dec.32.p", true)
2267 .Default(false);
2268 else if (Name.consume_front("atomic."))
2269 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2270 // nvvm.atomic.cas.gen.i.{cta,sys}
2271 Expand = StringSwitch<bool>(Name)
2272 .StartsWith("add.gen.", true)
2273 .StartsWith("exch.gen.", true)
2274 .StartsWith("max.gen.", true)
2275 .StartsWith("min.gen.", true)
2276 .StartsWith("inc.gen.", true)
2277 .StartsWith("dec.gen.", true)
2278 .StartsWith("and.gen.", true)
2279 .StartsWith("or.gen.", true)
2280 .StartsWith("xor.gen.", true)
2281 .StartsWith("cas.gen.", true)
2282 .Default(false);
2283 else if (Name.consume_front("bitcast."))
2284 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2285 Expand =
2286 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2287 else if (Name.consume_front("rotate."))
2288 // nvvm.rotate.{b32,b64,right.b64}
2289 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2290 else if (Name.consume_front("ptr.gen.to."))
2291 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2292 Expand = consumeNVVMPtrAddrSpace(Name);
2293 else if (Name.consume_front("ptr."))
2294 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2295 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2296 else if (Name.consume_front("ldg.global."))
2297 // nvvm.ldg.global.{i,p,f}
2298 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2299 Name.starts_with("p."));
2300 else
2301 Expand = StringSwitch<bool>(Name)
2302 .Case("barrier0", true)
2303 .Case("barrier.n", true)
2304 .Case("barrier.sync.cnt", true)
2305 .Case("barrier.sync", true)
2306 .Case("barrier", true)
2307 .Case("bar.sync", true)
2308 .Case("barrier0.popc", true)
2309 .Case("barrier0.and", true)
2310 .Case("barrier0.or", true)
2311 .Case("clz.ll", true)
2312 .Case("popc.ll", true)
2313 .Case("h2f", true)
2314 .Case("swap.lo.hi.b64", true)
2315 .Case("tanh.approx.f32", true)
2316 .Default(false);
2317
2318 if (Expand) {
2319 NewFn = nullptr;
2320 return true;
2321 }
2322 break; // No other 'nvvm.*'.
2323 }
2324 break;
2325 }
2326 case 'o':
2327 if (Name.starts_with("objectsize.")) {
2328 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2329 if (F->arg_size() == 2 || F->arg_size() == 3) {
2330 rename(F);
2331 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2332 Intrinsic::objectsize, Tys);
2333 return true;
2334 }
2335 }
2336 break;
2337
2338 case 'p':
2339 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2340 rename(F);
2342 F->getParent(), Intrinsic::ptr_annotation,
2343 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2344 return true;
2345 }
2346 break;
2347
2348 case 'r': {
2349 if (Name.consume_front("riscv.")) {
2350 Intrinsic::ID ID;
2352 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2353 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2354 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2355 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2357 if (ID != Intrinsic::not_intrinsic) {
2358 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2359 rename(F);
2360 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2361 return true;
2362 }
2363 break; // No other applicable upgrades.
2364 }
2365
2367 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2368 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2370 if (ID != Intrinsic::not_intrinsic) {
2371 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2372 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2373 rename(F);
2374 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2375 return true;
2376 }
2377 break; // No other applicable upgrades.
2378 }
2379
2381 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2382 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2383 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2384 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2385 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2386 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2388 if (ID != Intrinsic::not_intrinsic) {
2389 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2390 rename(F);
2391 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2392 return true;
2393 }
2394 break; // No other applicable upgrades.
2395 }
2396
2397 // Replace llvm.riscv.clmul with llvm.clmul.
2398 if (Name == "clmul.i32" || Name == "clmul.i64") {
2400 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2401 return true;
2402 }
2403
2404 break; // No other 'riscv.*' intrinsics
2405 }
2406 } break;
2407
2408 case 's':
2409 if (Name == "stackprotectorcheck") {
2410 NewFn = nullptr;
2411 return true;
2412 }
2413 if (Name.starts_with("strip.invariant.group")) {
2414 // For clang's usage it would be safe to just drop the
2415 // strip.invariant.group, but to be conservative replace with the
2416 // stronger launder.invariant.group instead.
2418 F->getParent(), Intrinsic::launder_invariant_group,
2419 F->getReturnType());
2420 return true;
2421 }
2422 break;
2423
2424 case 't':
2425 if (Name == "thread.pointer") {
2427 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2428 return true;
2429 }
2430 break;
2431
2432 case 'v': {
2433 if (Name == "var.annotation" && F->arg_size() == 4) {
2434 rename(F);
2436 F->getParent(), Intrinsic::var_annotation,
2437 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2438 return true;
2439 }
2440 if (Name.consume_front("vector.splice")) {
2441 if (Name.starts_with(".left") || Name.starts_with(".right"))
2442 break;
2443 return true;
2444 }
2445 if (shouldUpgradeVPIntrinsic(Name))
2446 return true;
2447 break;
2448 }
2449
2450 case 'w':
2451 if (Name.consume_front("wasm.")) {
2452 Intrinsic::ID ID =
2454 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2455 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2456 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2458 if (ID != Intrinsic::not_intrinsic) {
2459 rename(F);
2460 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2461 F->getReturnType());
2462 return true;
2463 }
2464
2465 if (Name.consume_front("dot.i8x16.i7x16.")) {
2467 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2468 .Case("add.signed",
2469 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2471 if (ID != Intrinsic::not_intrinsic) {
2472 rename(F);
2473 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2474 return true;
2475 }
2476 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2477 }
2478 break; // No other 'wasm.*'.
2479 }
2480 break;
2481
2482 case 'x':
2483 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2484 return true;
2485 }
2486
2487 auto *ST = dyn_cast<StructType>(F->getReturnType());
2488 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2489 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2490 // Replace return type with literal non-packed struct. Only do this for
2491 // intrinsics declared to return a struct, not for intrinsics with
2492 // overloaded return type, in which case the exact struct type will be
2493 // mangled into the name.
2494 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2495 FunctionType *FT = F->getFunctionType();
2496 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2497 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2498 std::string Name = F->getName().str();
2499 rename(F);
2500 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2501 Name, F->getParent());
2502
2503 // The new function may also need remangling.
2504 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2505 NewFn = *Result;
2506 return true;
2507 }
2508 }
2509
2510 // Remangle our intrinsic since we upgrade the mangling
2512 if (Result != std::nullopt) {
2513 NewFn = *Result;
2514 return true;
2515 }
2516
2518 return true;
2519
2520 // This may not belong here. This function is effectively being overloaded
2521 // to both detect an intrinsic which needs upgrading, and to provide the
2522 // upgraded form of the intrinsic. We should perhaps have two separate
2523 // functions for this.
2524
2525 return false;
2526}
2527
2529 bool CanUpgradeDebugIntrinsicsToRecords) {
2530 NewFn = nullptr;
2531 bool Upgraded =
2532 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2533
2534 // Upgrade intrinsic attributes. This does not change the function.
2535 if (NewFn)
2536 F = NewFn;
2537 if (Intrinsic::ID id = F->getIntrinsicID()) {
2538 // Only do this if the intrinsic signature is valid.
2539 SmallVector<Type *> OverloadTys;
2540 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2541 F->setAttributes(
2542 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2543 }
2544 return Upgraded;
2545}
2546
2548 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2549 GV->getName() == "llvm.global_dtors")) ||
2550 !GV->hasInitializer())
2551 return nullptr;
2553 if (!ATy)
2554 return nullptr;
2556 if (!STy || STy->getNumElements() != 2)
2557 return nullptr;
2558
2559 IRBuilder<> IRB(*GV->getParent());
2560 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2561 IRB.getPtrTy());
2562 Constant *Init = GV->getInitializer();
2563 unsigned N = Init->getNumOperands();
2564 std::vector<Constant *> NewCtors(N);
2565 for (unsigned i = 0; i != N; ++i) {
2566 auto Ctor = cast<Constant>(Init->getOperand(i));
2567 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2568 Ctor->getAggregateElement(1),
2570 }
2571 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2572
2573 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2574 NewInit, GV->getName());
2575}
2576
2577// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2578// to byte shuffles.
2580 unsigned Shift) {
2581 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2582 unsigned NumElts = ResultTy->getNumElements() * 8;
2583
2584 // Bitcast from a 64-bit element type to a byte element type.
2585 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2586 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2587
2588 // We'll be shuffling in zeroes.
2589 Value *Res = Constant::getNullValue(VecTy);
2590
2591 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2592 // we'll just return the zero vector.
2593 if (Shift < 16) {
2594 int Idxs[64];
2595 // 256/512-bit version is split into 2/4 16-byte lanes.
2596 for (unsigned l = 0; l != NumElts; l += 16)
2597 for (unsigned i = 0; i != 16; ++i) {
2598 unsigned Idx = NumElts + i - Shift;
2599 if (Idx < NumElts)
2600 Idx -= NumElts - 16; // end of lane, switch operand.
2601 Idxs[l + i] = Idx + l;
2602 }
2603
2604 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2605 }
2606
2607 // Bitcast back to a 64-bit element type.
2608 return Builder.CreateBitCast(Res, ResultTy, "cast");
2609}
2610
2611// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2612// to byte shuffles.
2614 unsigned Shift) {
2615 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2616 unsigned NumElts = ResultTy->getNumElements() * 8;
2617
2618 // Bitcast from a 64-bit element type to a byte element type.
2619 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2620 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2621
2622 // We'll be shuffling in zeroes.
2623 Value *Res = Constant::getNullValue(VecTy);
2624
2625 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2626 // we'll just return the zero vector.
2627 if (Shift < 16) {
2628 int Idxs[64];
2629 // 256/512-bit version is split into 2/4 16-byte lanes.
2630 for (unsigned l = 0; l != NumElts; l += 16)
2631 for (unsigned i = 0; i != 16; ++i) {
2632 unsigned Idx = i + Shift;
2633 if (Idx >= 16)
2634 Idx += NumElts - 16; // end of lane, switch operand.
2635 Idxs[l + i] = Idx + l;
2636 }
2637
2638 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2639 }
2640
2641 // Bitcast back to a 64-bit element type.
2642 return Builder.CreateBitCast(Res, ResultTy, "cast");
2643}
2644
2645static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2646 unsigned NumElts) {
2647 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2649 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2650 Mask = Builder.CreateBitCast(Mask, MaskTy);
2651
2652 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2653 // i8 and we need to extract down to the right number of elements.
2654 if (NumElts <= 4) {
2655 int Indices[4];
2656 for (unsigned i = 0; i != NumElts; ++i)
2657 Indices[i] = i;
2658 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2659 "extract");
2660 }
2661
2662 return Mask;
2663}
2664
2665static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2666 Value *Op1) {
2667 // If the mask is all ones just emit the first operation.
2668 if (const auto *C = dyn_cast<Constant>(Mask))
2669 if (C->isAllOnesValue())
2670 return Op0;
2671
2672 Mask = getX86MaskVec(Builder, Mask,
2673 cast<FixedVectorType>(Op0->getType())->getNumElements());
2674 return Builder.CreateSelect(Mask, Op0, Op1);
2675}
2676
2677static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2678 Value *Op1) {
2679 // If the mask is all ones just emit the first operation.
2680 if (const auto *C = dyn_cast<Constant>(Mask))
2681 if (C->isAllOnesValue())
2682 return Op0;
2683
2684 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2685 Mask->getType()->getIntegerBitWidth());
2686 Mask = Builder.CreateBitCast(Mask, MaskTy);
2687 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2688 return Builder.CreateSelect(Mask, Op0, Op1);
2689}
2690
2691// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2692// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2693// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2695 Value *Op1, Value *Shift,
2696 Value *Passthru, Value *Mask,
2697 bool IsVALIGN) {
2698 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2699
2700 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2701 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2702 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2703 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2704
2705 // Mask the immediate for VALIGN.
2706 if (IsVALIGN)
2707 ShiftVal &= (NumElts - 1);
2708
2709 // If palignr is shifting the pair of vectors more than the size of two
2710 // lanes, emit zero.
2711 if (ShiftVal >= 32)
2713
2714 // If palignr is shifting the pair of input vectors more than one lane,
2715 // but less than two lanes, convert to shifting in zeroes.
2716 if (ShiftVal > 16) {
2717 ShiftVal -= 16;
2718 Op1 = Op0;
2720 }
2721
2722 int Indices[64];
2723 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2724 for (unsigned l = 0; l < NumElts; l += 16) {
2725 for (unsigned i = 0; i != 16; ++i) {
2726 unsigned Idx = ShiftVal + i;
2727 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2728 Idx += NumElts - 16; // End of lane, switch operand.
2729 Indices[l + i] = Idx + l;
2730 }
2731 }
2732
2733 Value *Align = Builder.CreateShuffleVector(
2734 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2735
2736 return emitX86Select(Builder, Mask, Align, Passthru);
2737}
2738
2740 bool ZeroMask, bool IndexForm) {
2741 Type *Ty = CI.getType();
2742 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2743 unsigned EltWidth = Ty->getScalarSizeInBits();
2744 bool IsFloat = Ty->isFPOrFPVectorTy();
2745 Intrinsic::ID IID;
2746 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2747 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2748 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2749 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2750 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2751 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2752 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2753 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2754 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2755 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2756 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2757 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2758 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2759 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2760 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2761 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2762 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2763 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2764 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2765 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2766 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2767 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2768 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2769 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2770 else if (VecWidth == 128 && EltWidth == 16)
2771 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2772 else if (VecWidth == 256 && EltWidth == 16)
2773 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2774 else if (VecWidth == 512 && EltWidth == 16)
2775 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2776 else if (VecWidth == 128 && EltWidth == 8)
2777 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2778 else if (VecWidth == 256 && EltWidth == 8)
2779 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2780 else if (VecWidth == 512 && EltWidth == 8)
2781 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2782 else
2783 llvm_unreachable("Unexpected intrinsic");
2784
2785 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2786 CI.getArgOperand(2) };
2787
2788 // If this isn't index form we need to swap operand 0 and 1.
2789 if (!IndexForm)
2790 std::swap(Args[0], Args[1]);
2791
2792 Value *V = Builder.CreateIntrinsic(IID, Args);
2793 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2794 : Builder.CreateBitCast(CI.getArgOperand(1),
2795 Ty);
2796 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2797}
2798
2800 Intrinsic::ID IID) {
2801 Type *Ty = CI.getType();
2802 Value *Op0 = CI.getOperand(0);
2803 Value *Op1 = CI.getOperand(1);
2804 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2805
2806 if (CI.arg_size() == 4) { // For masked intrinsics.
2807 Value *VecSrc = CI.getOperand(2);
2808 Value *Mask = CI.getOperand(3);
2809 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2810 }
2811 return Res;
2812}
2813
2815 bool IsRotateRight) {
2816 Type *Ty = CI.getType();
2817 Value *Src = CI.getArgOperand(0);
2818 Value *Amt = CI.getArgOperand(1);
2819
2820 // Amount may be scalar immediate, in which case create a splat vector.
2821 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2822 // we only care about the lowest log2 bits anyway.
2823 if (Amt->getType() != Ty) {
2824 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2825 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2826 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2827 }
2828
2829 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2830 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2831
2832 if (CI.arg_size() == 4) { // For masked intrinsics.
2833 Value *VecSrc = CI.getOperand(2);
2834 Value *Mask = CI.getOperand(3);
2835 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2836 }
2837 return Res;
2838}
2839
2840static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2841 bool IsSigned) {
2842 Type *Ty = CI.getType();
2843 Value *LHS = CI.getArgOperand(0);
2844 Value *RHS = CI.getArgOperand(1);
2845
2847 switch (Imm) {
2848 case 0x0:
2850 break;
2851 case 0x1:
2853 break;
2854 case 0x2:
2856 break;
2857 case 0x3:
2859 break;
2860 case 0x4:
2862 break;
2863 case 0x5:
2865 break;
2866 case 0x6:
2867 return Constant::getNullValue(Ty); // FALSE
2868 case 0x7:
2869 return Constant::getAllOnesValue(Ty); // TRUE
2870 default:
2871 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2872 }
2873
2874 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2875 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2876 return Ext;
2877}
2878
2880 bool IsShiftRight, bool ZeroMask) {
2881 Type *Ty = CI.getType();
2882 Value *Op0 = CI.getArgOperand(0);
2883 Value *Op1 = CI.getArgOperand(1);
2884 Value *Amt = CI.getArgOperand(2);
2885
2886 if (IsShiftRight)
2887 std::swap(Op0, Op1);
2888
2889 // Amount may be scalar immediate, in which case create a splat vector.
2890 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2891 // we only care about the lowest log2 bits anyway.
2892 if (Amt->getType() != Ty) {
2893 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2894 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2895 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2896 }
2897
2898 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2899 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2900
2901 unsigned NumArgs = CI.arg_size();
2902 if (NumArgs >= 4) { // For masked intrinsics.
2903 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2904 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2905 CI.getArgOperand(0);
2906 Value *Mask = CI.getOperand(NumArgs - 1);
2907 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2908 }
2909 return Res;
2910}
2911
2913 Value *Mask, bool Aligned) {
2914 const Align Alignment =
2915 Aligned
2916 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2917 : Align(1);
2918
2919 // If the mask is all ones just emit a regular store.
2920 if (const auto *C = dyn_cast<Constant>(Mask))
2921 if (C->isAllOnesValue())
2922 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2923
2924 // Convert the mask from an integer type to a vector of i1.
2925 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2926 Mask = getX86MaskVec(Builder, Mask, NumElts);
2927 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2928}
2929
2931 Value *Passthru, Value *Mask, bool Aligned) {
2932 Type *ValTy = Passthru->getType();
2933 const Align Alignment =
2934 Aligned
2935 ? Align(
2937 8)
2938 : Align(1);
2939
2940 // If the mask is all ones just emit a regular store.
2941 if (const auto *C = dyn_cast<Constant>(Mask))
2942 if (C->isAllOnesValue())
2943 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2944
2945 // Convert the mask from an integer type to a vector of i1.
2946 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2947 Mask = getX86MaskVec(Builder, Mask, NumElts);
2948 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2949}
2950
2951static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2952 Type *Ty = CI.getType();
2953 Value *Op0 = CI.getArgOperand(0);
2954 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2955 {Op0, Builder.getInt1(false)});
2956 if (CI.arg_size() == 3)
2957 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2958 return Res;
2959}
2960
2961static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2962 Type *Ty = CI.getType();
2963
2964 // Arguments have a vXi32 type so cast to vXi64.
2965 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2966 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2967
2968 if (IsSigned) {
2969 // Shift left then arithmetic shift right.
2970 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2971 LHS = Builder.CreateShl(LHS, ShiftAmt);
2972 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2973 RHS = Builder.CreateShl(RHS, ShiftAmt);
2974 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2975 } else {
2976 // Clear the upper bits.
2977 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2978 LHS = Builder.CreateAnd(LHS, Mask);
2979 RHS = Builder.CreateAnd(RHS, Mask);
2980 }
2981
2982 Value *Res = Builder.CreateMul(LHS, RHS);
2983
2984 if (CI.arg_size() == 4)
2985 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2986
2987 return Res;
2988}
2989
2990// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2992 Value *Mask) {
2993 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2994 if (Mask) {
2995 const auto *C = dyn_cast<Constant>(Mask);
2996 if (!C || !C->isAllOnesValue())
2997 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2998 }
2999
3000 if (NumElts < 8) {
3001 int Indices[8];
3002 for (unsigned i = 0; i != NumElts; ++i)
3003 Indices[i] = i;
3004 for (unsigned i = NumElts; i != 8; ++i)
3005 Indices[i] = NumElts + i % NumElts;
3006 Vec = Builder.CreateShuffleVector(Vec,
3008 Indices);
3009 }
3010 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
3011}
3012
3014 unsigned CC, bool Signed) {
3015 Value *Op0 = CI.getArgOperand(0);
3016 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
3017
3018 Value *Cmp;
3019 if (CC == 3) {
3021 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3022 } else if (CC == 7) {
3024 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
3025 } else {
3027 switch (CC) {
3028 default: llvm_unreachable("Unknown condition code");
3029 case 0: Pred = ICmpInst::ICMP_EQ; break;
3030 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
3031 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
3032 case 4: Pred = ICmpInst::ICMP_NE; break;
3033 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
3034 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
3035 }
3036 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
3037 }
3038
3039 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
3040
3041 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
3042}
3043
3044// Replace a masked intrinsic with an older unmasked intrinsic.
3046 Intrinsic::ID IID) {
3047 Value *Rep =
3048 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
3049 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
3050}
3051
3053 Value* A = CI.getArgOperand(0);
3054 Value* B = CI.getArgOperand(1);
3055 Value* Src = CI.getArgOperand(2);
3056 Value* Mask = CI.getArgOperand(3);
3057
3058 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
3059 Value* Cmp = Builder.CreateIsNotNull(AndNode);
3060 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
3061 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
3062 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
3063 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
3064}
3065
3067 Value* Op = CI.getArgOperand(0);
3068 Type* ReturnOp = CI.getType();
3069 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
3070 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
3071 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
3072}
3073
3074// Replace intrinsic with unmasked version and a select.
3076 CallBase &CI, Value *&Rep) {
3077 Name = Name.substr(12); // Remove avx512.mask.
3078
3079 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3080 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3081 Intrinsic::ID IID;
3082 if (Name.starts_with("max.p")) {
3083 if (VecWidth == 128 && EltWidth == 32)
3084 IID = Intrinsic::x86_sse_max_ps;
3085 else if (VecWidth == 128 && EltWidth == 64)
3086 IID = Intrinsic::x86_sse2_max_pd;
3087 else if (VecWidth == 256 && EltWidth == 32)
3088 IID = Intrinsic::x86_avx_max_ps_256;
3089 else if (VecWidth == 256 && EltWidth == 64)
3090 IID = Intrinsic::x86_avx_max_pd_256;
3091 else
3092 llvm_unreachable("Unexpected intrinsic");
3093 } else if (Name.starts_with("min.p")) {
3094 if (VecWidth == 128 && EltWidth == 32)
3095 IID = Intrinsic::x86_sse_min_ps;
3096 else if (VecWidth == 128 && EltWidth == 64)
3097 IID = Intrinsic::x86_sse2_min_pd;
3098 else if (VecWidth == 256 && EltWidth == 32)
3099 IID = Intrinsic::x86_avx_min_ps_256;
3100 else if (VecWidth == 256 && EltWidth == 64)
3101 IID = Intrinsic::x86_avx_min_pd_256;
3102 else
3103 llvm_unreachable("Unexpected intrinsic");
3104 } else if (Name.starts_with("pshuf.b.")) {
3105 if (VecWidth == 128)
3106 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3107 else if (VecWidth == 256)
3108 IID = Intrinsic::x86_avx2_pshuf_b;
3109 else if (VecWidth == 512)
3110 IID = Intrinsic::x86_avx512_pshuf_b_512;
3111 else
3112 llvm_unreachable("Unexpected intrinsic");
3113 } else if (Name.starts_with("pmul.hr.sw.")) {
3114 if (VecWidth == 128)
3115 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3116 else if (VecWidth == 256)
3117 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3118 else if (VecWidth == 512)
3119 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3120 else
3121 llvm_unreachable("Unexpected intrinsic");
3122 } else if (Name.starts_with("pmulh.w")) {
3123 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3124 "Unexpected intrinsic");
3125 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::smulh);
3126 return true;
3127 } else if (Name.starts_with("pmulhu.w")) {
3128 assert((VecWidth == 128 || VecWidth == 256 || VecWidth == 512) &&
3129 "Unexpected intrinsic");
3130 Rep = upgradeX86BinaryIntrinsics(Builder, CI, Intrinsic::umulh);
3131 return true;
3132 } else if (Name.starts_with("pmaddw.d.")) {
3133 if (VecWidth == 128)
3134 IID = Intrinsic::x86_sse2_pmadd_wd;
3135 else if (VecWidth == 256)
3136 IID = Intrinsic::x86_avx2_pmadd_wd;
3137 else if (VecWidth == 512)
3138 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3139 else
3140 llvm_unreachable("Unexpected intrinsic");
3141 } else if (Name.starts_with("pmaddubs.w.")) {
3142 if (VecWidth == 128)
3143 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3144 else if (VecWidth == 256)
3145 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3146 else if (VecWidth == 512)
3147 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3148 else
3149 llvm_unreachable("Unexpected intrinsic");
3150 } else if (Name.starts_with("packsswb.")) {
3151 if (VecWidth == 128)
3152 IID = Intrinsic::x86_sse2_packsswb_128;
3153 else if (VecWidth == 256)
3154 IID = Intrinsic::x86_avx2_packsswb;
3155 else if (VecWidth == 512)
3156 IID = Intrinsic::x86_avx512_packsswb_512;
3157 else
3158 llvm_unreachable("Unexpected intrinsic");
3159 } else if (Name.starts_with("packssdw.")) {
3160 if (VecWidth == 128)
3161 IID = Intrinsic::x86_sse2_packssdw_128;
3162 else if (VecWidth == 256)
3163 IID = Intrinsic::x86_avx2_packssdw;
3164 else if (VecWidth == 512)
3165 IID = Intrinsic::x86_avx512_packssdw_512;
3166 else
3167 llvm_unreachable("Unexpected intrinsic");
3168 } else if (Name.starts_with("packuswb.")) {
3169 if (VecWidth == 128)
3170 IID = Intrinsic::x86_sse2_packuswb_128;
3171 else if (VecWidth == 256)
3172 IID = Intrinsic::x86_avx2_packuswb;
3173 else if (VecWidth == 512)
3174 IID = Intrinsic::x86_avx512_packuswb_512;
3175 else
3176 llvm_unreachable("Unexpected intrinsic");
3177 } else if (Name.starts_with("packusdw.")) {
3178 if (VecWidth == 128)
3179 IID = Intrinsic::x86_sse41_packusdw;
3180 else if (VecWidth == 256)
3181 IID = Intrinsic::x86_avx2_packusdw;
3182 else if (VecWidth == 512)
3183 IID = Intrinsic::x86_avx512_packusdw_512;
3184 else
3185 llvm_unreachable("Unexpected intrinsic");
3186 } else if (Name.starts_with("vpermilvar.")) {
3187 if (VecWidth == 128 && EltWidth == 32)
3188 IID = Intrinsic::x86_avx_vpermilvar_ps;
3189 else if (VecWidth == 128 && EltWidth == 64)
3190 IID = Intrinsic::x86_avx_vpermilvar_pd;
3191 else if (VecWidth == 256 && EltWidth == 32)
3192 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3193 else if (VecWidth == 256 && EltWidth == 64)
3194 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3195 else if (VecWidth == 512 && EltWidth == 32)
3196 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3197 else if (VecWidth == 512 && EltWidth == 64)
3198 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3199 else
3200 llvm_unreachable("Unexpected intrinsic");
3201 } else if (Name == "cvtpd2dq.256") {
3202 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3203 } else if (Name == "cvtpd2ps.256") {
3204 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3205 } else if (Name == "cvttpd2dq.256") {
3206 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3207 } else if (Name == "cvttps2dq.128") {
3208 IID = Intrinsic::x86_sse2_cvttps2dq;
3209 } else if (Name == "cvttps2dq.256") {
3210 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3211 } else if (Name.starts_with("permvar.")) {
3212 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3213 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3214 IID = Intrinsic::x86_avx2_permps;
3215 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3216 IID = Intrinsic::x86_avx2_permd;
3217 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3218 IID = Intrinsic::x86_avx512_permvar_df_256;
3219 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3220 IID = Intrinsic::x86_avx512_permvar_di_256;
3221 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3222 IID = Intrinsic::x86_avx512_permvar_sf_512;
3223 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3224 IID = Intrinsic::x86_avx512_permvar_si_512;
3225 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3226 IID = Intrinsic::x86_avx512_permvar_df_512;
3227 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3228 IID = Intrinsic::x86_avx512_permvar_di_512;
3229 else if (VecWidth == 128 && EltWidth == 16)
3230 IID = Intrinsic::x86_avx512_permvar_hi_128;
3231 else if (VecWidth == 256 && EltWidth == 16)
3232 IID = Intrinsic::x86_avx512_permvar_hi_256;
3233 else if (VecWidth == 512 && EltWidth == 16)
3234 IID = Intrinsic::x86_avx512_permvar_hi_512;
3235 else if (VecWidth == 128 && EltWidth == 8)
3236 IID = Intrinsic::x86_avx512_permvar_qi_128;
3237 else if (VecWidth == 256 && EltWidth == 8)
3238 IID = Intrinsic::x86_avx512_permvar_qi_256;
3239 else if (VecWidth == 512 && EltWidth == 8)
3240 IID = Intrinsic::x86_avx512_permvar_qi_512;
3241 else
3242 llvm_unreachable("Unexpected intrinsic");
3243 } else if (Name.starts_with("dbpsadbw.")) {
3244 if (VecWidth == 128)
3245 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3246 else if (VecWidth == 256)
3247 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3248 else if (VecWidth == 512)
3249 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3250 else
3251 llvm_unreachable("Unexpected intrinsic");
3252 } else if (Name.starts_with("pmultishift.qb.")) {
3253 if (VecWidth == 128)
3254 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3255 else if (VecWidth == 256)
3256 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3257 else if (VecWidth == 512)
3258 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3259 else
3260 llvm_unreachable("Unexpected intrinsic");
3261 } else if (Name.starts_with("conflict.")) {
3262 if (Name[9] == 'd' && VecWidth == 128)
3263 IID = Intrinsic::x86_avx512_conflict_d_128;
3264 else if (Name[9] == 'd' && VecWidth == 256)
3265 IID = Intrinsic::x86_avx512_conflict_d_256;
3266 else if (Name[9] == 'd' && VecWidth == 512)
3267 IID = Intrinsic::x86_avx512_conflict_d_512;
3268 else if (Name[9] == 'q' && VecWidth == 128)
3269 IID = Intrinsic::x86_avx512_conflict_q_128;
3270 else if (Name[9] == 'q' && VecWidth == 256)
3271 IID = Intrinsic::x86_avx512_conflict_q_256;
3272 else if (Name[9] == 'q' && VecWidth == 512)
3273 IID = Intrinsic::x86_avx512_conflict_q_512;
3274 else
3275 llvm_unreachable("Unexpected intrinsic");
3276 } else if (Name.starts_with("pavg.")) {
3277 if (Name[5] == 'b' && VecWidth == 128)
3278 IID = Intrinsic::x86_sse2_pavg_b;
3279 else if (Name[5] == 'b' && VecWidth == 256)
3280 IID = Intrinsic::x86_avx2_pavg_b;
3281 else if (Name[5] == 'b' && VecWidth == 512)
3282 IID = Intrinsic::x86_avx512_pavg_b_512;
3283 else if (Name[5] == 'w' && VecWidth == 128)
3284 IID = Intrinsic::x86_sse2_pavg_w;
3285 else if (Name[5] == 'w' && VecWidth == 256)
3286 IID = Intrinsic::x86_avx2_pavg_w;
3287 else if (Name[5] == 'w' && VecWidth == 512)
3288 IID = Intrinsic::x86_avx512_pavg_w_512;
3289 else
3290 llvm_unreachable("Unexpected intrinsic");
3291 } else
3292 return false;
3293
3294 SmallVector<Value *, 4> Args(CI.args());
3295 Args.pop_back();
3296 Args.pop_back();
3297 Rep = Builder.CreateIntrinsic(IID, Args);
3298 unsigned NumArgs = CI.arg_size();
3299 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3300 CI.getArgOperand(NumArgs - 2));
3301 return true;
3302}
3303
3304/// Upgrade comment in call to inline asm that represents an objc retain release
3305/// marker.
3306void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3307 size_t Pos;
3308 if (AsmStr->find("mov\tfp") == 0 &&
3309 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3310 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3311 AsmStr->replace(Pos, 1, ";");
3312 }
3313}
3314
3316 StringRef Name,
3317 const Intrinsic::ID (&IIDs)[2][3]) {
3318 auto Result = getNVVMFPArithUpgrade(Name, IIDs);
3319 assert(Result && "unsupported nvvm.{add,mul,fma}.* intrinsic");
3320 auto [IID, RoundingMode] = *Result;
3321
3322 // The legacy bf16 variants used integer storage types for the bf16
3323 // operands; bitcast them to the floating-point types of the new
3324 // overloaded intrinsic.
3325 Type *Ty = CI->getType();
3326 const bool IsLegacyBF16Storage = Ty->getScalarType()->isIntegerTy();
3327 if (IsLegacyBF16Storage)
3328 Ty = Name.ends_with("bf16x2")
3329 ? FixedVectorType::get(Builder.getBFloatTy(), 2)
3330 : static_cast<Type *>(Builder.getBFloatTy());
3331
3333 for (Value *Arg : CI->args())
3334 Args.push_back(IsLegacyBF16Storage ? Builder.CreateBitCast(Arg, Ty) : Arg);
3335 Args.push_back(Builder.getInt32(static_cast<int>(RoundingMode)));
3336
3337 Value *Rep = Builder.CreateIntrinsic(Ty, IID, Args);
3338 return IsLegacyBF16Storage ? Builder.CreateBitCast(Rep, CI->getType()) : Rep;
3339}
3340
3342 Function *F, IRBuilder<> &Builder) {
3343 Value *Rep = nullptr;
3344
3345 if (Name == "abs.i" || Name == "abs.ll") {
3346 Value *Arg = CI->getArgOperand(0);
3347 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3348 {Arg, Builder.getTrue()},
3349 /*FMFSource=*/nullptr, "abs");
3350 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3351 Type *Ty = (Name == "abs.bf16")
3352 ? Builder.getBFloatTy()
3353 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3354 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3355 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3356 Rep = Builder.CreateBitCast(Abs, CI->getType());
3357 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3358 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3359 : Intrinsic::nvvm_fabs;
3360 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3361 } else if (Name.consume_front("add.")) {
3362 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3363 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFAddIIDs);
3364 } else if (Name.consume_front("mul.")) {
3365 // nvvm.mul.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3366 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMulIIDs);
3367 } else if (Name.consume_front("ex2.approx.")) {
3368 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3369 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3370 : Intrinsic::nvvm_ex2_approx;
3371 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3372 } else if (Name.consume_front("fma.")) {
3373 // nvvm.fma.<rnd>{.ftz}{.sat|.relu}.{f,d,f16,f16x2,bf16,bf16x2}
3374 Rep = upgradeNVVMFPArithCall(Builder, CI, Name, NVVMFMAIIDs);
3375 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3376 Name.starts_with("atomic.load.add.f64.p")) {
3377 Value *Ptr = CI->getArgOperand(0);
3378 Value *Val = CI->getArgOperand(1);
3379 Rep = Builder.CreateAtomicRMW(
3381 CI->getContext().getOrInsertSyncScopeID("device"));
3383 LLVMContext::MD_atomic_ignore_denormal_mode,
3384 MDNode::get(CI->getContext(), {}));
3385 // The default scope for atomic.load.* intrinsics is device
3386 // (= gpu scope in ptx), but the default LLVM atomic scope is
3387 // "system"
3388 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3389 Name.starts_with("atomic.load.dec.32.p")) {
3390 Value *Ptr = CI->getArgOperand(0);
3391 Value *Val = CI->getArgOperand(1);
3392 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3394 Rep = Builder.CreateAtomicRMW(
3396 CI->getContext().getOrInsertSyncScopeID("device"));
3397 // See comment above.
3398 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3399 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3400 StringRef Op = Name.substr(StringRef("atomic.").size());
3401 Value *Ptr = CI->getArgOperand(0);
3402 Value *Val = CI->getArgOperand(1);
3404 Op.contains(".cta.") ? "block" : "");
3405 if (Op.starts_with("cas.")) {
3406 Value *New = CI->getArgOperand(2);
3407 Value *Pair = Builder.CreateAtomicCmpXchg(
3408 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3410 Rep = Builder.CreateExtractValue(Pair, 0);
3411 } else {
3412 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3413 // because we were actually missing those intrinsics!
3414 AtomicRMWInst::BinOp BinOp =
3416 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3417 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3428 "unexpected nvvm scoped atomic intrinsic");
3429 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3431 if (BinOp == AtomicRMWInst::FAdd) {
3432 cast<Instruction>(Rep)->setMetadata(
3433 LLVMContext::MD_atomic_ignore_denormal_mode,
3434 MDNode::get(CI->getContext(), {}));
3435 }
3436 }
3437 } else if (Name == "clz.ll") {
3438 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3439 Value *Arg = CI->getArgOperand(0);
3440 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3441 {Arg, Builder.getFalse()},
3442 /*FMFSource=*/nullptr, "ctlz");
3443 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3444 } else if (Name == "popc.ll") {
3445 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3446 // i64.
3447 Value *Arg = CI->getArgOperand(0);
3448 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3449 Arg, /*FMFSource=*/nullptr, "ctpop");
3450 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3451 } else if (Name == "h2f") {
3452 Value *Cast =
3453 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3454 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3455 } else if (Name.consume_front("bitcast.") &&
3456 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3457 Name == "d2ll")) {
3458 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3459 } else if (Name == "rotate.b32") {
3460 Value *Arg = CI->getOperand(0);
3461 Value *ShiftAmt = CI->getOperand(1);
3462 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3463 {Arg, Arg, ShiftAmt});
3464 } else if (Name == "rotate.b64") {
3465 Type *Int64Ty = Builder.getInt64Ty();
3466 Value *Arg = CI->getOperand(0);
3467 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3468 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3469 {Arg, Arg, ZExtShiftAmt});
3470 } else if (Name == "rotate.right.b64") {
3471 Type *Int64Ty = Builder.getInt64Ty();
3472 Value *Arg = CI->getOperand(0);
3473 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3474 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3475 {Arg, Arg, ZExtShiftAmt});
3476 } else if (Name == "swap.lo.hi.b64") {
3477 Type *Int64Ty = Builder.getInt64Ty();
3478 Value *Arg = CI->getOperand(0);
3479 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3480 {Arg, Arg, Builder.getInt64(32)});
3481 } else if ((Name.consume_front("ptr.gen.to.") &&
3482 consumeNVVMPtrAddrSpace(Name)) ||
3483 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3484 Name.starts_with(".to.gen"))) {
3485 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3486 } else if (Name.consume_front("ldg.global")) {
3487 Value *Ptr = CI->getArgOperand(0);
3488 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3489 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3490 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3491 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3492 MDNode *MD = MDNode::get(Builder.getContext(), {});
3493 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3494 return LD;
3495 } else if (Name == "tanh.approx.f32") {
3496 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3497 FastMathFlags FMF;
3498 FMF.setApproxFunc();
3499 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3500 FMF);
3501 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3502 Value *Arg =
3503 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3504 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3505 {}, {Arg});
3506 } else if (Name == "barrier") {
3507 Rep = Builder.CreateIntrinsic(
3508 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3509 {CI->getArgOperand(0), CI->getArgOperand(1)});
3510 } else if (Name == "barrier.sync") {
3511 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3512 {CI->getArgOperand(0)});
3513 } else if (Name == "barrier.sync.cnt") {
3514 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3515 {CI->getArgOperand(0), CI->getArgOperand(1)});
3516 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3517 Name == "barrier0.or") {
3518 Value *C = CI->getArgOperand(0);
3519 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3520
3521 Intrinsic::ID IID =
3523 .Case("barrier0.popc",
3524 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3525 .Case("barrier0.and",
3526 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3527 .Case("barrier0.or",
3528 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3529 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3530 Rep = Builder.CreateZExt(Bar, CI->getType());
3531 } else {
3533 if (IID != Intrinsic::not_intrinsic &&
3535 rename(F);
3536 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3538 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3539 Value *Arg = CI->getArgOperand(I);
3540 Type *OldType = Arg->getType();
3541 Type *NewType = NewFn->getArg(I)->getType();
3542 Args.push_back(
3543 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3544 ? Builder.CreateBitCast(Arg, NewType)
3545 : Arg);
3546 }
3547 Rep = Builder.CreateCall(NewFn, Args);
3548 if (F->getReturnType()->isIntegerTy())
3549 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3550 }
3551 }
3552
3553 return Rep;
3554}
3555
3557 IRBuilder<> &Builder) {
3558 LLVMContext &C = F->getContext();
3559 Value *Rep = nullptr;
3560
3561 if (Name.starts_with("sse4a.movnt.")) {
3563 Elts.push_back(
3564 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3565 MDNode *Node = MDNode::get(C, Elts);
3566
3567 Value *Arg0 = CI->getArgOperand(0);
3568 Value *Arg1 = CI->getArgOperand(1);
3569
3570 // Nontemporal (unaligned) store of the 0'th element of the float/double
3571 // vector.
3572 Value *Extract =
3573 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3574
3575 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3576 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3577 } else if (Name.starts_with("avx.movnt.") ||
3578 Name.starts_with("avx512.storent.")) {
3580 Elts.push_back(
3581 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3582 MDNode *Node = MDNode::get(C, Elts);
3583
3584 Value *Arg0 = CI->getArgOperand(0);
3585 Value *Arg1 = CI->getArgOperand(1);
3586
3587 StoreInst *SI = Builder.CreateAlignedStore(
3588 Arg1, Arg0,
3590 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3591 } else if (Name == "sse2.storel.dq") {
3592 Value *Arg0 = CI->getArgOperand(0);
3593 Value *Arg1 = CI->getArgOperand(1);
3594
3595 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3596 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3597 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3598 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3599 } else if (Name.starts_with("sse.storeu.") ||
3600 Name.starts_with("sse2.storeu.") ||
3601 Name.starts_with("avx.storeu.")) {
3602 Value *Arg0 = CI->getArgOperand(0);
3603 Value *Arg1 = CI->getArgOperand(1);
3604 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3605 } else if (Name == "avx512.mask.store.ss") {
3606 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3607 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3608 Mask, false);
3609 } else if (Name.starts_with("avx512.mask.store")) {
3610 // "avx512.mask.storeu." or "avx512.mask.store."
3611 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3612 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3613 CI->getArgOperand(2), Aligned);
3614 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3615 // Upgrade packed integer vector compare intrinsics to compare instructions.
3616 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3617 bool CmpEq = Name[9] == 'e';
3618 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3619 CI->getArgOperand(0), CI->getArgOperand(1));
3620 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3621 } else if (Name.starts_with("avx512.broadcastm")) {
3622 Type *ExtTy = Type::getInt32Ty(C);
3623 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3624 ExtTy = Type::getInt64Ty(C);
3625 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3626 ExtTy->getPrimitiveSizeInBits();
3627 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3628 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3629 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3630 Value *Vec = CI->getArgOperand(0);
3631 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3632 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3633 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3634 } else if (Name.starts_with("avx.sqrt.p") ||
3635 Name.starts_with("sse2.sqrt.p") ||
3636 Name.starts_with("sse.sqrt.p")) {
3637 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3638 {CI->getArgOperand(0)});
3639 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3640 if (CI->arg_size() == 4 &&
3641 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3642 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3643 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3644 : Intrinsic::x86_avx512_sqrt_pd_512;
3645
3646 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3647 Rep = Builder.CreateIntrinsic(IID, Args);
3648 } else {
3649 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3650 {CI->getArgOperand(0)});
3651 }
3652 Rep =
3653 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3654 } else if (Name.starts_with("avx512.ptestm") ||
3655 Name.starts_with("avx512.ptestnm")) {
3656 Value *Op0 = CI->getArgOperand(0);
3657 Value *Op1 = CI->getArgOperand(1);
3658 Value *Mask = CI->getArgOperand(2);
3659 Rep = Builder.CreateAnd(Op0, Op1);
3660 llvm::Type *Ty = Op0->getType();
3662 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3665 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3666 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3667 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3668 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3669 ->getNumElements();
3670 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3671 Rep =
3672 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3673 } else if (Name.starts_with("avx512.kunpck")) {
3674 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3675 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3676 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3677 int Indices[64];
3678 for (unsigned i = 0; i != NumElts; ++i)
3679 Indices[i] = i;
3680
3681 // First extract half of each vector. This gives better codegen than
3682 // doing it in a single shuffle.
3683 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3684 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3685 // Concat the vectors.
3686 // NOTE: Operands have to be swapped to match intrinsic definition.
3687 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3688 Rep = Builder.CreateBitCast(Rep, CI->getType());
3689 } else if (Name == "avx512.kand.w") {
3690 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3691 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3692 Rep = Builder.CreateAnd(LHS, RHS);
3693 Rep = Builder.CreateBitCast(Rep, CI->getType());
3694 } else if (Name == "avx512.kandn.w") {
3695 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3696 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3697 LHS = Builder.CreateNot(LHS);
3698 Rep = Builder.CreateAnd(LHS, RHS);
3699 Rep = Builder.CreateBitCast(Rep, CI->getType());
3700 } else if (Name == "avx512.kor.w") {
3701 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3702 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3703 Rep = Builder.CreateOr(LHS, RHS);
3704 Rep = Builder.CreateBitCast(Rep, CI->getType());
3705 } else if (Name == "avx512.kxor.w") {
3706 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3707 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3708 Rep = Builder.CreateXor(LHS, RHS);
3709 Rep = Builder.CreateBitCast(Rep, CI->getType());
3710 } else if (Name == "avx512.kxnor.w") {
3711 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3712 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3713 LHS = Builder.CreateNot(LHS);
3714 Rep = Builder.CreateXor(LHS, RHS);
3715 Rep = Builder.CreateBitCast(Rep, CI->getType());
3716 } else if (Name == "avx512.knot.w") {
3717 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3718 Rep = Builder.CreateNot(Rep);
3719 Rep = Builder.CreateBitCast(Rep, CI->getType());
3720 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3721 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3722 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3723 Rep = Builder.CreateOr(LHS, RHS);
3724 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3725 Value *C;
3726 if (Name[14] == 'c')
3727 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3728 else
3729 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3730 Rep = Builder.CreateICmpEQ(Rep, C);
3731 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3732 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3733 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3734 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3735 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3736 Type *I32Ty = Type::getInt32Ty(C);
3737 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3738 ConstantInt::get(I32Ty, 0));
3739 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3740 ConstantInt::get(I32Ty, 0));
3741 Value *EltOp;
3742 if (Name.contains(".add."))
3743 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3744 else if (Name.contains(".sub."))
3745 EltOp = Builder.CreateFSub(Elt0, Elt1);
3746 else if (Name.contains(".mul."))
3747 EltOp = Builder.CreateFMul(Elt0, Elt1);
3748 else
3749 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3750 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3751 ConstantInt::get(I32Ty, 0));
3752 } else if (Name.starts_with("avx512.mask.pcmp")) {
3753 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3754 bool CmpEq = Name[16] == 'e';
3755 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3756 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3757 Type *OpTy = CI->getArgOperand(0)->getType();
3758 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3759 Intrinsic::ID IID;
3760 switch (VecWidth) {
3761 default:
3762 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3763 break;
3764 case 128:
3765 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3766 break;
3767 case 256:
3768 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3769 break;
3770 case 512:
3771 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3772 break;
3773 }
3774
3775 Rep =
3776 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3777 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3778 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3779 Type *OpTy = CI->getArgOperand(0)->getType();
3780 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3781 unsigned EltWidth = OpTy->getScalarSizeInBits();
3782 Intrinsic::ID IID;
3783 if (VecWidth == 128 && EltWidth == 32)
3784 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3785 else if (VecWidth == 256 && EltWidth == 32)
3786 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3787 else if (VecWidth == 512 && EltWidth == 32)
3788 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3789 else if (VecWidth == 128 && EltWidth == 64)
3790 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3791 else if (VecWidth == 256 && EltWidth == 64)
3792 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3793 else if (VecWidth == 512 && EltWidth == 64)
3794 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3795 else
3796 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3797
3798 Rep =
3799 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3800 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3801 } else if (Name.starts_with("avx512.cmp.p")) {
3802 SmallVector<Value *, 4> Args(CI->args());
3803 Type *OpTy = Args[0]->getType();
3804 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3805 unsigned EltWidth = OpTy->getScalarSizeInBits();
3806 Intrinsic::ID IID;
3807 if (VecWidth == 128 && EltWidth == 32)
3808 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3809 else if (VecWidth == 256 && EltWidth == 32)
3810 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3811 else if (VecWidth == 512 && EltWidth == 32)
3812 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3813 else if (VecWidth == 128 && EltWidth == 64)
3814 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3815 else if (VecWidth == 256 && EltWidth == 64)
3816 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3817 else if (VecWidth == 512 && EltWidth == 64)
3818 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3819 else
3820 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3821
3823 if (VecWidth == 512)
3824 std::swap(Mask, Args.back());
3825 Args.push_back(Mask);
3826
3827 Rep = Builder.CreateIntrinsic(IID, Args);
3828 } else if (Name.starts_with("avx512.mask.cmp.")) {
3829 // Integer compare intrinsics.
3830 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3831 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3832 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3833 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3834 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3835 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3836 Name.starts_with("avx512.cvtw2mask.") ||
3837 Name.starts_with("avx512.cvtd2mask.") ||
3838 Name.starts_with("avx512.cvtq2mask.")) {
3839 Value *Op = CI->getArgOperand(0);
3841 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3842 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3843 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3844 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3845 Name.starts_with("avx512.mask.pabs")) {
3846 Rep = upgradeAbs(Builder, *CI);
3847 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3848 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3849 Name.starts_with("avx512.mask.pmaxs")) {
3850 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3851 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3852 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3853 Name.starts_with("avx512.mask.pmaxu")) {
3854 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3855 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3856 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3857 Name.starts_with("avx512.mask.pmins")) {
3858 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3859 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3860 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3861 Name.starts_with("avx512.mask.pminu")) {
3862 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3863 } else if (Name == "sse2.pmulh.w" || Name.starts_with("avx2.pmulh.w") ||
3864 Name.starts_with("avx512.pmulh.w")) {
3865 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smulh);
3866 } else if (Name == "sse2.pmulhu.w" || Name.starts_with("avx2.pmulhu.w") ||
3867 Name.starts_with("avx512.pmulhu.w")) {
3868 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umulh);
3869 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3870 Name == "avx512.pmulu.dq.512" ||
3871 Name.starts_with("avx512.mask.pmulu.dq.")) {
3872 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3873 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3874 Name == "avx512.pmul.dq.512" ||
3875 Name.starts_with("avx512.mask.pmul.dq.")) {
3876 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3877 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3878 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3879 Rep =
3880 Builder.CreateSIToFP(CI->getArgOperand(1),
3881 cast<VectorType>(CI->getType())->getElementType());
3882 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3883 } else if (Name == "avx512.cvtusi2sd") {
3884 Rep =
3885 Builder.CreateUIToFP(CI->getArgOperand(1),
3886 cast<VectorType>(CI->getType())->getElementType());
3887 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3888 } else if (Name == "sse2.cvtss2sd") {
3889 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3890 Rep = Builder.CreateFPExt(
3891 Rep, cast<VectorType>(CI->getType())->getElementType());
3892 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3893 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3894 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3895 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3896 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3897 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3898 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3899 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3900 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3901 Name == "avx512.mask.cvtqq2ps.256" ||
3902 Name == "avx512.mask.cvtqq2ps.512" ||
3903 Name == "avx512.mask.cvtuqq2ps.256" ||
3904 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3905 Name == "avx.cvt.ps2.pd.256" ||
3906 Name == "avx512.mask.cvtps2pd.128" ||
3907 Name == "avx512.mask.cvtps2pd.256") {
3908 auto *DstTy = cast<FixedVectorType>(CI->getType());
3909 Rep = CI->getArgOperand(0);
3910 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3911
3912 unsigned NumDstElts = DstTy->getNumElements();
3913 if (NumDstElts < SrcTy->getNumElements()) {
3914 assert(NumDstElts == 2 && "Unexpected vector size");
3915 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3916 }
3917
3918 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3919 bool IsUnsigned = Name.contains("cvtu");
3920 if (IsPS2PD)
3921 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3922 else if (CI->arg_size() == 4 &&
3923 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3924 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3925 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3926 : Intrinsic::x86_avx512_sitofp_round;
3927 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3928 {Rep, CI->getArgOperand(3)});
3929 } else {
3930 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3931 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3932 }
3933
3934 if (CI->arg_size() >= 3)
3935 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3936 CI->getArgOperand(1));
3937 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3938 Name.starts_with("vcvtph2ps.")) {
3939 auto *DstTy = cast<FixedVectorType>(CI->getType());
3940 Rep = CI->getArgOperand(0);
3941 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3942 unsigned NumDstElts = DstTy->getNumElements();
3943 if (NumDstElts != SrcTy->getNumElements()) {
3944 assert(NumDstElts == 4 && "Unexpected vector size");
3945 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3946 }
3947 Rep = Builder.CreateBitCast(
3948 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3949 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3950 if (CI->arg_size() >= 3)
3951 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3952 CI->getArgOperand(1));
3953 } else if (Name.starts_with("avx512.mask.load")) {
3954 // "avx512.mask.loadu." or "avx512.mask.load."
3955 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3956 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3957 CI->getArgOperand(2), Aligned);
3958 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3959 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3960 auto *PtrTy = CI->getOperand(0)->getType();
3961 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3962 ResultTy->getNumElements());
3963 Rep = Builder.CreateIntrinsic(
3964 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3965 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3966 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3967 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3968 auto *PtrTy = CI->getArgOperand(0)->getType();
3969 Value *MaskVec =
3970 getX86MaskVec(Builder, CI->getArgOperand(2),
3971 cast<FixedVectorType>(ResultTy)->getNumElements());
3972 Rep = Builder.CreateIntrinsic(
3973 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3974 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3975 } else if (Name.starts_with("avx512.mask.compress.") ||
3976 Name.starts_with("avx512.mask.expand.")) {
3977 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3978
3979 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3980 ResultTy->getNumElements());
3981
3982 bool IsCompress = Name[12] == 'c';
3983 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3984 : Intrinsic::x86_avx512_mask_expand;
3985 Rep = Builder.CreateIntrinsic(
3986 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3987 } else if (Name.starts_with("xop.vpcom")) {
3988 bool IsSigned;
3989 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3990 Name.ends_with("uq"))
3991 IsSigned = false;
3992 else if (Name.ends_with("b") || Name.ends_with("w") ||
3993 Name.ends_with("d") || Name.ends_with("q"))
3994 IsSigned = true;
3995 else
3996 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3997
3998 unsigned Imm;
3999 if (CI->arg_size() == 3) {
4000 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4001 } else {
4002 Name = Name.substr(9); // strip off "xop.vpcom"
4003 if (Name.starts_with("lt"))
4004 Imm = 0;
4005 else if (Name.starts_with("le"))
4006 Imm = 1;
4007 else if (Name.starts_with("gt"))
4008 Imm = 2;
4009 else if (Name.starts_with("ge"))
4010 Imm = 3;
4011 else if (Name.starts_with("eq"))
4012 Imm = 4;
4013 else if (Name.starts_with("ne"))
4014 Imm = 5;
4015 else if (Name.starts_with("false"))
4016 Imm = 6;
4017 else if (Name.starts_with("true"))
4018 Imm = 7;
4019 else
4020 llvm_unreachable("Unknown condition");
4021 }
4022
4023 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
4024 } else if (Name.starts_with("xop.vpcmov")) {
4025 Value *Sel = CI->getArgOperand(2);
4026 Value *NotSel = Builder.CreateNot(Sel);
4027 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
4028 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
4029 Rep = Builder.CreateOr(Sel0, Sel1);
4030 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
4031 Name.starts_with("avx512.mask.prol")) {
4032 Rep = upgradeX86Rotate(Builder, *CI, false);
4033 } else if (Name.starts_with("avx512.pror") ||
4034 Name.starts_with("avx512.mask.pror")) {
4035 Rep = upgradeX86Rotate(Builder, *CI, true);
4036 } else if (Name.starts_with("avx512.vpshld.") ||
4037 Name.starts_with("avx512.mask.vpshld") ||
4038 Name.starts_with("avx512.maskz.vpshld")) {
4039 bool ZeroMask = Name[11] == 'z';
4040 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
4041 } else if (Name.starts_with("avx512.vpshrd.") ||
4042 Name.starts_with("avx512.mask.vpshrd") ||
4043 Name.starts_with("avx512.maskz.vpshrd")) {
4044 bool ZeroMask = Name[11] == 'z';
4045 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
4046 } else if (Name == "sse42.crc32.64.8") {
4047 Value *Trunc0 =
4048 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
4049 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
4050 {Trunc0, CI->getArgOperand(1)});
4051 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
4052 } else if (Name.starts_with("avx.vbroadcast.s") ||
4053 Name.starts_with("avx512.vbroadcast.s")) {
4054 // Replace broadcasts with a series of insertelements.
4055 auto *VecTy = cast<FixedVectorType>(CI->getType());
4056 Type *EltTy = VecTy->getElementType();
4057 unsigned EltNum = VecTy->getNumElements();
4058 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
4059 Type *I32Ty = Type::getInt32Ty(C);
4060 Rep = PoisonValue::get(VecTy);
4061 for (unsigned I = 0; I < EltNum; ++I)
4062 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
4063 } else if (Name.starts_with("sse41.pmovsx") ||
4064 Name.starts_with("sse41.pmovzx") ||
4065 Name.starts_with("avx2.pmovsx") ||
4066 Name.starts_with("avx2.pmovzx") ||
4067 Name.starts_with("avx512.mask.pmovsx") ||
4068 Name.starts_with("avx512.mask.pmovzx")) {
4069 auto *DstTy = cast<FixedVectorType>(CI->getType());
4070 unsigned NumDstElts = DstTy->getNumElements();
4071
4072 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
4073 SmallVector<int, 8> ShuffleMask(NumDstElts);
4074 for (unsigned i = 0; i != NumDstElts; ++i)
4075 ShuffleMask[i] = i;
4076
4077 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
4078
4079 bool DoSext = Name.contains("pmovsx");
4080 Rep =
4081 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
4082 // If there are 3 arguments, it's a masked intrinsic so we need a select.
4083 if (CI->arg_size() == 3)
4084 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4085 CI->getArgOperand(1));
4086 } else if (Name == "avx512.mask.pmov.qd.256" ||
4087 Name == "avx512.mask.pmov.qd.512" ||
4088 Name == "avx512.mask.pmov.wb.256" ||
4089 Name == "avx512.mask.pmov.wb.512") {
4090 Type *Ty = CI->getArgOperand(1)->getType();
4091 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
4092 Rep =
4093 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4094 } else if (Name.starts_with("avx.vbroadcastf128") ||
4095 Name == "avx2.vbroadcasti128") {
4096 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4097 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
4098 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4099 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
4100 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
4101 if (NumSrcElts == 2)
4102 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
4103 else
4104 Rep = Builder.CreateShuffleVector(Load,
4105 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4106 } else if (Name.starts_with("avx512.mask.shuf.i") ||
4107 Name.starts_with("avx512.mask.shuf.f")) {
4108 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4109 Type *VT = CI->getType();
4110 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4111 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4112 unsigned ControlBitsMask = NumLanes - 1;
4113 unsigned NumControlBits = NumLanes / 2;
4114 SmallVector<int, 8> ShuffleMask(0);
4115
4116 for (unsigned l = 0; l != NumLanes; ++l) {
4117 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4118 // We actually need the other source.
4119 if (l >= NumLanes / 2)
4120 LaneMask += NumLanes;
4121 for (unsigned i = 0; i != NumElementsInLane; ++i)
4122 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4123 }
4124 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4125 CI->getArgOperand(1), ShuffleMask);
4126 Rep =
4127 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4128 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4129 Name.starts_with("avx512.mask.broadcasti")) {
4130 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4131 ->getNumElements();
4132 unsigned NumDstElts =
4133 cast<FixedVectorType>(CI->getType())->getNumElements();
4134
4135 SmallVector<int, 8> ShuffleMask(NumDstElts);
4136 for (unsigned i = 0; i != NumDstElts; ++i)
4137 ShuffleMask[i] = i % NumSrcElts;
4138
4139 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4140 CI->getArgOperand(0), ShuffleMask);
4141 Rep =
4142 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4143 } else if (Name.starts_with("avx2.pbroadcast") ||
4144 Name.starts_with("avx2.vbroadcast") ||
4145 Name.starts_with("avx512.pbroadcast") ||
4146 Name.starts_with("avx512.mask.broadcast.s")) {
4147 // Replace vp?broadcasts with a vector shuffle.
4148 Value *Op = CI->getArgOperand(0);
4149 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4150 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4153 Rep = Builder.CreateShuffleVector(Op, M);
4154
4155 if (CI->arg_size() == 3)
4156 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4157 CI->getArgOperand(1));
4158 } else if (Name.starts_with("sse2.padds.") ||
4159 Name.starts_with("avx2.padds.") ||
4160 Name.starts_with("avx512.padds.") ||
4161 Name.starts_with("avx512.mask.padds.")) {
4162 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4163 } else if (Name.starts_with("sse2.psubs.") ||
4164 Name.starts_with("avx2.psubs.") ||
4165 Name.starts_with("avx512.psubs.") ||
4166 Name.starts_with("avx512.mask.psubs.")) {
4167 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4168 } else if (Name.starts_with("sse2.paddus.") ||
4169 Name.starts_with("avx2.paddus.") ||
4170 Name.starts_with("avx512.mask.paddus.")) {
4171 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4172 } else if (Name.starts_with("sse2.psubus.") ||
4173 Name.starts_with("avx2.psubus.") ||
4174 Name.starts_with("avx512.mask.psubus.")) {
4175 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4176 } else if (Name.starts_with("avx512.mask.palignr.")) {
4177 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4178 CI->getArgOperand(1), CI->getArgOperand(2),
4179 CI->getArgOperand(3), CI->getArgOperand(4),
4180 false);
4181 } else if (Name.starts_with("avx512.mask.valign.")) {
4183 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4184 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4185 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4186 // 128/256-bit shift left specified in bits.
4187 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4188 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4189 Shift / 8); // Shift is in bits.
4190 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4191 // 128/256-bit shift right specified in bits.
4192 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4193 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4194 Shift / 8); // Shift is in bits.
4195 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4196 Name == "avx512.psll.dq.512") {
4197 // 128/256/512-bit shift left specified in bytes.
4198 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4199 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4200 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4201 Name == "avx512.psrl.dq.512") {
4202 // 128/256/512-bit shift right specified in bytes.
4203 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4204 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4205 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4206 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4207 Name.starts_with("avx2.pblendd.")) {
4208 Value *Op0 = CI->getArgOperand(0);
4209 Value *Op1 = CI->getArgOperand(1);
4210 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4211 auto *VecTy = cast<FixedVectorType>(CI->getType());
4212 unsigned NumElts = VecTy->getNumElements();
4213
4214 SmallVector<int, 16> Idxs(NumElts);
4215 for (unsigned i = 0; i != NumElts; ++i)
4216 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4217
4218 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4219 } else if (Name.starts_with("avx.vinsertf128.") ||
4220 Name == "avx2.vinserti128" ||
4221 Name.starts_with("avx512.mask.insert")) {
4222 Value *Op0 = CI->getArgOperand(0);
4223 Value *Op1 = CI->getArgOperand(1);
4224 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4225 unsigned DstNumElts =
4226 cast<FixedVectorType>(CI->getType())->getNumElements();
4227 unsigned SrcNumElts =
4228 cast<FixedVectorType>(Op1->getType())->getNumElements();
4229 unsigned Scale = DstNumElts / SrcNumElts;
4230
4231 // Mask off the high bits of the immediate value; hardware ignores those.
4232 Imm = Imm % Scale;
4233
4234 // Extend the second operand into a vector the size of the destination.
4235 SmallVector<int, 8> Idxs(DstNumElts);
4236 for (unsigned i = 0; i != SrcNumElts; ++i)
4237 Idxs[i] = i;
4238 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4239 Idxs[i] = SrcNumElts;
4240 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4241
4242 // Insert the second operand into the first operand.
4243
4244 // Note that there is no guarantee that instruction lowering will actually
4245 // produce a vinsertf128 instruction for the created shuffles. In
4246 // particular, the 0 immediate case involves no lane changes, so it can
4247 // be handled as a blend.
4248
4249 // Example of shuffle mask for 32-bit elements:
4250 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4251 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4252
4253 // First fill with identify mask.
4254 for (unsigned i = 0; i != DstNumElts; ++i)
4255 Idxs[i] = i;
4256 // Then replace the elements where we need to insert.
4257 for (unsigned i = 0; i != SrcNumElts; ++i)
4258 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4259 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4260
4261 // If the intrinsic has a mask operand, handle that.
4262 if (CI->arg_size() == 5)
4263 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4264 CI->getArgOperand(3));
4265 } else if (Name.starts_with("avx.vextractf128.") ||
4266 Name == "avx2.vextracti128" ||
4267 Name.starts_with("avx512.mask.vextract")) {
4268 Value *Op0 = CI->getArgOperand(0);
4269 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4270 unsigned DstNumElts =
4271 cast<FixedVectorType>(CI->getType())->getNumElements();
4272 unsigned SrcNumElts =
4273 cast<FixedVectorType>(Op0->getType())->getNumElements();
4274 unsigned Scale = SrcNumElts / DstNumElts;
4275
4276 // Mask off the high bits of the immediate value; hardware ignores those.
4277 Imm = Imm % Scale;
4278
4279 // Get indexes for the subvector of the input vector.
4280 SmallVector<int, 8> Idxs(DstNumElts);
4281 for (unsigned i = 0; i != DstNumElts; ++i) {
4282 Idxs[i] = i + (Imm * DstNumElts);
4283 }
4284 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4285
4286 // If the intrinsic has a mask operand, handle that.
4287 if (CI->arg_size() == 4)
4288 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4289 CI->getArgOperand(2));
4290 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4291 Name.starts_with("avx512.mask.perm.di.")) {
4292 Value *Op0 = CI->getArgOperand(0);
4293 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4294 auto *VecTy = cast<FixedVectorType>(CI->getType());
4295 unsigned NumElts = VecTy->getNumElements();
4296
4297 SmallVector<int, 8> Idxs(NumElts);
4298 for (unsigned i = 0; i != NumElts; ++i)
4299 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4300
4301 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4302
4303 if (CI->arg_size() == 4)
4304 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4305 CI->getArgOperand(2));
4306 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4307 // The immediate permute control byte looks like this:
4308 // [1:0] - select 128 bits from sources for low half of destination
4309 // [2] - ignore
4310 // [3] - zero low half of destination
4311 // [5:4] - select 128 bits from sources for high half of destination
4312 // [6] - ignore
4313 // [7] - zero high half of destination
4314
4315 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4316
4317 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4318 unsigned HalfSize = NumElts / 2;
4319 SmallVector<int, 8> ShuffleMask(NumElts);
4320
4321 // Determine which operand(s) are actually in use for this instruction.
4322 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4323 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4324
4325 // If needed, replace operands based on zero mask.
4326 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4327 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4328
4329 // Permute low half of result.
4330 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4331 for (unsigned i = 0; i < HalfSize; ++i)
4332 ShuffleMask[i] = StartIndex + i;
4333
4334 // Permute high half of result.
4335 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4336 for (unsigned i = 0; i < HalfSize; ++i)
4337 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4338
4339 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4340
4341 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4342 Name.starts_with("avx512.mask.vpermil.p") ||
4343 Name.starts_with("avx512.mask.pshuf.d.")) {
4344 Value *Op0 = CI->getArgOperand(0);
4345 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4346 auto *VecTy = cast<FixedVectorType>(CI->getType());
4347 unsigned NumElts = VecTy->getNumElements();
4348 // Calculate the size of each index in the immediate.
4349 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4350 unsigned IdxMask = ((1 << IdxSize) - 1);
4351
4352 SmallVector<int, 8> Idxs(NumElts);
4353 // Lookup the bits for this element, wrapping around the immediate every
4354 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4355 // to offset by the first index of each group.
4356 for (unsigned i = 0; i != NumElts; ++i)
4357 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4358
4359 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4360
4361 if (CI->arg_size() == 4)
4362 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4363 CI->getArgOperand(2));
4364 } else if (Name == "sse2.pshufl.w" ||
4365 Name.starts_with("avx512.mask.pshufl.w.")) {
4366 Value *Op0 = CI->getArgOperand(0);
4367 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4368 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4369
4370 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4371 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4372
4373 SmallVector<int, 16> Idxs(NumElts);
4374 for (unsigned l = 0; l != NumElts; l += 8) {
4375 for (unsigned i = 0; i != 4; ++i)
4376 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4377 for (unsigned i = 4; i != 8; ++i)
4378 Idxs[i + l] = i + l;
4379 }
4380
4381 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4382
4383 if (CI->arg_size() == 4)
4384 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4385 CI->getArgOperand(2));
4386 } else if (Name == "sse2.pshufh.w" ||
4387 Name.starts_with("avx512.mask.pshufh.w.")) {
4388 Value *Op0 = CI->getArgOperand(0);
4389 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4390 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4391
4392 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4393 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4394
4395 SmallVector<int, 16> Idxs(NumElts);
4396 for (unsigned l = 0; l != NumElts; l += 8) {
4397 for (unsigned i = 0; i != 4; ++i)
4398 Idxs[i + l] = i + l;
4399 for (unsigned i = 0; i != 4; ++i)
4400 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4401 }
4402
4403 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4404
4405 if (CI->arg_size() == 4)
4406 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4407 CI->getArgOperand(2));
4408 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4409 Value *Op0 = CI->getArgOperand(0);
4410 Value *Op1 = CI->getArgOperand(1);
4411 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4412 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4413
4414 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4415 unsigned HalfLaneElts = NumLaneElts / 2;
4416
4417 SmallVector<int, 16> Idxs(NumElts);
4418 for (unsigned i = 0; i != NumElts; ++i) {
4419 // Base index is the starting element of the lane.
4420 Idxs[i] = i - (i % NumLaneElts);
4421 // If we are half way through the lane switch to the other source.
4422 if ((i % NumLaneElts) >= HalfLaneElts)
4423 Idxs[i] += NumElts;
4424 // Now select the specific element. By adding HalfLaneElts bits from
4425 // the immediate. Wrapping around the immediate every 8-bits.
4426 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4427 }
4428
4429 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4430
4431 Rep =
4432 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4433 } else if (Name.starts_with("avx512.mask.movddup") ||
4434 Name.starts_with("avx512.mask.movshdup") ||
4435 Name.starts_with("avx512.mask.movsldup")) {
4436 Value *Op0 = CI->getArgOperand(0);
4437 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4438 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4439
4440 unsigned Offset = 0;
4441 if (Name.starts_with("avx512.mask.movshdup."))
4442 Offset = 1;
4443
4444 SmallVector<int, 16> Idxs(NumElts);
4445 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4446 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4447 Idxs[i + l + 0] = i + l + Offset;
4448 Idxs[i + l + 1] = i + l + Offset;
4449 }
4450
4451 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4452
4453 Rep =
4454 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4455 } else if (Name.starts_with("avx512.mask.punpckl") ||
4456 Name.starts_with("avx512.mask.unpckl.")) {
4457 Value *Op0 = CI->getArgOperand(0);
4458 Value *Op1 = CI->getArgOperand(1);
4459 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4460 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4461
4462 SmallVector<int, 64> Idxs(NumElts);
4463 for (int l = 0; l != NumElts; l += NumLaneElts)
4464 for (int i = 0; i != NumLaneElts; ++i)
4465 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4466
4467 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4468
4469 Rep =
4470 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4471 } else if (Name.starts_with("avx512.mask.punpckh") ||
4472 Name.starts_with("avx512.mask.unpckh.")) {
4473 Value *Op0 = CI->getArgOperand(0);
4474 Value *Op1 = CI->getArgOperand(1);
4475 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4476 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4477
4478 SmallVector<int, 64> Idxs(NumElts);
4479 for (int l = 0; l != NumElts; l += NumLaneElts)
4480 for (int i = 0; i != NumLaneElts; ++i)
4481 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4482
4483 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4484
4485 Rep =
4486 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4487 } else if (Name.starts_with("avx512.mask.and.") ||
4488 Name.starts_with("avx512.mask.pand.")) {
4489 VectorType *FTy = cast<VectorType>(CI->getType());
4491 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4492 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4493 Rep = Builder.CreateBitCast(Rep, FTy);
4494 Rep =
4495 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4496 } else if (Name.starts_with("avx512.mask.andn.") ||
4497 Name.starts_with("avx512.mask.pandn.")) {
4498 VectorType *FTy = cast<VectorType>(CI->getType());
4500 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4501 Rep = Builder.CreateAnd(Rep,
4502 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4503 Rep = Builder.CreateBitCast(Rep, FTy);
4504 Rep =
4505 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4506 } else if (Name.starts_with("avx512.mask.or.") ||
4507 Name.starts_with("avx512.mask.por.")) {
4508 VectorType *FTy = cast<VectorType>(CI->getType());
4510 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4511 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4512 Rep = Builder.CreateBitCast(Rep, FTy);
4513 Rep =
4514 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4515 } else if (Name.starts_with("avx512.mask.xor.") ||
4516 Name.starts_with("avx512.mask.pxor.")) {
4517 VectorType *FTy = cast<VectorType>(CI->getType());
4519 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4520 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4521 Rep = Builder.CreateBitCast(Rep, FTy);
4522 Rep =
4523 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4524 } else if (Name.starts_with("avx512.mask.padd.")) {
4525 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4526 Rep =
4527 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4528 } else if (Name.starts_with("avx512.mask.psub.")) {
4529 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4530 Rep =
4531 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4532 } else if (Name.starts_with("avx512.mask.pmull.")) {
4533 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4534 Rep =
4535 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4536 } else if (Name.starts_with("avx512.mask.add.p")) {
4537 if (Name.ends_with(".512")) {
4538 Intrinsic::ID IID;
4539 if (Name[17] == 's')
4540 IID = Intrinsic::x86_avx512_add_ps_512;
4541 else
4542 IID = Intrinsic::x86_avx512_add_pd_512;
4543
4544 Rep = Builder.CreateIntrinsic(
4545 IID,
4546 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4547 } else {
4548 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4549 }
4550 Rep =
4551 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4552 } else if (Name.starts_with("avx512.mask.div.p")) {
4553 if (Name.ends_with(".512")) {
4554 Intrinsic::ID IID;
4555 if (Name[17] == 's')
4556 IID = Intrinsic::x86_avx512_div_ps_512;
4557 else
4558 IID = Intrinsic::x86_avx512_div_pd_512;
4559
4560 Rep = Builder.CreateIntrinsic(
4561 IID,
4562 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4563 } else {
4564 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4565 }
4566 Rep =
4567 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4568 } else if (Name.starts_with("avx512.mask.mul.p")) {
4569 if (Name.ends_with(".512")) {
4570 Intrinsic::ID IID;
4571 if (Name[17] == 's')
4572 IID = Intrinsic::x86_avx512_mul_ps_512;
4573 else
4574 IID = Intrinsic::x86_avx512_mul_pd_512;
4575
4576 Rep = Builder.CreateIntrinsic(
4577 IID,
4578 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4579 } else {
4580 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4581 }
4582 Rep =
4583 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4584 } else if (Name.starts_with("avx512.mask.sub.p")) {
4585 if (Name.ends_with(".512")) {
4586 Intrinsic::ID IID;
4587 if (Name[17] == 's')
4588 IID = Intrinsic::x86_avx512_sub_ps_512;
4589 else
4590 IID = Intrinsic::x86_avx512_sub_pd_512;
4591
4592 Rep = Builder.CreateIntrinsic(
4593 IID,
4594 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4595 } else {
4596 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4597 }
4598 Rep =
4599 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4600 } else if ((Name.starts_with("avx512.mask.max.p") ||
4601 Name.starts_with("avx512.mask.min.p")) &&
4602 Name.drop_front(18) == ".512") {
4603 bool IsDouble = Name[17] == 'd';
4604 bool IsMin = Name[13] == 'i';
4605 static const Intrinsic::ID MinMaxTbl[2][2] = {
4606 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4607 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4608 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4609
4610 Rep = Builder.CreateIntrinsic(
4611 IID,
4612 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4613 Rep =
4614 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4615 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4616 Rep =
4617 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4618 {CI->getArgOperand(0), Builder.getInt1(false)});
4619 Rep =
4620 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4621 } else if (Name.starts_with("avx512.mask.psll")) {
4622 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4623 bool IsVariable = Name[16] == 'v';
4624 char Size = Name[16] == '.' ? Name[17]
4625 : Name[17] == '.' ? Name[18]
4626 : Name[18] == '.' ? Name[19]
4627 : Name[20];
4628
4629 Intrinsic::ID IID;
4630 if (IsVariable && Name[17] != '.') {
4631 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4632 IID = Intrinsic::x86_avx2_psllv_q;
4633 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4634 IID = Intrinsic::x86_avx2_psllv_q_256;
4635 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4636 IID = Intrinsic::x86_avx2_psllv_d;
4637 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4638 IID = Intrinsic::x86_avx2_psllv_d_256;
4639 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4640 IID = Intrinsic::x86_avx512_psllv_w_128;
4641 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4642 IID = Intrinsic::x86_avx512_psllv_w_256;
4643 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4644 IID = Intrinsic::x86_avx512_psllv_w_512;
4645 else
4646 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4647 } else if (Name.ends_with(".128")) {
4648 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4649 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4650 : Intrinsic::x86_sse2_psll_d;
4651 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4652 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4653 : Intrinsic::x86_sse2_psll_q;
4654 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4655 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4656 : Intrinsic::x86_sse2_psll_w;
4657 else
4658 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4659 } else if (Name.ends_with(".256")) {
4660 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4661 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4662 : Intrinsic::x86_avx2_psll_d;
4663 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4664 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4665 : Intrinsic::x86_avx2_psll_q;
4666 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4667 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4668 : Intrinsic::x86_avx2_psll_w;
4669 else
4670 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4671 } else {
4672 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4673 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4674 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4675 : Intrinsic::x86_avx512_psll_d_512;
4676 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4677 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4678 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4679 : Intrinsic::x86_avx512_psll_q_512;
4680 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4681 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4682 : Intrinsic::x86_avx512_psll_w_512;
4683 else
4684 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4685 }
4686
4687 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4688 } else if (Name.starts_with("avx512.mask.psrl")) {
4689 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4690 bool IsVariable = Name[16] == 'v';
4691 char Size = Name[16] == '.' ? Name[17]
4692 : Name[17] == '.' ? Name[18]
4693 : Name[18] == '.' ? Name[19]
4694 : Name[20];
4695
4696 Intrinsic::ID IID;
4697 if (IsVariable && Name[17] != '.') {
4698 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4699 IID = Intrinsic::x86_avx2_psrlv_q;
4700 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4701 IID = Intrinsic::x86_avx2_psrlv_q_256;
4702 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4703 IID = Intrinsic::x86_avx2_psrlv_d;
4704 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4705 IID = Intrinsic::x86_avx2_psrlv_d_256;
4706 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4707 IID = Intrinsic::x86_avx512_psrlv_w_128;
4708 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4709 IID = Intrinsic::x86_avx512_psrlv_w_256;
4710 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4711 IID = Intrinsic::x86_avx512_psrlv_w_512;
4712 else
4713 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4714 } else if (Name.ends_with(".128")) {
4715 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4716 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4717 : Intrinsic::x86_sse2_psrl_d;
4718 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4719 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4720 : Intrinsic::x86_sse2_psrl_q;
4721 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4722 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4723 : Intrinsic::x86_sse2_psrl_w;
4724 else
4725 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4726 } else if (Name.ends_with(".256")) {
4727 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4728 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4729 : Intrinsic::x86_avx2_psrl_d;
4730 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4731 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4732 : Intrinsic::x86_avx2_psrl_q;
4733 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4734 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4735 : Intrinsic::x86_avx2_psrl_w;
4736 else
4737 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4738 } else {
4739 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4740 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4741 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4742 : Intrinsic::x86_avx512_psrl_d_512;
4743 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4744 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4745 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4746 : Intrinsic::x86_avx512_psrl_q_512;
4747 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4748 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4749 : Intrinsic::x86_avx512_psrl_w_512;
4750 else
4751 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4752 }
4753
4754 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4755 } else if (Name.starts_with("avx512.mask.psra")) {
4756 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4757 bool IsVariable = Name[16] == 'v';
4758 char Size = Name[16] == '.' ? Name[17]
4759 : Name[17] == '.' ? Name[18]
4760 : Name[18] == '.' ? Name[19]
4761 : Name[20];
4762
4763 Intrinsic::ID IID;
4764 if (IsVariable && Name[17] != '.') {
4765 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4766 IID = Intrinsic::x86_avx2_psrav_d;
4767 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4768 IID = Intrinsic::x86_avx2_psrav_d_256;
4769 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4770 IID = Intrinsic::x86_avx512_psrav_w_128;
4771 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4772 IID = Intrinsic::x86_avx512_psrav_w_256;
4773 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4774 IID = Intrinsic::x86_avx512_psrav_w_512;
4775 else
4776 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4777 } else if (Name.ends_with(".128")) {
4778 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4779 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4780 : Intrinsic::x86_sse2_psra_d;
4781 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4782 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4783 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4784 : Intrinsic::x86_avx512_psra_q_128;
4785 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4786 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4787 : Intrinsic::x86_sse2_psra_w;
4788 else
4789 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4790 } else if (Name.ends_with(".256")) {
4791 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4792 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4793 : Intrinsic::x86_avx2_psra_d;
4794 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4795 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4796 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4797 : Intrinsic::x86_avx512_psra_q_256;
4798 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4799 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4800 : Intrinsic::x86_avx2_psra_w;
4801 else
4802 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4803 } else {
4804 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4805 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4806 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4807 : Intrinsic::x86_avx512_psra_d_512;
4808 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4809 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4810 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4811 : Intrinsic::x86_avx512_psra_q_512;
4812 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4813 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4814 : Intrinsic::x86_avx512_psra_w_512;
4815 else
4816 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4817 }
4818
4819 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4820 } else if (Name.starts_with("avx512.mask.move.s")) {
4821 Rep = upgradeMaskedMove(Builder, *CI);
4822 } else if (Name.starts_with("avx512.cvtmask2")) {
4823 Rep = upgradeMaskToInt(Builder, *CI);
4824 } else if (Name.ends_with(".movntdqa")) {
4826 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4827
4828 LoadInst *LI = Builder.CreateAlignedLoad(
4829 CI->getType(), CI->getArgOperand(0),
4831 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4832 Rep = LI;
4833 } else if (Name.starts_with("fma.vfmadd.") ||
4834 Name.starts_with("fma.vfmsub.") ||
4835 Name.starts_with("fma.vfnmadd.") ||
4836 Name.starts_with("fma.vfnmsub.")) {
4837 bool NegMul = Name[6] == 'n';
4838 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4839 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4840
4841 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4842 CI->getArgOperand(2)};
4843
4844 if (IsScalar) {
4845 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4846 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4847 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4848 }
4849
4850 if (NegMul && !IsScalar)
4851 Ops[0] = Builder.CreateFNeg(Ops[0]);
4852 if (NegMul && IsScalar)
4853 Ops[1] = Builder.CreateFNeg(Ops[1]);
4854 if (NegAcc)
4855 Ops[2] = Builder.CreateFNeg(Ops[2]);
4856
4857 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4858
4859 if (IsScalar)
4860 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4861 } else if (Name.starts_with("fma4.vfmadd.s")) {
4862 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4863 CI->getArgOperand(2)};
4864
4865 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4866 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4867 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4868
4869 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4870
4871 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4872 Rep, (uint64_t)0);
4873 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4874 Name.starts_with("avx512.maskz.vfmadd.s") ||
4875 Name.starts_with("avx512.mask3.vfmadd.s") ||
4876 Name.starts_with("avx512.mask3.vfmsub.s") ||
4877 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4878 bool IsMask3 = Name[11] == '3';
4879 bool IsMaskZ = Name[11] == 'z';
4880 // Drop the "avx512.mask." to make it easier.
4881 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4882 bool NegMul = Name[2] == 'n';
4883 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4884
4885 Value *A = CI->getArgOperand(0);
4886 Value *B = CI->getArgOperand(1);
4887 Value *C = CI->getArgOperand(2);
4888
4889 if (NegMul && (IsMask3 || IsMaskZ))
4890 A = Builder.CreateFNeg(A);
4891 if (NegMul && !(IsMask3 || IsMaskZ))
4892 B = Builder.CreateFNeg(B);
4893 if (NegAcc)
4894 C = Builder.CreateFNeg(C);
4895
4896 A = Builder.CreateExtractElement(A, (uint64_t)0);
4897 B = Builder.CreateExtractElement(B, (uint64_t)0);
4898 C = Builder.CreateExtractElement(C, (uint64_t)0);
4899
4900 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4901 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4902 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4903
4904 Intrinsic::ID IID;
4905 if (Name.back() == 'd')
4906 IID = Intrinsic::x86_avx512_vfmadd_f64;
4907 else
4908 IID = Intrinsic::x86_avx512_vfmadd_f32;
4909 Rep = Builder.CreateIntrinsic(IID, Ops);
4910 } else {
4911 Rep = Builder.CreateFMA(A, B, C);
4912 }
4913
4914 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4915 : IsMask3 ? C
4916 : A;
4917
4918 // For Mask3 with NegAcc, we need to create a new extractelement that
4919 // avoids the negation above.
4920 if (NegAcc && IsMask3)
4921 PassThru =
4922 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4923
4924 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4925 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4926 (uint64_t)0);
4927 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4928 Name.starts_with("avx512.mask.vfnmadd.p") ||
4929 Name.starts_with("avx512.mask.vfnmsub.p") ||
4930 Name.starts_with("avx512.mask3.vfmadd.p") ||
4931 Name.starts_with("avx512.mask3.vfmsub.p") ||
4932 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4933 Name.starts_with("avx512.maskz.vfmadd.p")) {
4934 bool IsMask3 = Name[11] == '3';
4935 bool IsMaskZ = Name[11] == 'z';
4936 // Drop the "avx512.mask." to make it easier.
4937 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4938 bool NegMul = Name[2] == 'n';
4939 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4940
4941 Value *A = CI->getArgOperand(0);
4942 Value *B = CI->getArgOperand(1);
4943 Value *C = CI->getArgOperand(2);
4944
4945 if (NegMul && (IsMask3 || IsMaskZ))
4946 A = Builder.CreateFNeg(A);
4947 if (NegMul && !(IsMask3 || IsMaskZ))
4948 B = Builder.CreateFNeg(B);
4949 if (NegAcc)
4950 C = Builder.CreateFNeg(C);
4951
4952 if (CI->arg_size() == 5 &&
4953 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4954 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4955 Intrinsic::ID IID;
4956 // Check the character before ".512" in string.
4957 if (Name[Name.size() - 5] == 's')
4958 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4959 else
4960 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4961
4962 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4963 } else {
4964 Rep = Builder.CreateFMA(A, B, C);
4965 }
4966
4967 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4968 : IsMask3 ? CI->getArgOperand(2)
4969 : CI->getArgOperand(0);
4970
4971 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4972 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4973 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4974 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4975 Intrinsic::ID IID;
4976 if (VecWidth == 128 && EltWidth == 32)
4977 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4978 else if (VecWidth == 256 && EltWidth == 32)
4979 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4980 else if (VecWidth == 128 && EltWidth == 64)
4981 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4982 else if (VecWidth == 256 && EltWidth == 64)
4983 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4984 else
4985 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4986
4987 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4988 CI->getArgOperand(2)};
4989 Ops[2] = Builder.CreateFNeg(Ops[2]);
4990 Rep = Builder.CreateIntrinsic(IID, Ops);
4991 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4992 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4993 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4994 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4995 bool IsMask3 = Name[11] == '3';
4996 bool IsMaskZ = Name[11] == 'z';
4997 // Drop the "avx512.mask." to make it easier.
4998 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4999 bool IsSubAdd = Name[3] == 's';
5000 if (CI->arg_size() == 5) {
5001 Intrinsic::ID IID;
5002 // Check the character before ".512" in string.
5003 if (Name[Name.size() - 5] == 's')
5004 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
5005 else
5006 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
5007
5008 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5009 CI->getArgOperand(2), CI->getArgOperand(4)};
5010 if (IsSubAdd)
5011 Ops[2] = Builder.CreateFNeg(Ops[2]);
5012
5013 Rep = Builder.CreateIntrinsic(IID, Ops);
5014 } else {
5015 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
5016
5017 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5018 CI->getArgOperand(2)};
5019
5021 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
5022 Value *Odd = Builder.CreateCall(FMA, Ops);
5023 Ops[2] = Builder.CreateFNeg(Ops[2]);
5024 Value *Even = Builder.CreateCall(FMA, Ops);
5025
5026 if (IsSubAdd)
5027 std::swap(Even, Odd);
5028
5029 SmallVector<int, 32> Idxs(NumElts);
5030 for (int i = 0; i != NumElts; ++i)
5031 Idxs[i] = i + (i % 2) * NumElts;
5032
5033 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
5034 }
5035
5036 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
5037 : IsMask3 ? CI->getArgOperand(2)
5038 : CI->getArgOperand(0);
5039
5040 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5041 } else if (Name.starts_with("avx512.mask.pternlog.") ||
5042 Name.starts_with("avx512.maskz.pternlog.")) {
5043 bool ZeroMask = Name[11] == 'z';
5044 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5045 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
5046 Intrinsic::ID IID;
5047 if (VecWidth == 128 && EltWidth == 32)
5048 IID = Intrinsic::x86_avx512_pternlog_d_128;
5049 else if (VecWidth == 256 && EltWidth == 32)
5050 IID = Intrinsic::x86_avx512_pternlog_d_256;
5051 else if (VecWidth == 512 && EltWidth == 32)
5052 IID = Intrinsic::x86_avx512_pternlog_d_512;
5053 else if (VecWidth == 128 && EltWidth == 64)
5054 IID = Intrinsic::x86_avx512_pternlog_q_128;
5055 else if (VecWidth == 256 && EltWidth == 64)
5056 IID = Intrinsic::x86_avx512_pternlog_q_256;
5057 else if (VecWidth == 512 && EltWidth == 64)
5058 IID = Intrinsic::x86_avx512_pternlog_q_512;
5059 else
5060 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5061
5062 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5063 CI->getArgOperand(2), CI->getArgOperand(3)};
5064 Rep = Builder.CreateIntrinsic(IID, Args);
5065 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5066 : CI->getArgOperand(0);
5067 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
5068 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
5069 Name.starts_with("avx512.maskz.vpmadd52")) {
5070 bool ZeroMask = Name[11] == 'z';
5071 bool High = Name[20] == 'h' || Name[21] == 'h';
5072 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5073 Intrinsic::ID IID;
5074 if (VecWidth == 128 && !High)
5075 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
5076 else if (VecWidth == 256 && !High)
5077 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
5078 else if (VecWidth == 512 && !High)
5079 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
5080 else if (VecWidth == 128 && High)
5081 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
5082 else if (VecWidth == 256 && High)
5083 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
5084 else if (VecWidth == 512 && High)
5085 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
5086 else
5087 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5088
5089 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5090 CI->getArgOperand(2)};
5091 Rep = Builder.CreateIntrinsic(IID, Args);
5092 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5093 : CI->getArgOperand(0);
5094 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5095 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
5096 Name.starts_with("avx512.mask.vpermt2var.") ||
5097 Name.starts_with("avx512.maskz.vpermt2var.")) {
5098 bool ZeroMask = Name[11] == 'z';
5099 bool IndexForm = Name[17] == 'i';
5100 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
5101 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
5102 Name.starts_with("avx512.maskz.vpdpbusd.") ||
5103 Name.starts_with("avx512.mask.vpdpbusds.") ||
5104 Name.starts_with("avx512.maskz.vpdpbusds.")) {
5105 bool ZeroMask = Name[11] == 'z';
5106 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5107 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5108 Intrinsic::ID IID;
5109 if (VecWidth == 128 && !IsSaturating)
5110 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5111 else if (VecWidth == 256 && !IsSaturating)
5112 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5113 else if (VecWidth == 512 && !IsSaturating)
5114 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5115 else if (VecWidth == 128 && IsSaturating)
5116 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5117 else if (VecWidth == 256 && IsSaturating)
5118 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5119 else if (VecWidth == 512 && IsSaturating)
5120 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5121 else
5122 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5123
5124 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5125 CI->getArgOperand(2)};
5126
5127 // Input arguments types were incorrectly set to vectors of i32 before but
5128 // they should be vectors of i8. Insert bit cast when encountering the old
5129 // types
5130 if (Args[1]->getType()->isVectorTy() &&
5131 cast<VectorType>(Args[1]->getType())
5132 ->getElementType()
5133 ->isIntegerTy(32) &&
5134 Args[2]->getType()->isVectorTy() &&
5135 cast<VectorType>(Args[2]->getType())
5136 ->getElementType()
5137 ->isIntegerTy(32)) {
5138 Type *NewArgType = nullptr;
5139 if (VecWidth == 128)
5140 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5141 else if (VecWidth == 256)
5142 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5143 else if (VecWidth == 512)
5144 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5145 else
5146 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5147 CI);
5148
5149 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5150 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5151 }
5152
5153 Rep = Builder.CreateIntrinsic(IID, Args);
5154 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5155 : CI->getArgOperand(0);
5156 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5157 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5158 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5159 Name.starts_with("avx512.mask.vpdpwssds.") ||
5160 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5161 bool ZeroMask = Name[11] == 'z';
5162 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5163 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5164 Intrinsic::ID IID;
5165 if (VecWidth == 128 && !IsSaturating)
5166 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5167 else if (VecWidth == 256 && !IsSaturating)
5168 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5169 else if (VecWidth == 512 && !IsSaturating)
5170 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5171 else if (VecWidth == 128 && IsSaturating)
5172 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5173 else if (VecWidth == 256 && IsSaturating)
5174 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5175 else if (VecWidth == 512 && IsSaturating)
5176 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5177 else
5178 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5179
5180 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5181 CI->getArgOperand(2)};
5182
5183 // Input arguments types were incorrectly set to vectors of i32 before but
5184 // they should be vectors of i16. Insert bit cast when encountering the old
5185 // types
5186 if (Args[1]->getType()->isVectorTy() &&
5187 cast<VectorType>(Args[1]->getType())
5188 ->getElementType()
5189 ->isIntegerTy(32) &&
5190 Args[2]->getType()->isVectorTy() &&
5191 cast<VectorType>(Args[2]->getType())
5192 ->getElementType()
5193 ->isIntegerTy(32)) {
5194 Type *NewArgType = nullptr;
5195 if (VecWidth == 128)
5196 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5197 else if (VecWidth == 256)
5198 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5199 else if (VecWidth == 512)
5200 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5201 else
5202 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5203 CI);
5204
5205 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5206 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5207 }
5208
5209 Rep = Builder.CreateIntrinsic(IID, Args);
5210 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5211 : CI->getArgOperand(0);
5212 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5213 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5214 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5215 Name == "subborrow.u32" || Name == "subborrow.u64") {
5216 Intrinsic::ID IID;
5217 if (Name[0] == 'a' && Name.back() == '2')
5218 IID = Intrinsic::x86_addcarry_32;
5219 else if (Name[0] == 'a' && Name.back() == '4')
5220 IID = Intrinsic::x86_addcarry_64;
5221 else if (Name[0] == 's' && Name.back() == '2')
5222 IID = Intrinsic::x86_subborrow_32;
5223 else if (Name[0] == 's' && Name.back() == '4')
5224 IID = Intrinsic::x86_subborrow_64;
5225 else
5226 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5227
5228 // Make a call with 3 operands.
5229 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5230 CI->getArgOperand(2)};
5231 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5232
5233 // Extract the second result and store it.
5234 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5235 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5236 // Replace the original call result with the first result of the new call.
5237 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5238
5239 CI->replaceAllUsesWith(CF);
5240 Rep = nullptr;
5241 } else if (Name.starts_with("avx512.mask.") &&
5242 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5243 // Rep will be updated by the call in the condition.
5244 } else if (Name.starts_with("bmi.pdep.")) {
5245 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5246 } else if (Name.starts_with("bmi.pext.")) {
5247 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5248 } else
5249 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5250
5251 return Rep;
5252}
5253
5255 Function *F, IRBuilder<> &Builder) {
5256 if (Name.starts_with("neon.bfcvt")) {
5257 if (Name.starts_with("neon.bfcvtn2")) {
5258 SmallVector<int, 32> LoMask(4);
5259 std::iota(LoMask.begin(), LoMask.end(), 0);
5260 SmallVector<int, 32> ConcatMask(8);
5261 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5262 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5263 Value *Trunc =
5264 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5265 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5266 } else if (Name.starts_with("neon.bfcvtn")) {
5267 SmallVector<int, 32> ConcatMask(8);
5268 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5269 Type *V4BF16 =
5270 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5271 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5272 dbgs() << "Trunc: " << *Trunc << "\n";
5273 return Builder.CreateShuffleVector(
5274 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5275 } else {
5276 return Builder.CreateFPTrunc(CI->getOperand(0),
5277 Type::getBFloatTy(F->getContext()));
5278 }
5279 } else if (Name.starts_with("sve.fcvt")) {
5280 Intrinsic::ID NewID =
5282 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5283 .Case("sve.fcvtnt.bf16f32",
5284 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5286 if (NewID == Intrinsic::not_intrinsic)
5287 llvm_unreachable("Unhandled Intrinsic!");
5288
5289 SmallVector<Value *, 3> Args(CI->args());
5290
5291 // The original intrinsics incorrectly used a predicate based on the
5292 // smallest element type rather than the largest.
5293 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5294 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5295
5296 if (Args[1]->getType() != BadPredTy)
5297 llvm_unreachable("Unexpected predicate type!");
5298
5299 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5300 BadPredTy, Args[1]);
5301 Args[1] = Builder.CreateIntrinsic(
5302 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5303
5304 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5305 CI->getName());
5306 }
5307
5308 if (Name == "neon.vcvtfp2hf")
5309 return Builder.CreateBitCast(
5310 Builder.CreateFPTrunc(
5311 CI->getOperand(0),
5312 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5313 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5314 if (Name == "neon.vcvthf2fp")
5315 return Builder.CreateFPExt(
5316 Builder.CreateBitCast(
5317 CI->getOperand(0),
5318 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5319 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5320
5321 llvm_unreachable("Unhandled Intrinsic!");
5322}
5323
5325 IRBuilder<> &Builder) {
5326 if (Name == "mve.vctp64.old") {
5327 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5328 // correct type.
5329 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5330 CI->getArgOperand(0),
5331 /*FMFSource=*/nullptr, CI->getName());
5332 Value *C1 = Builder.CreateIntrinsic(
5333 Intrinsic::arm_mve_pred_v2i,
5334 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5335 return Builder.CreateIntrinsic(
5336 Intrinsic::arm_mve_pred_i2v,
5337 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5338 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5339 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5340 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5341 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5342 Name ==
5343 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5344 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5345 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5346 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5347 Name ==
5348 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5349 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5350 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5351 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5352 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5353 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5354 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5355 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5356 std::vector<Type *> Tys;
5357 unsigned ID = CI->getIntrinsicID();
5358 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5359 switch (ID) {
5360 case Intrinsic::arm_mve_mull_int_predicated:
5361 case Intrinsic::arm_mve_vqdmull_predicated:
5362 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5363 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5364 break;
5365 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5366 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5367 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5368 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5369 V2I1Ty};
5370 break;
5371 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5372 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5373 CI->getOperand(1)->getType(), V2I1Ty};
5374 break;
5375 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5376 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5377 CI->getOperand(2)->getType(), V2I1Ty};
5378 break;
5379 case Intrinsic::arm_cde_vcx1q_predicated:
5380 case Intrinsic::arm_cde_vcx1qa_predicated:
5381 case Intrinsic::arm_cde_vcx2q_predicated:
5382 case Intrinsic::arm_cde_vcx2qa_predicated:
5383 case Intrinsic::arm_cde_vcx3q_predicated:
5384 case Intrinsic::arm_cde_vcx3qa_predicated:
5385 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5386 break;
5387 default:
5388 llvm_unreachable("Unhandled Intrinsic!");
5389 }
5390
5391 std::vector<Value *> Ops;
5392 for (Value *Op : CI->args()) {
5393 Type *Ty = Op->getType();
5394 if (Ty->getScalarSizeInBits() == 1) {
5395 Value *C1 = Builder.CreateIntrinsic(
5396 Intrinsic::arm_mve_pred_v2i,
5397 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5398 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5399 }
5400 Ops.push_back(Op);
5401 }
5402
5403 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5404 CI->getName());
5405 }
5406 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5407}
5408
5409// These are expected to have the arguments:
5410// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5411//
5412// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5413//
5415 Function *F, IRBuilder<> &Builder) {
5416 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5417 // for compatibility.
5418 auto UpgradeLegacyWMMAIUIntrinsicCall =
5419 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5420 ArrayRef<Type *> OverloadTys) -> Value * {
5421 // Prepare arguments, append clamp=0 for compatibility
5422 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5423 Args.push_back(Builder.getFalse());
5424
5425 // Insert the declaration for the right overload types
5427 F->getParent(), F->getIntrinsicID(), OverloadTys);
5428
5429 // Copy operand bundles if any
5431 CI->getOperandBundlesAsDefs(Bundles);
5432
5433 // Create the new call and copy calling properties
5434 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5435 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5436 NewCall->setCallingConv(CI->getCallingConv());
5437 NewCall->setAttributes(CI->getAttributes());
5438 NewCall->copyMetadata(*CI);
5439 return NewCall;
5440 };
5441
5442 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5443 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5444 "intrinsic should have 7 arguments");
5445 Type *T1 = CI->getArgOperand(4)->getType();
5446 Type *T2 = CI->getArgOperand(1)->getType();
5447 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5448 }
5449 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5450 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5451 "intrinsic should have 8 arguments");
5452 Type *T1 = CI->getArgOperand(4)->getType();
5453 Type *T2 = CI->getArgOperand(1)->getType();
5454 Type *T3 = CI->getArgOperand(3)->getType();
5455 Type *T4 = CI->getArgOperand(5)->getType();
5456 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5457 }
5458
5459 switch (F->getIntrinsicID()) {
5460 default:
5461 break;
5462 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5463 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5464 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5465 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5466 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5467 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5468 // Drop src0 and src1 modifiers.
5469 const Value *Op0 = CI->getArgOperand(0);
5470 const Value *Op2 = CI->getArgOperand(2);
5471 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5472 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5473 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5474 if (!ModA->isZero() || !ModB->isZero())
5475 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5476
5478 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5479 Args.push_back(CI->getArgOperand(I));
5480
5481 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5482 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5483 Overloads.push_back(Args[3]->getType());
5485 F->getParent(), F->getIntrinsicID(), Overloads);
5486
5488 CI->getOperandBundlesAsDefs(Bundles);
5489
5490 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5491 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5492 NewCall->setCallingConv(CI->getCallingConv());
5493 NewCall->setAttributes(CI->getAttributes());
5494 NewCall->copyMetadata(*CI);
5495 NewCall->takeName(CI);
5496 return NewCall;
5497 }
5498 }
5499
5500 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5501 Value *LHS = CI->getArgOperand(0);
5502 Value *RHS = CI->getArgOperand(1);
5504 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5505 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5506 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5507 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5508 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5509 NewCall->setCallingConv(CI->getCallingConv());
5510 NewCall->copyMetadata(*CI);
5511 NewCall->takeName(CI);
5512 return NewCall;
5513 }
5514
5515 if (Name.starts_with("addrspacecast.nonnull")) {
5516 if (CI->getNumOperands() < 2) // Malformed bitcode.
5517 return nullptr;
5518 Value *ASC = Builder.CreateAddrSpaceCast(
5519 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5520 ASC->takeName(CI);
5521 return ASC;
5522 }
5523
5524 AtomicRMWInst::BinOp RMWOp =
5526 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5527 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5528 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5529 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5530 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5531 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5532 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5533 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5534 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5535 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5536 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5537 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5538 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5539
5540 unsigned NumOperands = CI->getNumOperands();
5541 if (NumOperands < 3) // Malformed bitcode.
5542 return nullptr;
5543
5544 Value *Ptr = CI->getArgOperand(0);
5545 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5546 if (!PtrTy) // Malformed.
5547 return nullptr;
5548
5549 Value *Val = CI->getArgOperand(1);
5550 if (Val->getType() != CI->getType()) // Malformed.
5551 return nullptr;
5552
5553 ConstantInt *OrderArg = nullptr;
5554 bool IsVolatile = false;
5555
5556 // These should have 5 arguments (plus the callee). A separate version of the
5557 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5558 if (NumOperands > 3)
5559 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5560
5561 // Ignore scope argument at 3
5562
5563 if (NumOperands > 5) {
5564 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5565 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5566 }
5567
5569 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5570 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5573
5574 LLVMContext &Ctx = F->getContext();
5575
5576 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5577 Type *RetTy = CI->getType();
5578 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5579 if (VT->getElementType()->isIntegerTy(16)) {
5580 VectorType *AsBF16 =
5581 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5582 Val = Builder.CreateBitCast(Val, AsBF16);
5583 }
5584 }
5585
5586 // The scope argument never really worked correctly. Use agent as the most
5587 // conservative option which should still always produce the instruction.
5588 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5589 AtomicRMWInst *RMW =
5590 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5591
5592 unsigned AddrSpace = PtrTy->getAddressSpace();
5593 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5594 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5595 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5596 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5597 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5598 }
5599
5600 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5601 MDBuilder MDB(F->getContext());
5602 MDNode *RangeNotPrivate =
5605 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5606 }
5607
5608 if (IsVolatile)
5609 RMW->setVolatile(true);
5610
5611 return Builder.CreateBitCast(RMW, RetTy);
5612}
5613
5614/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5615/// plain MDNode, as it's the verifier's job to check these are the correct
5616/// types later.
5617static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5618 if (Op < CI->arg_size()) {
5619 if (MetadataAsValue *MAV =
5621 Metadata *MD = MAV->getMetadata();
5622 return dyn_cast_if_present<MDNode>(MD);
5623 }
5624 }
5625 return nullptr;
5626}
5627
5628/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5629static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5630 if (Op < CI->arg_size())
5632 return MAV->getMetadata();
5633 return nullptr;
5634}
5635
5636/// Convert debug intrinsic calls to non-instruction debug records.
5637/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5638/// \p CI - The debug intrinsic call.
5640 DbgRecord *DR = nullptr;
5641 if (Name == "label") {
5643 } else if (Name == "assign") {
5646 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5647 unwrapMAVMetadataOp(CI, 4),
5648 /*The address is a Value ref, it will be stored as a Metadata */
5649 unwrapMAVOp(CI, 5));
5650 } else if (Name == "declare") {
5653 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5654 } else if (Name == "addr") {
5655 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5656 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5657 // Don't try to add something to the expression if it's not an expression.
5658 // Instead, allow the verifier to fail later.
5659 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5660 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5661 }
5664 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5665 } else if (Name == "value") {
5666 // An old version of dbg.value had an extra offset argument.
5667 unsigned VarOp = 1;
5668 unsigned ExprOp = 2;
5669 if (CI->arg_size() == 4) {
5671 // Nonzero offset dbg.values get dropped without a replacement.
5672 if (!Offset || !Offset->isNullValue())
5673 return;
5674 VarOp = 2;
5675 ExprOp = 3;
5676 }
5679 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5680 nullptr);
5681 }
5682 DR->setDebugLoc(CI->getDebugLoc());
5683 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5684 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5685}
5686
5689 if (!Offset)
5690 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5691 int64_t OffsetVal = Offset->getSExtValue();
5692 return Builder.CreateIntrinsic(OffsetVal >= 0
5693 ? Intrinsic::vector_splice_left
5694 : Intrinsic::vector_splice_right,
5695 CI->getType(),
5696 {CI->getArgOperand(0), CI->getArgOperand(1),
5697 Builder.getInt32(std::abs(OffsetVal))});
5698}
5699
5701 Function *F, IRBuilder<> &Builder) {
5702 if (Name.starts_with("to.fp16")) {
5703 Value *Cast =
5704 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5705 return Builder.CreateBitCast(Cast, CI->getType());
5706 }
5707
5708 if (Name.starts_with("from.fp16")) {
5709 Value *Cast =
5710 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5711 return Builder.CreateFPExt(Cast, CI->getType());
5712 }
5713
5714 return nullptr;
5715}
5716
5718 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5719 if (!MD || !isa<MDString>(MD))
5721 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5722 .Case("eq", ICmpInst::ICMP_EQ)
5723 .Case("ne", ICmpInst::ICMP_NE)
5724 .Case("ugt", ICmpInst::ICMP_UGT)
5725 .Case("uge", ICmpInst::ICMP_UGE)
5726 .Case("ult", ICmpInst::ICMP_ULT)
5727 .Case("ule", ICmpInst::ICMP_ULE)
5728 .Case("sgt", ICmpInst::ICMP_SGT)
5729 .Case("sge", ICmpInst::ICMP_SGE)
5730 .Case("slt", ICmpInst::ICMP_SLT)
5731 .Case("sle", ICmpInst::ICMP_SLE)
5733}
5734
5736 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5737 if (!MD || !isa<MDString>(MD))
5739 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5740 .Case("oeq", FCmpInst::FCMP_OEQ)
5741 .Case("ogt", FCmpInst::FCMP_OGT)
5742 .Case("oge", FCmpInst::FCMP_OGE)
5743 .Case("olt", FCmpInst::FCMP_OLT)
5744 .Case("ole", FCmpInst::FCMP_OLE)
5745 .Case("one", FCmpInst::FCMP_ONE)
5746 .Case("ord", FCmpInst::FCMP_ORD)
5747 .Case("uno", FCmpInst::FCMP_UNO)
5748 .Case("ueq", FCmpInst::FCMP_UEQ)
5749 .Case("ugt", FCmpInst::FCMP_UGT)
5750 .Case("uge", FCmpInst::FCMP_UGE)
5751 .Case("ult", FCmpInst::FCMP_ULT)
5752 .Case("ule", FCmpInst::FCMP_ULE)
5753 .Case("une", FCmpInst::FCMP_UNE)
5755}
5756
5758 IRBuilder<> &Builder) {
5759 Value *Rep;
5760 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5761 if (Opcode && Instruction::isUnaryOp(Opcode))
5762 Rep =
5763 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5764 else if (Opcode && Instruction::isBinaryOp(Opcode))
5765 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5766 CI->getArgOperand(0), CI->getArgOperand(1));
5767 else if (Opcode && Instruction::isCast(Opcode))
5768 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5769 CI->getType());
5770 else if (Opcode == Instruction::ICmp)
5771 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5772 CI->getArgOperand(0), CI->getArgOperand(1));
5773 else if (Opcode == Instruction::FCmp)
5774 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5775 CI->getArgOperand(0), CI->getArgOperand(1));
5776 else if (Opcode == Instruction::Select)
5777 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5778 CI->getArgOperand(2));
5779 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5780 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5781 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5782 } else
5783 llvm_unreachable("Unexpected vp intrinsic");
5784 Rep->takeName(CI);
5785 return Rep;
5786}
5787
5789 IRBuilder<> &Builder) {
5790 Intrinsic::ID IID = NewFn->getIntrinsicID();
5791
5792 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5793 if (Defaults.empty())
5794 return false;
5795
5796 unsigned OldArgCount = CI->arg_size();
5797 unsigned NewArgCount = NewFn->arg_size();
5798
5799 if (OldArgCount < FirstDefault)
5800 return false;
5801
5802 // More arguments than the new intrinsic accepts, cannot upgrade.
5803 if (OldArgCount > NewArgCount)
5804 return false;
5805
5806 // The call already passes the defaulted arguments explicitly; only the
5807 // callee is still the old, shorter declaration, so retarget it.
5808 if (OldArgCount == NewArgCount) {
5809 if (CI->getFunctionType() != NewFn->getFunctionType())
5810 return false;
5811 CI->setCalledFunction(NewFn);
5812 return true;
5813 }
5814
5815 // OldArgCount < NewArgCount: Fill in each missing trailing default
5816 // argument from the table.
5817 SmallVector<Value *, 8> NewArgs(CI->args());
5818
5819 FunctionType *NewFT = NewFn->getFunctionType();
5820 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5821 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5822 "missing argument outside the default range");
5823 Type *ParamTy = NewFT->getParamType(Idx);
5824
5825 // Only integer types are supported (i1, i8, i16, i32, i64).
5826 if (!ParamTy->isIntegerTy())
5827 return false;
5828 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5829 }
5830
5831 // Preserve operand bundles by creating the call with them.
5833 CI->getOperandBundlesAsDefs(OpBundles);
5834 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5835
5836 NewCall->takeName(CI);
5837 NewCall->setCallingConv(CI->getCallingConv());
5838 NewCall->copyMetadata(*CI);
5839 if (auto *OldCI = dyn_cast<CallInst>(CI))
5840 NewCall->setTailCallKind(OldCI->getTailCallKind());
5841
5842 CI->replaceAllUsesWith(NewCall);
5843 CI->eraseFromParent();
5844 return true;
5845}
5846
5847/// Upgrade a call to an old intrinsic. All argument and return casting must be
5848/// provided to seamlessly integrate with existing context.
5850 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5851 // checks the callee's function type matches. It's likely we need to handle
5852 // type changes here.
5854 if (!F)
5855 return;
5856
5857 LLVMContext &C = CI->getContext();
5858 IRBuilder<> Builder(CI->getIterator());
5859 if (isa<FPMathOperator>(CI))
5860 Builder.setFastMathFlags(CI->getFastMathFlags());
5861
5862 if (!NewFn) {
5863 // Get the Function's name.
5864 StringRef Name = F->getName();
5865 if (!Name.consume_front("llvm."))
5866 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5867
5868 bool IsX86 = Name.consume_front("x86.");
5869 bool IsNVVM = Name.consume_front("nvvm.");
5870 bool IsAArch64 = Name.consume_front("aarch64.");
5871 bool IsARM = Name.consume_front("arm.");
5872 bool IsAMDGCN = Name.consume_front("amdgcn.");
5873 bool IsDbg = Name.consume_front("dbg.");
5874 bool IsOldSplice =
5875 (Name.consume_front("experimental.vector.splice") ||
5876 Name.consume_front("vector.splice")) &&
5877 !(Name.starts_with(".left") || Name.starts_with(".right"));
5878 Value *Rep = nullptr;
5879
5880 if (!IsX86 && Name == "stackprotectorcheck") {
5881 Rep = nullptr;
5882 } else if (IsNVVM) {
5883 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5884 } else if (IsX86) {
5885 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5886 } else if (IsAArch64) {
5887 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5888 } else if (IsARM) {
5889 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5890 } else if (IsAMDGCN) {
5891 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5892 } else if (IsDbg) {
5894 } else if (IsOldSplice) {
5895 Rep = upgradeVectorSplice(CI, Builder);
5896 } else if (Name.consume_front("convert.")) {
5897 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5898 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5899 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5900 Rep = nullptr;
5901 } else if (shouldUpgradeVPIntrinsic(Name)) {
5902 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5903 } else {
5904 llvm_unreachable("Unknown function for CallBase upgrade.");
5905 }
5906
5907 if (Rep)
5908 CI->replaceAllUsesWith(Rep);
5909 CI->eraseFromParent();
5910 return;
5911 }
5912
5913 const auto &DefaultCase = [&]() -> void {
5914 if (F == NewFn)
5915 return;
5916
5917 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5918 // Handle generic mangling change.
5919 assert(
5920 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5921 "Unknown function for CallBase upgrade and isn't just a name change");
5922 CI->setCalledFunction(NewFn);
5923 return;
5924 }
5925
5926 // This must be an upgrade from a named to a literal struct.
5927 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5928 assert(OldST != NewFn->getReturnType() &&
5929 "Return type must have changed");
5930 assert(OldST->getNumElements() ==
5931 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5932 "Must have same number of elements");
5933
5934 SmallVector<Value *> Args(CI->args());
5935 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5936 NewCI->setAttributes(CI->getAttributes());
5937 Value *Res = PoisonValue::get(OldST);
5938 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5939 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5940 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5941 }
5942 CI->replaceAllUsesWith(Res);
5943 CI->eraseFromParent();
5944 return;
5945 }
5946
5947 // We're probably about to produce something invalid. Let the verifier catch
5948 // it instead of dying here.
5949 CI->setCalledOperand(
5951 return;
5952 };
5953 CallInst *NewCall = nullptr;
5954 switch (NewFn->getIntrinsicID()) {
5955 default: {
5956 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5957 return;
5958 DefaultCase();
5959 return;
5960 }
5961 case Intrinsic::arm_neon_vst1:
5962 case Intrinsic::arm_neon_vst2:
5963 case Intrinsic::arm_neon_vst3:
5964 case Intrinsic::arm_neon_vst4:
5965 case Intrinsic::arm_neon_vst2lane:
5966 case Intrinsic::arm_neon_vst3lane:
5967 case Intrinsic::arm_neon_vst4lane: {
5968 SmallVector<Value *, 4> Args(CI->args());
5969 NewCall = Builder.CreateCall(NewFn, Args);
5970 break;
5971 }
5972 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5973 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5974 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5975 LLVMContext &Ctx = F->getParent()->getContext();
5976 SmallVector<Value *, 4> Args(CI->args());
5977 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5978 cast<ConstantInt>(Args[3])->getZExtValue());
5979 NewCall = Builder.CreateCall(NewFn, Args);
5980 break;
5981 }
5982 case Intrinsic::aarch64_sve_ld3_sret:
5983 case Intrinsic::aarch64_sve_ld4_sret:
5984 case Intrinsic::aarch64_sve_ld2_sret: {
5985 // Is this a trivial remangle of the name to support ptr address spaces?
5986 if (isa<StructType>(F->getReturnType())) {
5987 DefaultCase();
5988 return;
5989 }
5990
5991 StringRef Name = F->getName();
5992 Name = Name.substr(5);
5993 unsigned N = StringSwitch<unsigned>(Name)
5994 .StartsWith("aarch64.sve.ld2", 2)
5995 .StartsWith("aarch64.sve.ld3", 3)
5996 .StartsWith("aarch64.sve.ld4", 4)
5997 .Default(0);
5998 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5999 unsigned MinElts = RetTy->getMinNumElements() / N;
6000 SmallVector<Value *, 2> Args(CI->args());
6001 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
6002 Value *Ret = llvm::PoisonValue::get(RetTy);
6003 for (unsigned I = 0; I < N; I++) {
6004 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
6005 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
6006 }
6007 NewCall = dyn_cast<CallInst>(Ret);
6008 break;
6009 }
6010
6011 case Intrinsic::coro_end_async:
6012 case Intrinsic::coro_end: {
6013 SmallVector<Value *, 3> Args(CI->args());
6014 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
6015 Args.push_back(ConstantTokenNone::get(CI->getContext()));
6016 NewCall = Builder.CreateCall(NewFn, Args);
6017
6018 if (!CI->getType()->isVoidTy()) {
6019 if (!CI->use_empty()) {
6021 CI->getModule(), Intrinsic::coro_is_in_ramp);
6022 Value *InRamp = Builder.CreateCall(IsInRamp);
6023 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
6024 }
6025 CI->eraseFromParent();
6026 return;
6027 }
6028
6029 break;
6030 }
6031
6032 case Intrinsic::vector_extract: {
6033 StringRef Name = F->getName();
6034 Name = Name.substr(5); // Strip llvm
6035 if (!Name.starts_with("aarch64.sve.tuple.get")) {
6036 DefaultCase();
6037 return;
6038 }
6039 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6040 unsigned MinElts = RetTy->getMinNumElements();
6041 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6042 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
6043 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
6044 break;
6045 }
6046
6047 case Intrinsic::vector_insert: {
6048 StringRef Name = F->getName();
6049 Name = Name.substr(5);
6050 if (!Name.starts_with("aarch64.sve.tuple")) {
6051 DefaultCase();
6052 return;
6053 }
6054 if (Name.starts_with("aarch64.sve.tuple.set")) {
6055 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
6056 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
6057 Value *NewIdx =
6058 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
6059 NewCall = Builder.CreateCall(
6060 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
6061 break;
6062 }
6063 if (Name.starts_with("aarch64.sve.tuple.create")) {
6064 unsigned N = StringSwitch<unsigned>(Name)
6065 .StartsWith("aarch64.sve.tuple.create2", 2)
6066 .StartsWith("aarch64.sve.tuple.create3", 3)
6067 .StartsWith("aarch64.sve.tuple.create4", 4)
6068 .Default(0);
6069 assert(N > 1 && "Create is expected to be between 2-4");
6070 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
6071 Value *Ret = llvm::PoisonValue::get(RetTy);
6072 unsigned MinElts = RetTy->getMinNumElements() / N;
6073 for (unsigned I = 0; I < N; I++) {
6074 Value *V = CI->getArgOperand(I);
6075 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
6076 }
6077 NewCall = dyn_cast<CallInst>(Ret);
6078 }
6079 break;
6080 }
6081
6082 case Intrinsic::arm_neon_bfdot:
6083 case Intrinsic::arm_neon_bfmmla:
6084 case Intrinsic::arm_neon_bfmlalb:
6085 case Intrinsic::arm_neon_bfmlalt:
6086 case Intrinsic::aarch64_neon_bfdot:
6087 case Intrinsic::aarch64_neon_bfmmla:
6088 case Intrinsic::aarch64_neon_bfmlalb:
6089 case Intrinsic::aarch64_neon_bfmlalt: {
6091 assert(CI->arg_size() == 3 &&
6092 "Mismatch between function args and call args");
6093 size_t OperandWidth =
6095 assert((OperandWidth == 64 || OperandWidth == 128) &&
6096 "Unexpected operand width");
6097 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
6098 auto Iter = CI->args().begin();
6099 Args.push_back(*Iter++);
6100 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6101 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6102 NewCall = Builder.CreateCall(NewFn, Args);
6103 break;
6104 }
6105
6106 case Intrinsic::bitreverse:
6107 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6108 break;
6109
6110 case Intrinsic::ctlz:
6111 case Intrinsic::cttz: {
6112 if (CI->arg_size() != 1) {
6113 DefaultCase();
6114 return;
6115 }
6116
6117 NewCall =
6118 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
6119 break;
6120 }
6121
6122 case Intrinsic::objectsize: {
6123 Value *NullIsUnknownSize =
6124 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6125 Value *Dynamic =
6126 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6127 NewCall = Builder.CreateCall(
6128 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6129 break;
6130 }
6131
6132 case Intrinsic::ctpop:
6133 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6134 break;
6135 case Intrinsic::dbg_value: {
6136 StringRef Name = F->getName();
6137 Name = Name.substr(5); // Strip llvm.
6138 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6139 if (Name.starts_with("dbg.addr")) {
6141 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6142 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6143 NewCall =
6144 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6145 MetadataAsValue::get(C, Expr)});
6146 break;
6147 }
6148
6149 // Upgrade from the old version that had an extra offset argument.
6150 assert(CI->arg_size() == 4);
6151 // Drop nonzero offsets instead of attempting to upgrade them.
6153 if (Offset->isNullValue()) {
6154 NewCall = Builder.CreateCall(
6155 NewFn,
6156 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6157 break;
6158 }
6159 CI->eraseFromParent();
6160 return;
6161 }
6162
6163 case Intrinsic::ptr_annotation:
6164 // Upgrade from versions that lacked the annotation attribute argument.
6165 if (CI->arg_size() != 4) {
6166 DefaultCase();
6167 return;
6168 }
6169
6170 // Create a new call with an added null annotation attribute argument.
6171 NewCall = Builder.CreateCall(
6172 NewFn,
6173 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6174 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6175 NewCall->takeName(CI);
6176 CI->replaceAllUsesWith(NewCall);
6177 CI->eraseFromParent();
6178 return;
6179
6180 case Intrinsic::var_annotation:
6181 // Upgrade from versions that lacked the annotation attribute argument.
6182 if (CI->arg_size() != 4) {
6183 DefaultCase();
6184 return;
6185 }
6186 // Create a new call with an added null annotation attribute argument.
6187 NewCall = Builder.CreateCall(
6188 NewFn,
6189 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6190 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6191 NewCall->takeName(CI);
6192 CI->replaceAllUsesWith(NewCall);
6193 CI->eraseFromParent();
6194 return;
6195
6196 case Intrinsic::riscv_aes32dsi:
6197 case Intrinsic::riscv_aes32dsmi:
6198 case Intrinsic::riscv_aes32esi:
6199 case Intrinsic::riscv_aes32esmi:
6200 case Intrinsic::riscv_sm4ks:
6201 case Intrinsic::riscv_sm4ed: {
6202 // The last argument to these intrinsics used to be i8 and changed to i32.
6203 // The type overload for sm4ks and sm4ed was removed.
6204 Value *Arg2 = CI->getArgOperand(2);
6205 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6206 return;
6207
6208 Value *Arg0 = CI->getArgOperand(0);
6209 Value *Arg1 = CI->getArgOperand(1);
6210 if (CI->getType()->isIntegerTy(64)) {
6211 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6212 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6213 }
6214
6215 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6216 cast<ConstantInt>(Arg2)->getZExtValue());
6217
6218 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6219 Value *Res = NewCall;
6220 if (Res->getType() != CI->getType())
6221 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6222 NewCall->takeName(CI);
6223 CI->replaceAllUsesWith(Res);
6224 CI->eraseFromParent();
6225 return;
6226 }
6227 case Intrinsic::nvvm_mapa_shared_cluster: {
6228 // Create a new call with the correct address space.
6229 NewCall =
6230 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6231 Value *Res = NewCall;
6232 Res = Builder.CreateAddrSpaceCast(
6233 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6234 NewCall->takeName(CI);
6235 CI->replaceAllUsesWith(Res);
6236 CI->eraseFromParent();
6237 return;
6238 }
6239 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6240 SmallVector<Value *, 4> Args(CI->args());
6241 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6243 Args[0] = Builder.CreateAddrSpaceCast(
6244 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6245
6246 // Append the missing trailing flag_valid_pattern (0 = disabled).
6247 Args.push_back(Builder.getInt32(0));
6248
6249 NewCall = Builder.CreateCall(NewFn, Args);
6250 NewCall->takeName(CI);
6251 CI->replaceAllUsesWith(NewCall);
6252 CI->eraseFromParent();
6253 return;
6254 }
6255 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6256 // (dst, mbar, src, size, ch, flag_ch)
6257 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6258 // i32 0 /* flag_valid_pattern=disabled */)
6260 for (unsigned I = 0; I < 4; ++I)
6261 Args.push_back(CI->getArgOperand(I));
6262 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6263 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6264 Args.push_back(CI->getArgOperand(4)); // cache_hint
6265 Args.push_back(CI->getArgOperand(5)); // flag_ch
6266 Args.push_back(Builder.getInt1(false)); // flag_oob
6267 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6268
6269 NewCall = Builder.CreateCall(NewFn, Args);
6270 NewCall->takeName(CI);
6271 CI->replaceAllUsesWith(NewCall);
6272 CI->eraseFromParent();
6273 return;
6274 }
6275 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6276 // Create a new call with the correct address space.
6277 SmallVector<Value *, 4> Args(CI->args());
6278 Args[0] = Builder.CreateAddrSpaceCast(
6279 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6280
6281 NewCall = Builder.CreateCall(NewFn, Args);
6282 NewCall->takeName(CI);
6283 CI->replaceAllUsesWith(NewCall);
6284 CI->eraseFromParent();
6285 return;
6286 }
6287 // clang-format off
6288#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6289 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6291#undef G2S_CLUSTER_CASE
6292 {
6293 SmallVector<Value *, 16> Args(CI->args());
6294 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6296 Args[0] = Builder.CreateAddrSpaceCast(
6297 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6298
6299 // Append the missing trailing arguments with default values (cta_group,
6300 // flag_valid_pattern).
6301 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6302 Args.push_back(Builder.getInt32(0));
6303
6304 NewCall = Builder.CreateCall(NewFn, Args);
6305 NewCall->takeName(CI);
6306 CI->replaceAllUsesWith(NewCall);
6307 CI->eraseFromParent();
6308 return;
6309 }
6310
6311#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6312 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6314#undef G2S_CTA_CASE
6315 {
6316 SmallVector<Value *, 16> Args(CI->args());
6317 // Append the missing trailing flag_valid_pattern argument with default
6318 // value 0.
6319 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6320 "expected only the trailing flag_valid_pattern to be missing");
6321 Args.push_back(Builder.getInt32(0));
6322
6323 NewCall = Builder.CreateCall(NewFn, Args);
6324 NewCall->takeName(CI);
6325 CI->replaceAllUsesWith(NewCall);
6326 CI->eraseFromParent();
6327 return;
6328 }
6329#undef NVVM_TMA_G2S_MODES
6330 // clang-format on
6331
6332 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6333 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6334 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6335 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6336 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6337 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6338 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6339 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6340 StringRef Name = F->getName();
6341 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6342 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6343
6344 SmallVector<Value *, 16> Args(CI->args());
6345 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6346 NewCall = Builder.CreateCall(NewFn, Args);
6347 break;
6348 }
6349 // clang-format off
6350 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cta_space_cluster:
6351 case Intrinsic::nvvm_mbarrier_expect_tx_scope_cluster_space_cluster:
6352 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cta_space_cluster:
6353 case Intrinsic::nvvm_mbarrier_complete_tx_scope_cluster_space_cluster:
6354 case Intrinsic::nvvm_mbarrier_arrive_scope_cta_space_cluster:
6355 case Intrinsic::nvvm_mbarrier_arrive_scope_cluster_space_cluster:
6356 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cta_space_cluster:
6357 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_scope_cluster_space_cluster:
6358 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cta_space_cluster:
6359 case Intrinsic::nvvm_mbarrier_arrive_drop_scope_cluster_space_cluster:
6360 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cta_space_cluster:
6361 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_scope_cluster_space_cluster:
6362 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cta_space_cluster:
6363 case Intrinsic::nvvm_mbarrier_arrive_relaxed_scope_cluster_space_cluster:
6364 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cta_space_cluster:
6365 case Intrinsic::nvvm_mbarrier_arrive_expect_tx_relaxed_scope_cluster_space_cluster:
6366 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cta_space_cluster:
6367 case Intrinsic::nvvm_mbarrier_arrive_drop_relaxed_scope_cluster_space_cluster:
6368 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cta_space_cluster:
6369 case Intrinsic::nvvm_mbarrier_arrive_drop_expect_tx_relaxed_scope_cluster_space_cluster: {
6370 // clang-format on
6371 SmallVector<Value *, 8> Args(CI->args());
6372 Args.push_back(Builder.getInt32(0)); // cta-mask
6373 Args.push_back(Builder.getFalse()); // is-multicast
6374 NewCall = Builder.CreateCall(NewFn, Args);
6375 NewCall->takeName(CI);
6376 CI->replaceAllUsesWith(NewCall);
6377 CI->eraseFromParent();
6378 return;
6379 }
6380 case Intrinsic::nvvm_mbarrier_init: {
6381 SmallVector<Value *, 3> Args(CI->args());
6382 // The .shared variant folded into the overloaded form without gaining an
6383 // operand, so only the pre-layout two-argument form needs one appended.
6384 if (Args.size() == 2)
6385 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6386 NewCall = Builder.CreateCall(NewFn, Args);
6387 break;
6388 }
6389 case Intrinsic::riscv_sha256sig0:
6390 case Intrinsic::riscv_sha256sig1:
6391 case Intrinsic::riscv_sha256sum0:
6392 case Intrinsic::riscv_sha256sum1:
6393 case Intrinsic::riscv_sm3p0:
6394 case Intrinsic::riscv_sm3p1: {
6395 // The last argument to these intrinsics used to be i8 and changed to i32.
6396 // The type overload for sm4ks and sm4ed was removed.
6397 if (!CI->getType()->isIntegerTy(64))
6398 return;
6399
6400 Value *Arg =
6401 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6402
6403 NewCall = Builder.CreateCall(NewFn, Arg);
6404 Value *Res =
6405 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6406 NewCall->takeName(CI);
6407 CI->replaceAllUsesWith(Res);
6408 CI->eraseFromParent();
6409 return;
6410 }
6411
6412 case Intrinsic::x86_xop_vfrcz_ss:
6413 case Intrinsic::x86_xop_vfrcz_sd:
6414 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6415 break;
6416
6417 case Intrinsic::x86_xop_vpermil2pd:
6418 case Intrinsic::x86_xop_vpermil2ps:
6419 case Intrinsic::x86_xop_vpermil2pd_256:
6420 case Intrinsic::x86_xop_vpermil2ps_256: {
6421 SmallVector<Value *, 4> Args(CI->args());
6422 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6423 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6424 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6425 NewCall = Builder.CreateCall(NewFn, Args);
6426 break;
6427 }
6428
6429 case Intrinsic::x86_sse41_ptestc:
6430 case Intrinsic::x86_sse41_ptestz:
6431 case Intrinsic::x86_sse41_ptestnzc: {
6432 // The arguments for these intrinsics used to be v4f32, and changed
6433 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6434 // So, the only thing required is a bitcast for both arguments.
6435 // First, check the arguments have the old type.
6436 Value *Arg0 = CI->getArgOperand(0);
6437 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6438 return;
6439
6440 // Old intrinsic, add bitcasts
6441 Value *Arg1 = CI->getArgOperand(1);
6442
6443 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6444
6445 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6446 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6447
6448 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6449 break;
6450 }
6451
6452 case Intrinsic::x86_rdtscp: {
6453 // This used to take 1 arguments. If we have no arguments, it is already
6454 // upgraded.
6455 if (CI->getNumOperands() == 0)
6456 return;
6457
6458 NewCall = Builder.CreateCall(NewFn);
6459 // Extract the second result and store it.
6460 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6461 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6462 // Replace the original call result with the first result of the new call.
6463 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6464
6465 NewCall->takeName(CI);
6466 CI->replaceAllUsesWith(TSC);
6467 CI->eraseFromParent();
6468 return;
6469 }
6470
6471 case Intrinsic::x86_sse41_insertps:
6472 case Intrinsic::x86_sse41_dppd:
6473 case Intrinsic::x86_sse41_dpps:
6474 case Intrinsic::x86_sse41_mpsadbw:
6475 case Intrinsic::x86_avx_dp_ps_256:
6476 case Intrinsic::x86_avx2_mpsadbw: {
6477 // Need to truncate the last argument from i32 to i8 -- this argument models
6478 // an inherently 8-bit immediate operand to these x86 instructions.
6479 SmallVector<Value *, 4> Args(CI->args());
6480
6481 // Replace the last argument with a trunc.
6482 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6483 NewCall = Builder.CreateCall(NewFn, Args);
6484 break;
6485 }
6486
6487 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6488 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6489 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6490 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6491 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6492 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6493 SmallVector<Value *, 4> Args(CI->args());
6494 unsigned NumElts =
6495 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6496 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6497
6498 NewCall = Builder.CreateCall(NewFn, Args);
6499 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6500
6501 NewCall->takeName(CI);
6502 CI->replaceAllUsesWith(Res);
6503 CI->eraseFromParent();
6504 return;
6505 }
6506
6507 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6508 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6509 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6510 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6511 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6512 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6513 SmallVector<Value *, 4> Args(CI->args());
6514 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6515 if (NewFn->getIntrinsicID() ==
6516 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6517 Args[1] = Builder.CreateBitCast(
6518 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6519
6520 NewCall = Builder.CreateCall(NewFn, Args);
6521 Value *Res = Builder.CreateBitCast(
6522 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6523
6524 NewCall->takeName(CI);
6525 CI->replaceAllUsesWith(Res);
6526 CI->eraseFromParent();
6527 return;
6528 }
6529 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6530 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6531 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6532 SmallVector<Value *, 4> Args(CI->args());
6533 unsigned NumElts =
6534 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6535 Args[1] = Builder.CreateBitCast(
6536 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6537 Args[2] = Builder.CreateBitCast(
6538 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6539
6540 NewCall = Builder.CreateCall(NewFn, Args);
6541 break;
6542 }
6543
6544 case Intrinsic::amdgcn_asyncmark:
6545 case Intrinsic::amdgcn_wait_asyncmark: {
6546 // Legacy asyncmark intrinsics missed the stage mask operand. Append a zero
6547 // mask, which names every stage and so preserves the behavior they had.
6548 SmallVector<Value *, 2> Args(CI->args());
6549 Args.push_back(Builder.getInt32(0));
6550
6552 CI->getOperandBundlesAsDefs(Bundles);
6553
6554 NewCall = Builder.CreateCall(NewFn, Args, Bundles);
6555 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6556 NewCall->setCallingConv(CI->getCallingConv());
6557 NewCall->setAttributes(CI->getAttributes());
6558 NewCall->copyMetadata(*CI);
6559 break;
6560 }
6561
6562 case Intrinsic::thread_pointer: {
6563 NewCall = Builder.CreateCall(NewFn, {});
6564 break;
6565 }
6566
6567 case Intrinsic::memcpy:
6568 case Intrinsic::memmove:
6569 case Intrinsic::memset: {
6570 // We have to make sure that the call signature is what we're expecting.
6571 // We only want to change the old signatures by removing the alignment arg:
6572 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6573 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6574 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6575 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6576 // Note: i8*'s in the above can be any pointer type
6577 if (CI->arg_size() != 5) {
6578 DefaultCase();
6579 return;
6580 }
6581 // Remove alignment argument (3), and add alignment attributes to the
6582 // dest/src pointers.
6583 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6584 CI->getArgOperand(2), CI->getArgOperand(4)};
6585 NewCall = Builder.CreateCall(NewFn, Args);
6586 AttributeList OldAttrs = CI->getAttributes();
6587 AttributeList NewAttrs = AttributeList::get(
6588 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6589 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6590 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6591 NewCall->setAttributes(NewAttrs);
6592 auto *MemCI = cast<MemIntrinsic>(NewCall);
6593 // All mem intrinsics support dest alignment.
6595 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6596 // Memcpy/Memmove also support source alignment.
6597 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6598 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6599 break;
6600 }
6601
6602 case Intrinsic::masked_load:
6603 case Intrinsic::masked_gather:
6604 case Intrinsic::masked_store:
6605 case Intrinsic::masked_scatter: {
6606 if (CI->arg_size() != 4) {
6607 DefaultCase();
6608 return;
6609 }
6610
6611 auto GetMaybeAlign = [](Value *Op) {
6612 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6613 uint64_t Val = CI->getZExtValue();
6614 if (Val == 0)
6615 return MaybeAlign();
6616 if (isPowerOf2_64(Val))
6617 return MaybeAlign(Val);
6618 }
6619 reportFatalUsageError("Invalid alignment argument");
6620 };
6621 auto GetAlign = [&](Value *Op) {
6622 MaybeAlign Align = GetMaybeAlign(Op);
6623 if (Align)
6624 return *Align;
6625 reportFatalUsageError("Invalid zero alignment argument");
6626 };
6627
6628 const DataLayout &DL = CI->getDataLayout();
6629 switch (NewFn->getIntrinsicID()) {
6630 case Intrinsic::masked_load:
6631 NewCall = Builder.CreateMaskedLoad(
6632 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6633 CI->getArgOperand(2), CI->getArgOperand(3));
6634 break;
6635 case Intrinsic::masked_gather:
6636 NewCall = Builder.CreateMaskedGather(
6637 CI->getType(), CI->getArgOperand(0),
6638 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6639 CI->getType()->getScalarType()),
6640 CI->getArgOperand(2), CI->getArgOperand(3));
6641 break;
6642 case Intrinsic::masked_store:
6643 NewCall = Builder.CreateMaskedStore(
6644 CI->getArgOperand(0), CI->getArgOperand(1),
6645 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6646 break;
6647 case Intrinsic::masked_scatter:
6648 NewCall = Builder.CreateMaskedScatter(
6649 CI->getArgOperand(0), CI->getArgOperand(1),
6650 DL.getValueOrABITypeAlignment(
6651 GetMaybeAlign(CI->getArgOperand(2)),
6652 CI->getArgOperand(0)->getType()->getScalarType()),
6653 CI->getArgOperand(3));
6654 break;
6655 default:
6656 llvm_unreachable("Unexpected intrinsic ID");
6657 }
6658 // Previous metadata is still valid.
6659 NewCall->copyMetadata(*CI);
6660 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6661 break;
6662 }
6663
6664 case Intrinsic::lifetime_start:
6665 case Intrinsic::lifetime_end: {
6666 if (CI->arg_size() != 2) {
6667 DefaultCase();
6668 return;
6669 }
6670
6671 Value *Ptr = CI->getArgOperand(1);
6672 // Try to strip pointer casts, such that the lifetime works on an alloca.
6673 Ptr = Ptr->stripPointerCasts();
6674 if (isa<AllocaInst>(Ptr)) {
6675 // Don't use NewFn, as we might have looked through an addrspacecast.
6676 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6677 NewCall = Builder.CreateLifetimeStart(Ptr);
6678 else
6679 NewCall = Builder.CreateLifetimeEnd(Ptr);
6680 break;
6681 }
6682
6683 // Otherwise remove the lifetime marker.
6684 CI->eraseFromParent();
6685 return;
6686 }
6687
6688 case Intrinsic::x86_avx512_vpdpbusd_128:
6689 case Intrinsic::x86_avx512_vpdpbusd_256:
6690 case Intrinsic::x86_avx512_vpdpbusd_512:
6691 case Intrinsic::x86_avx512_vpdpbusds_128:
6692 case Intrinsic::x86_avx512_vpdpbusds_256:
6693 case Intrinsic::x86_avx512_vpdpbusds_512:
6694 case Intrinsic::x86_avx2_vpdpbssd_128:
6695 case Intrinsic::x86_avx2_vpdpbssd_256:
6696 case Intrinsic::x86_avx10_vpdpbssd_512:
6697 case Intrinsic::x86_avx2_vpdpbssds_128:
6698 case Intrinsic::x86_avx2_vpdpbssds_256:
6699 case Intrinsic::x86_avx10_vpdpbssds_512:
6700 case Intrinsic::x86_avx2_vpdpbsud_128:
6701 case Intrinsic::x86_avx2_vpdpbsud_256:
6702 case Intrinsic::x86_avx10_vpdpbsud_512:
6703 case Intrinsic::x86_avx2_vpdpbsuds_128:
6704 case Intrinsic::x86_avx2_vpdpbsuds_256:
6705 case Intrinsic::x86_avx10_vpdpbsuds_512:
6706 case Intrinsic::x86_avx2_vpdpbuud_128:
6707 case Intrinsic::x86_avx2_vpdpbuud_256:
6708 case Intrinsic::x86_avx10_vpdpbuud_512:
6709 case Intrinsic::x86_avx2_vpdpbuuds_128:
6710 case Intrinsic::x86_avx2_vpdpbuuds_256:
6711 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6712 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6713 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6714 CI->getArgOperand(2)};
6715 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6716 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6717 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6718
6719 NewCall = Builder.CreateCall(NewFn, Args);
6720 break;
6721 }
6722 case Intrinsic::x86_avx512_vpdpwssd_128:
6723 case Intrinsic::x86_avx512_vpdpwssd_256:
6724 case Intrinsic::x86_avx512_vpdpwssd_512:
6725 case Intrinsic::x86_avx512_vpdpwssds_128:
6726 case Intrinsic::x86_avx512_vpdpwssds_256:
6727 case Intrinsic::x86_avx512_vpdpwssds_512:
6728 case Intrinsic::x86_avx2_vpdpwsud_128:
6729 case Intrinsic::x86_avx2_vpdpwsud_256:
6730 case Intrinsic::x86_avx10_vpdpwsud_512:
6731 case Intrinsic::x86_avx2_vpdpwsuds_128:
6732 case Intrinsic::x86_avx2_vpdpwsuds_256:
6733 case Intrinsic::x86_avx10_vpdpwsuds_512:
6734 case Intrinsic::x86_avx2_vpdpwusd_128:
6735 case Intrinsic::x86_avx2_vpdpwusd_256:
6736 case Intrinsic::x86_avx10_vpdpwusd_512:
6737 case Intrinsic::x86_avx2_vpdpwusds_128:
6738 case Intrinsic::x86_avx2_vpdpwusds_256:
6739 case Intrinsic::x86_avx10_vpdpwusds_512:
6740 case Intrinsic::x86_avx2_vpdpwuud_128:
6741 case Intrinsic::x86_avx2_vpdpwuud_256:
6742 case Intrinsic::x86_avx10_vpdpwuud_512:
6743 case Intrinsic::x86_avx2_vpdpwuuds_128:
6744 case Intrinsic::x86_avx2_vpdpwuuds_256:
6745 case Intrinsic::x86_avx10_vpdpwuuds_512:
6746 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6747 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6748 CI->getArgOperand(2)};
6749 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6750 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6751 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6752
6753 NewCall = Builder.CreateCall(NewFn, Args);
6754 break;
6755 }
6756 assert(NewCall && "Should have either set this variable or returned through "
6757 "the default case");
6758 NewCall->takeName(CI);
6759 CI->replaceAllUsesWith(NewCall);
6760 CI->eraseFromParent();
6761}
6762
6764 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6765
6766 // Check if this function should be upgraded and get the replacement function
6767 // if there is one.
6768 Function *NewFn;
6769 if (UpgradeIntrinsicFunction(F, NewFn)) {
6770 // Replace all users of the old function with the new function or new
6771 // instructions. This is not a range loop because the call is deleted.
6772 for (User *U : make_early_inc_range(F->users()))
6773 if (CallBase *CB = dyn_cast<CallBase>(U))
6774 UpgradeIntrinsicCall(CB, NewFn);
6775
6776 // Remove old function, no longer used, from the module.
6777 if (F != NewFn)
6778 F->eraseFromParent();
6779 }
6780}
6781
6783 const unsigned NumOperands = MD.getNumOperands();
6784 if (NumOperands == 0)
6785 return &MD; // Invalid, punt to a verifier error.
6786
6787 // Check if the tag uses struct-path aware TBAA format.
6788 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6789 return &MD;
6790
6791 auto &Context = MD.getContext();
6792 if (NumOperands == 3) {
6793 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6794 MDNode *ScalarType = MDNode::get(Context, Elts);
6795 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6796 Metadata *Elts2[] = {ScalarType, ScalarType,
6799 MD.getOperand(2)};
6800 return MDNode::get(Context, Elts2);
6801 }
6802 // Create a MDNode <MD, MD, offset 0>
6804 Type::getInt64Ty(Context)))};
6805 return MDNode::get(Context, Elts);
6806}
6807
6809 // !tbaa.struct is a list of (offset, size, tag) triples. Upgrade any
6810 // old-style scalar field tag to struct-path form via UpgradeTBAANode.
6811 unsigned NumOperands = MD.getNumOperands();
6812 if (NumOperands == 0 || NumOperands % 3 != 0)
6813 return &MD; // Malformed; leave it for the verifier to reject.
6814
6816 bool Changed = false;
6817 for (unsigned I = 2; I < NumOperands; I += 3) {
6818 auto *Tag = dyn_cast_or_null<MDNode>(Elts[I]);
6819 if (!Tag)
6820 continue;
6821 MDNode *Upgraded = UpgradeTBAANode(*Tag);
6822 if (Upgraded == Tag)
6823 continue;
6824 Elts[I] = Upgraded;
6825 Changed = true;
6826 }
6827 return Changed ? MDNode::get(MD.getContext(), Elts) : &MD;
6828}
6829
6830/// Rebuild the alias scope or domain \p Old with \p Ops, keeping its identity:
6831/// a self reference has to point at the replacement, and a distinct node must
6832/// not be uniqued. The first operand of both a scope and a domain is its name,
6833/// which is either a string or a self reference.
6836 LLVMContext &Context = Old->getContext();
6837 if (Ops[0] != Old)
6838 return Old->isDistinct() ? MDNode::getDistinct(Context, Ops)
6839 : MDNode::get(Context, Ops);
6840
6841 Ops[0] = nullptr;
6842 MDNode *New = MDNode::getDistinct(Context, Ops);
6843 New->replaceOperandWith(0, New);
6844 return New;
6845}
6846
6848 DenseMap<MDNode *, MDNode *> &Upgraded) {
6849 unsigned NumOperands = Domain->getNumOperands();
6850 bool HadDescription = NumOperands == 2;
6851 // Already upgraded, or invalid and left to the verifier.
6852 if (NumOperands == 0 || NumOperands > 2 ||
6853 (HadDescription && mdconst::hasa<ConstantInt>(Domain->getOperand(1))))
6854 return Domain;
6855
6856 MDNode *&Upgrade = Upgraded[Domain];
6857 if (Upgrade)
6858 return Upgrade;
6859
6860 LLVMContext &Context = Domain->getContext();
6862 Domain->getOperand(0),
6864 if (HadDescription)
6865 Ops.push_back(Domain->getOperand(1));
6866
6868 return Upgrade;
6869}
6870
6872 DenseMap<MDNode *, MDNode *> &Upgraded) {
6873 if (MDNode *Upgrade = Upgraded.lookup(Scope))
6874 return Upgrade;
6875
6876 // Extract domain, or leave invalid scope to the verifier.
6877 if (Scope->getNumOperands() < 2 || !isa<MDNode>(Scope->getOperand(1)))
6878 return Scope;
6879 auto *Domain = cast<MDNode>(Scope->getOperand(1));
6880
6881 MDNode *UpgradedDomain = upgradeAliasScopeDomain(Domain, Upgraded);
6882 if (UpgradedDomain == Domain)
6883 return Scope;
6884
6885 SmallVector<Metadata *, 3> Ops(Scope->op_begin(), Scope->op_end());
6886 Ops[1] = UpgradedDomain;
6887 MDNode *Upgrade = rebuildScopeOrDomainNode(Scope, Ops);
6888 Upgraded[Scope] = Upgrade;
6889 return Upgrade;
6890}
6891
6892MDNode *
6894 DenseMap<MDNode *, MDNode *> &UpgradedAliasScopes) {
6895 if (MDNode *Upgrade = UpgradedAliasScopes.lookup(&ScopeList))
6896 return Upgrade;
6897
6899 bool Changed = false;
6900 for (const MDOperand &Op : ScopeList.operands()) {
6901 Metadata *Scope = Op;
6902 if (auto *ScopeNode = dyn_cast<MDNode>(Op))
6903 Scope = upgradeAliasScope(ScopeNode, UpgradedAliasScopes);
6904 Changed |= Scope != Op.get();
6905 Ops.push_back(Scope);
6906 }
6907 if (!Changed)
6908 return &ScopeList;
6909
6910 MDNode *Upgrade = MDNode::get(ScopeList.getContext(), Ops);
6911 UpgradedAliasScopes[&ScopeList] = Upgrade;
6912 return Upgrade;
6913}
6914
6916 Instruction *&Temp) {
6917 if (Opc != Instruction::BitCast)
6918 return nullptr;
6919
6920 Temp = nullptr;
6921 Type *SrcTy = V->getType();
6922 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6923 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6924 LLVMContext &Context = V->getContext();
6925
6926 // We have no information about target data layout, so we assume that
6927 // the maximum pointer size is 64bit.
6928 Type *MidTy = Type::getInt64Ty(Context);
6929 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6930
6931 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6932 }
6933
6934 return nullptr;
6935}
6936
6938 if (Opc != Instruction::BitCast)
6939 return nullptr;
6940
6941 Type *SrcTy = C->getType();
6942 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6943 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6944 LLVMContext &Context = C->getContext();
6945
6946 // We have no information about target data layout, so we assume that
6947 // the maximum pointer size is 64bit.
6948 Type *MidTy = Type::getInt64Ty(Context);
6949
6951 DestTy);
6952 }
6953
6954 return nullptr;
6955}
6956
6957static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6958 if (Flag.getNumOperands() < 3)
6959 return std::nullopt;
6960 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6961 return Name->getString();
6962 return std::nullopt;
6963}
6964
6965/// Check the debug info version number, if it is out-dated, drop the debug
6966/// info. Return true if module is modified.
6968 if (!getIROptions(M.getContext()).auto_upgrade_debug_info)
6969 return false;
6970
6971 llvm::TimeTraceScope timeScope("Upgrade debug info");
6972 // We need to get metadata before the module is verified (i.e., getModuleFlag
6973 // makes assumptions that we haven't verified yet). Carefully extract the flag
6974 // from the metadata.
6975 unsigned Version = 0;
6976 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6977 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6978 if (auto Name = getModuleFlagNameSafely(*Flag))
6979 return *Name == "Debug Info Version";
6980 return false;
6981 });
6982 if (OpIt != ModFlags->op_end()) {
6983 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6984 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6985 Version = CI->getZExtValue();
6986 }
6987 }
6988
6990 bool BrokenDebugInfo = false;
6991 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6992 report_fatal_error("Broken module found, compilation aborted!");
6993 if (!BrokenDebugInfo)
6994 // Everything is ok.
6995 return false;
6996 else {
6997 // Diagnose malformed debug info.
6999 M.getContext().diagnose(Diag);
7000 }
7001 }
7002 bool Modified = StripDebugInfo(M);
7004 // Diagnose a version mismatch.
7006 M.getContext().diagnose(DiagVersion);
7007 }
7008 return Modified;
7009}
7010
7011static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
7012 GlobalValue *GV, const Metadata *V) {
7013 Function *F = cast<Function>(GV);
7014
7015 constexpr StringLiteral DefaultValue = "1";
7016 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
7017 unsigned Length = 0;
7018
7019 if (F->hasFnAttribute(Attr)) {
7020 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
7021 // parse these elements placing them into Vect3
7022 StringRef S = F->getFnAttribute(Attr).getValueAsString();
7023 for (; Length < 3 && !S.empty(); Length++) {
7024 auto [Part, Rest] = S.split(',');
7025 Vect3[Length] = Part.trim();
7026 S = Rest;
7027 }
7028 }
7029
7030 const unsigned Dim = DimC - 'x';
7031 assert(Dim < 3 && "Unexpected dim char");
7032
7033 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
7034
7035 // local variable required for StringRef in Vect3 to point to.
7036 const std::string VStr = llvm::utostr(VInt);
7037 Vect3[Dim] = VStr;
7038 Length = std::max(Length, Dim + 1);
7039
7040 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
7041 F->addFnAttr(Attr, NewAttr);
7042}
7043
7044static inline bool isXYZ(StringRef S) {
7045 return S == "x" || S == "y" || S == "z";
7046}
7047
7049 const Metadata *V) {
7050 if (K == "kernel") {
7052 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
7053 return true;
7054 }
7055 if (K == "align") {
7056 // V is a bitfeild specifying two 16-bit values. The alignment value is
7057 // specfied in low 16-bits, The index is specified in the high bits. For the
7058 // index, 0 indicates the return value while higher values correspond to
7059 // each parameter (idx = param + 1).
7060 const uint64_t AlignIdxValuePair =
7061 mdconst::extract<ConstantInt>(V)->getZExtValue();
7062 const unsigned Idx = (AlignIdxValuePair >> 16);
7063 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
7064 cast<Function>(GV)->addAttributeAtIndex(
7065 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
7066 return true;
7067 }
7068 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
7069 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
7071 return true;
7072 }
7073 if (K == "minctasm") {
7074 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
7075 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
7076 return true;
7077 }
7078 if (K == "maxnreg") {
7079 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
7080 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
7081 return true;
7082 }
7083 if (K.consume_front("maxntid") && isXYZ(K)) {
7085 return true;
7086 }
7087 if (K.consume_front("reqntid") && isXYZ(K)) {
7089 return true;
7090 }
7091 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
7093 return true;
7094 }
7095 if (K == "grid_constant") {
7096 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
7097 for (const auto &Op : cast<MDNode>(V)->operands()) {
7098 // For some reason, the index is 1-based in the metadata. Good thing we're
7099 // able to auto-upgrade it!
7100 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
7101 cast<Function>(GV)->addParamAttr(Index, Attr);
7102 }
7103 return true;
7104 }
7105
7106 return false;
7107}
7108
7110 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
7111 if (!NamedMD)
7112 return;
7113
7114 SmallVector<MDNode *, 8> NewNodes;
7116 for (MDNode *MD : NamedMD->operands()) {
7117 if (!SeenNodes.insert(MD).second)
7118 continue;
7119
7120 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
7121 if (!GV)
7122 continue;
7123
7124 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
7125
7126 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
7127 // Each nvvm.annotations metadata entry will be of the following form:
7128 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
7129 // start index = 1, to skip the global variable key
7130 // increment = 2, to skip the value for each property-value pairs
7131 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
7132 MDString *K = cast<MDString>(MD->getOperand(j));
7133 const MDOperand &V = MD->getOperand(j + 1);
7134 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
7135 if (!Upgraded)
7136 NewOperands.append({K, V});
7137 }
7138
7139 if (NewOperands.size() > 1)
7140 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
7141 }
7142
7143 NamedMD->clearOperands();
7144 for (MDNode *N : NewNodes)
7145 NamedMD->addOperand(N);
7146}
7147
7148/// This checks for objc retain release marker which should be upgraded. It
7149/// returns true if module is modified.
7151 bool Changed = false;
7152 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
7153 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
7154 if (ModRetainReleaseMarker) {
7155 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
7156 if (Op) {
7157 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
7158 if (ID) {
7159 SmallVector<StringRef, 4> ValueComp;
7160 ID->getString().split(ValueComp, "#");
7161 if (ValueComp.size() == 2) {
7162 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
7163 ID = MDString::get(M.getContext(), NewValue);
7164 }
7165 M.addModuleFlag(Module::Error, MarkerKey, ID);
7166 M.eraseNamedMetadata(ModRetainReleaseMarker);
7167 Changed = true;
7168 }
7169 }
7170 }
7171 return Changed;
7172}
7173
7175 // This lambda converts normal function calls to ARC runtime functions to
7176 // intrinsic calls.
7177 auto UpgradeToIntrinsic = [&](const char *OldFunc,
7178 llvm::Intrinsic::ID IntrinsicFunc) {
7179 Function *Fn = M.getFunction(OldFunc);
7180
7181 if (!Fn)
7182 return;
7183
7184 Function *NewFn =
7185 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
7186
7187 for (User *U : make_early_inc_range(Fn->users())) {
7189 if (!CI || CI->getCalledFunction() != Fn)
7190 continue;
7191
7192 IRBuilder<> Builder(CI->getIterator());
7193 FunctionType *NewFuncTy = NewFn->getFunctionType();
7195
7196 // Don't upgrade the intrinsic if it's not valid to bitcast the return
7197 // value to the return type of the old function.
7198 if (NewFuncTy->getReturnType() != CI->getType() &&
7199 !CastInst::castIsValid(Instruction::BitCast, CI,
7200 NewFuncTy->getReturnType()))
7201 continue;
7202
7203 bool InvalidCast = false;
7204
7205 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
7206 Value *Arg = CI->getArgOperand(I);
7207
7208 // Bitcast argument to the parameter type of the new function if it's
7209 // not a variadic argument.
7210 if (I < NewFuncTy->getNumParams()) {
7211 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
7212 // to the parameter type of the new function.
7213 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
7214 NewFuncTy->getParamType(I))) {
7215 InvalidCast = true;
7216 break;
7217 }
7218 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
7219 }
7220 Args.push_back(Arg);
7221 }
7222
7223 if (InvalidCast)
7224 continue;
7225
7226 // Create a call instruction that calls the new function.
7227 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
7228 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
7229 NewCall->takeName(CI);
7230
7231 // Bitcast the return value back to the type of the old call.
7232 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
7233
7234 if (!CI->use_empty())
7235 CI->replaceAllUsesWith(NewRetVal);
7236 CI->eraseFromParent();
7237 }
7238
7239 if (Fn->use_empty())
7240 Fn->eraseFromParent();
7241 };
7242
7243 // Unconditionally convert a call to "clang.arc.use" to a call to
7244 // "llvm.objc.clang.arc.use".
7245 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7246
7247 // Upgrade the retain release marker. If there is no need to upgrade
7248 // the marker, that means either the module is already new enough to contain
7249 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7251 return;
7252
7253 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7254 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7255 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7256 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7257 {"objc_autoreleaseReturnValue",
7258 llvm::Intrinsic::objc_autoreleaseReturnValue},
7259 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7260 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7261 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7262 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7263 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7264 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7265 {"objc_release", llvm::Intrinsic::objc_release},
7266 {"objc_retain", llvm::Intrinsic::objc_retain},
7267 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7268 {"objc_retainAutoreleaseReturnValue",
7269 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7270 {"objc_retainAutoreleasedReturnValue",
7271 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7272 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7273 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7274 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7275 {"objc_unsafeClaimAutoreleasedReturnValue",
7276 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7277 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7278 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7279 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7280 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7281 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7282 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7283 {"objc_arc_annotation_topdown_bbstart",
7284 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7285 {"objc_arc_annotation_topdown_bbend",
7286 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7287 {"objc_arc_annotation_bottomup_bbstart",
7288 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7289 {"objc_arc_annotation_bottomup_bbend",
7290 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7291
7292 for (auto &I : RuntimeFuncs)
7293 UpgradeToIntrinsic(I.first, I.second);
7294}
7295
7296// Upgrade the way signing of pointers to init/fini functions is described.
7297//
7298// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7299// constants, if signing was requested. After the upgrade, these arrays contain
7300// plain function pointers and the desired signing schema is described via a
7301// pair of module flags.
7302//
7303// Note that the upgrade is only performed if all elements of *both* arrays
7304// agree on a common signing schema.
7306 // As we cannot always decide whether the particular module should have
7307 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7308 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7309 // returns as soon as it spots any non-signed init/fini pointer: either we
7310 // should request non-signed pointers (safe to omit both flags) or there is
7311 // no common schema (and thus we do not modify anything).
7312 //
7313 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7314 // or whether we should request address diversity in addition to the basic
7315 // constant diversity. There is no value representing "decided not to sign"
7316 // for the reasons explained above.
7317 std::optional<bool> UseAddressDisc;
7318
7319 // Do not attempt upgrading if the new module flags already exist.
7320 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7321 for (const MDNode *Flag : ModFlags->operands()) {
7322 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7323 if (Name && (*Name == "ptrauth-init-fini" ||
7324 *Name == "ptrauth-init-fini-address-discrimination"))
7325 return false;
7326 }
7327 }
7328
7329 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7330 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7331 constexpr unsigned ExpectedAddressMarker = 1;
7332
7333 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7334 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7335 return nullptr; // Nothing to upgrade or unknown pattern found.
7336
7337 bool HasAddressDisc;
7338 if (!CPA->hasAddressDiscriminator())
7339 HasAddressDisc = false;
7340 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7341 HasAddressDisc = true;
7342 else
7343 return nullptr; // Unknown pattern.
7344
7345 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7346 return nullptr; // Disagreement with the decided mode.
7347
7348 UseAddressDisc = HasAddressDisc;
7349 return CPA->getPointer();
7350 };
7351
7352 // Do not apply any changes until we know the upgrade is non-ambiguous.
7353 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7354 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7355
7356 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7357 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7358 if (!GV || !GV->hasInitializer())
7359 continue; // Skip, but it is okay to upgrade the other variable.
7360
7361 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7362 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7363 return false;
7364
7365 std::vector<Constant *> NewStructors;
7366 NewStructors.reserve(OldStructorsArray->getNumOperands());
7367
7368 for (Use &U : OldStructorsArray->operands()) {
7369 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7370 if (!Structor || Structor->getNumOperands() != 3)
7371 return false;
7372
7373 Constant *Prio = Structor->getOperand(0);
7374 Constant *Func = Structor->getOperand(1);
7375 Constant *Arg = Structor->getOperand(2);
7376
7377 Func = UpgradeSinglePointer(Func);
7378 if (!Func)
7379 return false;
7380
7381 NewStructors.push_back(
7382 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7383 }
7384
7385 Constant *NewInit =
7386 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7387 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7388 }
7389
7390 if (GlobalArraysToUpgrade.empty())
7391 return false;
7392 assert(UseAddressDisc.has_value());
7393
7394 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7395 GV->setInitializer(NewInit);
7396
7397 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7398 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7399 *UseAddressDisc);
7400
7401 return true;
7402}
7403
7405 bool Changed = false;
7407
7408 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7409 if (!ModFlags)
7410 return Changed;
7411
7412 bool HasObjCFlag = false, HasClassProperties = false;
7413 bool HasClassROSigning = false;
7414 bool HasSwiftVersionFlag = false;
7415 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7416 uint32_t SwiftABIVersion;
7417 auto Int8Ty = Type::getInt8Ty(M.getContext());
7418 auto Int32Ty = Type::getInt32Ty(M.getContext());
7419
7420 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7421 MDNode *Op = ModFlags->getOperand(I);
7422 if (Op->getNumOperands() != 3)
7423 continue;
7424 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7425 if (!ID)
7426 continue;
7427 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7428 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7429 Type::getInt32Ty(M.getContext()), B)),
7430 MDString::get(M.getContext(), ID->getString()),
7431 Op->getOperand(2)};
7432 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7433 Changed = true;
7434 };
7435
7436 if (ID->getString() == "Objective-C Image Info Version")
7437 HasObjCFlag = true;
7438 if (ID->getString() == "Objective-C Class Properties")
7439 HasClassProperties = true;
7440 if (ID->getString() == "Objective-C Enforce ClassRO Pointer Signing")
7441 HasClassROSigning = true;
7442 // Upgrade PIC from Error/Max to Min.
7443 if (ID->getString() == "PIC Level") {
7444 if (auto *Behavior =
7446 uint64_t V = Behavior->getLimitedValue();
7447 if (V == Module::Error || V == Module::Max)
7448 SetBehavior(Module::Min);
7449 }
7450 }
7451 // Upgrade "PIE Level" from Error to Max.
7452 if (ID->getString() == "PIE Level")
7453 if (auto *Behavior =
7455 if (Behavior->getLimitedValue() == Module::Error)
7456 SetBehavior(Module::Max);
7457
7458 // Upgrade branch protection and return address signing module flags. The
7459 // module flag behavior for these fields were Error and now they are Min.
7460 // The one exception is "sign-return-address-harden".
7461 if (ID->getString() == "branch-target-enforcement" ||
7462 (ID->getString().starts_with("sign-return-address") &&
7463 ID->getString() != "sign-return-address-harden")) {
7464 if (auto *Behavior =
7466 if (Behavior->getLimitedValue() == Module::Error) {
7467 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7468 Metadata *Ops[3] = {
7469 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7470 Op->getOperand(1), Op->getOperand(2)};
7471 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7472 Changed = true;
7473 }
7474 }
7475 }
7476
7477 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7478 // section name so that llvm-lto will not complain about mismatching
7479 // module flags that is functionally the same.
7480 if (ID->getString() == "Objective-C Image Info Section") {
7481 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7482 SmallVector<StringRef, 4> ValueComp;
7483 Value->getString().split(ValueComp, " ");
7484 if (ValueComp.size() != 1) {
7485 std::string NewValue;
7486 for (auto &S : ValueComp)
7487 NewValue += S.str();
7488 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7489 MDString::get(M.getContext(), NewValue)};
7490 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7491 Changed = true;
7492 }
7493 }
7494 }
7495
7496 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7497 // If the higher bits are set, it adds new module flag for swift info.
7498 if (ID->getString() == "Objective-C Garbage Collection") {
7499 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7500 if (Md) {
7501 assert(Md->getValue() && "Expected non-empty metadata");
7502 auto Type = Md->getValue()->getType();
7503 if (Type == Int8Ty)
7504 continue;
7505 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7506 if ((Val & 0xff) != Val) {
7507 HasSwiftVersionFlag = true;
7508 SwiftABIVersion = (Val & 0xff00) >> 8;
7509 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7510 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7511 }
7512 Metadata *Ops[3] = {
7513 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7514 Op->getOperand(1),
7515 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7516 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7517 Changed = true;
7518 }
7519 }
7520
7521 if (ID->getString() == "amdgpu_code_object_version") {
7522 Metadata *Ops[3] = {
7523 Op->getOperand(0),
7524 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7525 Op->getOperand(2)};
7526 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7527 Changed = true;
7528 }
7529
7530 // clang/PowerPC used to use "float-abi" to describe the long double format;
7531 // it has been renamed to "long-double-type", with its values changed to the
7532 // corresponding IR floating-point type names.
7533 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7535 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7536 Format = S->getString();
7537
7538 // The "float-abi" key is now reserved for the target-independent
7539 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7540 // (including unrecognized ones, which were never valid) to the default.
7542 LongDoubleFormat NewFormat =
7544 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7545 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7547 Metadata *Ops[3] = {
7548 Op->getOperand(0),
7549 MDString::get(M.getContext(), "long-double-type"),
7550 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7551 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7552 Changed = true;
7553 }
7554 }
7555 }
7556
7557 // "Objective-C Class Properties" is recently added for Objective-C. We
7558 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7559 // flag of value 0, so we can correclty downgrade this flag when trying to
7560 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7561 // this module flag.
7562 if (HasObjCFlag && !HasClassProperties) {
7563 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7564 (uint32_t)0);
7565 Changed = true;
7566 }
7567
7568 // Likewise, ObjC bitcode from before "Objective-C Enforce ClassRO Pointer
7569 // Signing" was added does not sign class_ro_t pointers. Give it the flag with
7570 // value 0, so that linking it with bitcode that signs them does not set
7571 // SignedClassRO.
7572 if (HasObjCFlag && !HasClassROSigning) {
7573 M.addModuleFlag(Module::Override,
7574 "Objective-C Enforce ClassRO Pointer Signing", (uint32_t)0);
7575 Changed = true;
7576 }
7577
7578 if (HasSwiftVersionFlag) {
7579 M.addModuleFlag(Module::Error, "Swift ABI Version",
7580 SwiftABIVersion);
7581 M.addModuleFlag(Module::Error, "Swift Major Version",
7582 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7583 M.addModuleFlag(Module::Error, "Swift Minor Version",
7584 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7585 Changed = true;
7586 }
7587
7588 return Changed;
7589}
7590
7592 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7593 // If this metadata has operands, we expect all of them to be either from
7594 // before or from after the format change handled here, so we can bail out
7595 // fast if the first (if any) operands is of the new format.
7596 auto MatchesVersion = [](const MDNode *Op) {
7597 return Op->getNumOperands() >= 3 &&
7598 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7599 cast<ConstantAsMetadata>(Op->getOperand(2))
7600 ->getType()
7601 ->isIntegerTy(64);
7602 };
7603
7604 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7605 MatchesVersion(CFIConsts->getOperand(0)))
7606 return false;
7607
7608 bool Changed = false;
7609 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7610 MDNode *Op = CFIConsts->getOperand(I);
7611 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7612 assert(Op->getNumOperands() >= 2 &&
7613 "Expected at least 2 operands - name and linkage type");
7614 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7615 StringRef Name = NameMD->getString();
7618
7620 Elts.push_back(Op->getOperand(0));
7621 Elts.push_back(Op->getOperand(1));
7623 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7624
7625 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7626 Elts.push_back(Op->getOperand(J));
7627
7628 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7629 Changed = true;
7630 }
7631
7632 return Changed;
7633}
7634
7636 auto TrimSpaces = [](StringRef Section) -> std::string {
7637 SmallVector<StringRef, 5> Components;
7638 Section.split(Components, ',');
7639
7640 SmallString<32> Buffer;
7641 raw_svector_ostream OS(Buffer);
7642
7643 for (auto Component : Components)
7644 OS << ',' << Component.trim();
7645
7646 return std::string(OS.str().substr(1));
7647 };
7648
7649 for (auto &GV : M.globals()) {
7650 if (!GV.hasSection())
7651 continue;
7652
7653 StringRef Section = GV.getSection();
7654
7655 if (!Section.starts_with("__DATA, __objc_catlist"))
7656 continue;
7657
7658 // __DATA, __objc_catlist, regular, no_dead_strip
7659 // __DATA,__objc_catlist,regular,no_dead_strip
7660 GV.setSection(TrimSpaces(Section));
7661 }
7662}
7663
7665 DenseMap<MDNode *, MDNode *> UpgradedAliasScopes;
7666
7667 for (Function &F : M)
7668 for (Instruction &I : instructions(F)) {
7669 for (auto Kind : {LLVMContext::MD_alias_scope, LLVMContext::MD_noalias}) {
7670 if (MDNode *Old = I.getMetadata(Kind)) {
7671 MDNode *New = UpgradeAliasScopeList(*Old, UpgradedAliasScopes);
7672 if (New != Old)
7673 I.setMetadata(Kind, New);
7674 }
7675 }
7676 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I)) {
7677 unsigned ArgNo = Intrinsic::NoAliasScopeDeclScopeArg;
7678 if (Decl->getNumOperands() > ArgNo)
7679 if (auto *ListAsValue =
7680 dyn_cast<MetadataAsValue>(Decl->getOperand(ArgNo)))
7681 if (auto *List = dyn_cast<MDNode>(ListAsValue->getMetadata())) {
7682 MDNode *New = UpgradeAliasScopeList(*List, UpgradedAliasScopes);
7683 if (New != List)
7684 Decl->setOperand(ArgNo,
7685 MetadataAsValue::get(Decl->getContext(), New));
7686 }
7687 }
7688 }
7689}
7690
7691namespace {
7692// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7693// callsites within a function that did not also have the strictfp attribute.
7694// Since 10.0, if strict FP semantics are needed within a function, the
7695// function must have the strictfp attribute and all calls within the function
7696// must also have the strictfp attribute. This latter restriction is
7697// necessary to prevent unwanted libcall simplification when a function is
7698// being cloned (such as for inlining).
7699//
7700// The "dangling" strictfp attribute usage was only used to prevent constant
7701// folding and other libcall simplification. The nobuiltin attribute on the
7702// callsite has the same effect.
7703struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7704 StrictFPUpgradeVisitor() = default;
7705
7706 void visitCallBase(CallBase &Call) {
7707 if (!Call.isStrictFP())
7708 return;
7710 return;
7711 // If we get here, the caller doesn't have the strictfp attribute
7712 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7713 Call.removeFnAttr(Attribute::StrictFP);
7714 Call.addFnAttr(Attribute::NoBuiltin);
7715 }
7716};
7717
7718/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7719struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7720 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7721 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7722
7723 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7724 if (!RMW.isFloatingPointOperation())
7725 return;
7726
7727 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7728 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7729 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7730 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7731 }
7732};
7733} // namespace
7734
7736 // If a function definition doesn't have the strictfp attribute,
7737 // convert any callsite strictfp attributes to nobuiltin.
7738 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7739 StrictFPUpgradeVisitor SFPV;
7740 SFPV.visit(F);
7741 }
7742
7743 // Remove all incompatibile attributes from function.
7744 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7745 F.getReturnType(), F.getAttributes().getRetAttrs()));
7746 for (auto &Arg : F.args())
7747 Arg.removeAttrs(
7748 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7749
7750 bool AddingAttrs = false, RemovingAttrs = false;
7751 AttrBuilder AttrsToAdd(F.getContext());
7752 AttributeMask AttrsToRemove;
7753
7754 // Older versions of LLVM treated an "implicit-section-name" attribute
7755 // similarly to directly setting the section on a Function.
7756 if (Attribute A = F.getFnAttribute("implicit-section-name");
7757 A.isValid() && A.isStringAttribute()) {
7758 F.setSection(A.getValueAsString());
7759 AttrsToRemove.addAttribute("implicit-section-name");
7760 RemovingAttrs = true;
7761 }
7762
7763 if (Attribute A = F.getFnAttribute("nooutline");
7764 A.isValid() && A.isStringAttribute()) {
7765 AttrsToRemove.addAttribute("nooutline");
7766 AttrsToAdd.addAttribute(Attribute::NoOutline);
7767 AddingAttrs = RemovingAttrs = true;
7768 }
7769
7770 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7771 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7772 AttrsToRemove.addAttribute("uniform-work-group-size");
7773 RemovingAttrs = true;
7774 if (A.getValueAsString() == "true") {
7775 AttrsToAdd.addAttribute("uniform-work-group-size");
7776 AddingAttrs = true;
7777 }
7778 }
7779
7780 if (!F.empty()) {
7781 // For some reason this is called twice, and the first time is before any
7782 // instructions are loaded into the body.
7783
7784 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7785 A.isValid()) {
7786
7787 if (A.getValueAsBool()) {
7788 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7789 Visitor.visit(F);
7790 }
7791
7792 // We will leave behind dead attribute uses on external declarations, but
7793 // clang never added these to declarations anyway.
7794 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7795 RemovingAttrs = true;
7796 }
7797 }
7798
7799 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7800 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7801
7802 bool HandleDenormalMode = false;
7803
7804 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7805 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7806 if (ParsedMode.isValid()) {
7807 DenormalFPMath = ParsedMode;
7808 AttrsToRemove.addAttribute("denormal-fp-math");
7809 AddingAttrs = RemovingAttrs = true;
7810 HandleDenormalMode = true;
7811 }
7812 }
7813
7814 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7815 Attr.isValid()) {
7816 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7817 if (ParsedMode.isValid()) {
7818 DenormalFPMathF32 = ParsedMode;
7819 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7820 AddingAttrs = RemovingAttrs = true;
7821 HandleDenormalMode = true;
7822 }
7823 }
7824
7825 if (HandleDenormalMode)
7826 AttrsToAdd.addDenormalFPEnvAttr(
7827 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7828
7829 if (RemovingAttrs)
7830 F.removeFnAttrs(AttrsToRemove);
7831
7832 if (AddingAttrs)
7833 F.addFnAttrs(AttrsToAdd);
7834}
7835
7836// Check if the function attribute is not present and set it.
7838 StringRef Value) {
7839 if (!F.hasFnAttribute(FnAttrName)) {
7840 F.addFnAttr(FnAttrName, Value);
7841 LLVM_DEBUG(dbgs() << "Set attribute: " << FnAttrName << "=\"" << Value
7842 << "\", function: " << F.getName() << "\n");
7843 }
7844}
7845
7846// Check if the function attribute is not present and set it if needed.
7847// If the attribute is "false" then removes it.
7848// If the attribute is "true" resets it to a valueless attribute.
7849static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7850 if (!F.hasFnAttribute(FnAttrName)) {
7851 if (Set) {
7852 F.addFnAttr(FnAttrName);
7853 LLVM_DEBUG(dbgs() << "Added attribute: " << FnAttrName
7854 << ", function: " << F.getName() << "\n");
7855 }
7856 } else {
7857 auto A = F.getFnAttribute(FnAttrName);
7858 if ("false" == A.getValueAsString()) {
7859 F.removeFnAttr(FnAttrName);
7860 LLVM_DEBUG(dbgs() << "Removed attribute: " << FnAttrName
7861 << "=\"false\", function: " << F.getName() << "\n");
7862 } else if ("true" == A.getValueAsString()) {
7863 F.removeFnAttr(FnAttrName);
7864 F.addFnAttr(FnAttrName);
7865 LLVM_DEBUG(dbgs() << "Converted attribute: " << FnAttrName
7866 << "=\"true\", function: " << F.getName() << "\n");
7867 }
7868 }
7869}
7870
7872 StringRef Key, uint32_t Val) {
7873 M.setModuleFlag(Behavior, Key, Val);
7874 LLVM_DEBUG(dbgs() << "Converted module flag: " << "{" << Behavior << ", "
7875 << Key << ", " << Val << "}\n");
7876}
7877
7879 Triple T(M.getTargetTriple());
7880 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7881 return;
7882
7883 uint64_t BTEValue = 0;
7884 uint64_t BPPLRValue = 0;
7885 uint64_t GCSValue = 0;
7886 uint64_t SRAValue = 0;
7887 uint64_t SRAALLValue = 0;
7888 uint64_t SRABKeyValue = 0;
7889
7890 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7891 if (ModFlags) {
7892 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7893 MDNode *Op = ModFlags->getOperand(I);
7894 if (Op->getNumOperands() != 3)
7895 continue;
7896
7897 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7898 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7899 if (!ID || !CI)
7900 continue;
7901
7902 StringRef IDStr = ID->getString();
7903 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7904 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7905 : IDStr == "guarded-control-stack" ? &GCSValue
7906 : IDStr == "sign-return-address" ? &SRAValue
7907 : IDStr == "sign-return-address-all" ? &SRAALLValue
7908 : IDStr == "sign-return-address-with-bkey"
7909 ? &SRABKeyValue
7910 : nullptr;
7911 if (!ValPtr)
7912 continue;
7913
7914 *ValPtr = CI->getZExtValue();
7915 if (*ValPtr == 2)
7916 return;
7917
7918 LLVM_DEBUG(dbgs() << "Found module flag: " << IDStr << "(" << *ValPtr
7919 << ")\n");
7920 }
7921 }
7922
7923 bool BTE = BTEValue == 1;
7924 bool BPPLR = BPPLRValue == 1;
7925 bool GCS = GCSValue == 1;
7926 bool SRA = SRAValue == 1;
7927
7928 StringRef SignTypeValue = "non-leaf";
7929 if (SRA && SRAALLValue == 1)
7930 SignTypeValue = "all";
7931
7932 StringRef SignKeyValue = "a_key";
7933 if (SRA && SRABKeyValue == 1)
7934 SignKeyValue = "b_key";
7935
7936 for (Function &F : M.getFunctionList()) {
7937 if (F.isDeclaration())
7938 continue;
7939
7940 if (SRA) {
7941 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7942 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7943 } else {
7944 if (auto A = F.getFnAttribute("sign-return-address");
7945 A.isValid() && "none" == A.getValueAsString()) {
7946 F.removeFnAttr("sign-return-address");
7947 F.removeFnAttr("sign-return-address-key");
7948 }
7949 }
7950 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7951 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7952 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7953 }
7954
7955 if (BTE)
7956 ConvertModuleFlag(M, llvm::Module::Min, "branch-target-enforcement", 2);
7957 if (BPPLR)
7958 ConvertModuleFlag(M, llvm::Module::Min, "branch-protection-pauth-lr", 2);
7959 if (GCS)
7960 ConvertModuleFlag(M, llvm::Module::Min, "guarded-control-stack", 2);
7961 if (SRA) {
7962 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address", 2);
7963 if (SRAALLValue == 1)
7964 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-all", 2);
7965 if (SRABKeyValue == 1) {
7966 ConvertModuleFlag(M, llvm::Module::Min, "sign-return-address-with-bkey",
7967 2);
7968 }
7969 }
7970}
7971
7972/// Return the replacement tags if \p T still uses a removed two-operand form.
7974 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7975 return nullptr;
7976 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7977 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7978}
7979
7980/// Build the single-operand node that replaces a boolean operand: nonzero
7981/// selects the enable tag, zero the disable tag.
7983 const BooleanLoopTags &Tags,
7984 const MDOperand &Op) {
7985 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7986 return MDTuple::get(C,
7987 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7988}
7989
7990static bool isOldLoopArgument(Metadata *MD) {
7991 auto *T = dyn_cast_or_null<MDTuple>(MD);
7992 if (!T)
7993 return false;
7994 if (T->getNumOperands() < 1)
7995 return false;
7996 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7997 if (!S)
7998 return false;
7999 if (S->getString().starts_with("llvm.vectorizer."))
8000 return true;
8001 return getOldBooleanLoopTags(T) != nullptr;
8002}
8003
8005 StringRef OldPrefix = "llvm.vectorizer.";
8006 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
8007
8008 if (OldTag == "llvm.vectorizer.unroll")
8009 return MDString::get(C, "llvm.loop.interleave.count");
8010
8011 return MDString::get(
8012 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
8013 .str());
8014}
8015
8017 auto *T = dyn_cast_or_null<MDTuple>(MD);
8018 if (!T)
8019 return MD;
8020 if (T->getNumOperands() < 1)
8021 return MD;
8022 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
8023 if (!OldTag)
8024 return MD;
8025
8026 LLVMContext &C = T->getContext();
8027
8028 /// Rewrite a removed two-operand boolean form to the single-operand pair.
8029 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
8030 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
8031
8032 if (!OldTag->getString().starts_with("llvm.vectorizer."))
8033 return MD;
8034
8035 // This has an old tag. Upgrade it.
8036 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
8037
8038 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
8039 // vectorize.enable/disable pair, not a two-operand enable node.
8040 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
8041 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
8042 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
8043
8045 Ops.reserve(T->getNumOperands());
8046 Ops.push_back(NewTag);
8047 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
8048 Ops.push_back(T->getOperand(I));
8049
8050 return MDTuple::get(C, Ops);
8051}
8052
8054 auto *T = dyn_cast<MDTuple>(&N);
8055 if (!T)
8056 return &N;
8057
8058 if (none_of(T->operands(), isOldLoopArgument))
8059 return &N;
8060
8061 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
8062 // any MDNode carrying those tags with more than one operand, so a leftover
8063 // reference (from the distinct loop-ID) would still trigger a diagnostic.
8064 // In-place mutation is safe on distinct MDNodes.
8065 if (T->isDistinct()) {
8066 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
8067 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
8068 if (OpT && getOldBooleanLoopTags(OpT))
8069 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
8070 }
8071 if (none_of(T->operands(), isOldLoopArgument))
8072 return &N;
8073 }
8074
8075 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
8076 // attachment; the original distinct loop-ID is kept as the first operand.
8078 Ops.reserve(T->getNumOperands());
8079 for (Metadata *MD : T->operands())
8080 Ops.push_back(upgradeLoopArgument(MD));
8081
8082 return MDTuple::get(T->getContext(), Ops);
8083}
8084
8086 Triple T(TT);
8087 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
8088 // the address space of globals to 1. This does not apply to SPIRV Logical.
8089 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
8090 !DL.contains("-G") && !DL.starts_with("G")) {
8091 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
8092 }
8093
8094 if (T.isLoongArch64() || T.isRISCV64()) {
8095 // Make i32 a native type for 64-bit LoongArch and RISC-V.
8096 auto I = DL.find("-n64-");
8097 if (I != StringRef::npos)
8098 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
8099 return DL.str();
8100 }
8101
8102 // AMDGPU data layout upgrades.
8103 std::string Res = DL.str();
8104 if (T.isAMDGPU()) {
8105 // Define address spaces for constants.
8106 if (!DL.contains("-G") && !DL.starts_with("G"))
8107 Res.append(Res.empty() ? "G1" : "-G1");
8108
8109 // AMDGCN data layout upgrades.
8110 if (T.isAMDGCN()) {
8111
8112 // Add missing non-integral declarations.
8113 // This goes before adding new address spaces to prevent incoherent string
8114 // values.
8115 if (!DL.contains("-ni") && !DL.starts_with("ni"))
8116 Res.append("-ni:7:8:9");
8117 // Update ni:7 to ni:7:8:9.
8118 if (DL.ends_with("ni:7"))
8119 Res.append(":8:9");
8120 if (DL.ends_with("ni:7:8"))
8121 Res.append(":9");
8122
8123 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
8124 // resources) An empty data layout has already been upgraded to G1 by now.
8125 if (!DL.contains("-p7") && !DL.starts_with("p7"))
8126 Res.append("-p7:160:256:256:32");
8127 if (!DL.contains("-p8") && !DL.starts_with("p8"))
8128 Res.append("-p8:128:128:128:48");
8129 constexpr StringRef OldP8("-p8:128:128-");
8130 if (DL.contains(OldP8))
8131 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
8132 if (!DL.contains("-p9") && !DL.starts_with("p9"))
8133 Res.append("-p9:192:256:256:32");
8134
8135 // Add sizing for address space 10 through 15.
8136 // AS 10-14 are reserved and defaulted to 32:32
8137 // AS 15 is in use and is 32:32.
8138 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
8139 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
8140 Res.append(("-" + AS + ":32:32").str());
8141 }
8142 }
8143
8144 // Upgrade the ELF mangling mode.
8145 if (!DL.contains("m:e"))
8146 Res = Res.empty() ? "m:e" : "m:e-" + Res;
8147
8148 return Res;
8149 }
8150
8151 if (T.isSystemZ() && !DL.empty()) {
8152 // Make sure the stack alignment is present.
8153 if (!DL.contains("-S64"))
8154 return "E-S64" + DL.drop_front(1).str();
8155 return DL.str();
8156 }
8157
8158 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
8159 // If the datalayout matches the expected format, add pointer size address
8160 // spaces to the datalayout.
8161 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
8162 if (!DL.contains(AddrSpaces)) {
8164 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
8165 if (R.match(Res, &Groups))
8166 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
8167 }
8168 };
8169
8170 // AArch64 data layout upgrades.
8171 if (T.isAArch64()) {
8172 // Add "-Fn32"
8173 if (!DL.empty() && !DL.contains("-Fn32"))
8174 Res.append("-Fn32");
8175 AddPtr32Ptr64AddrSpaces();
8176 return Res;
8177 }
8178
8179 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
8180 T.isWasm()) {
8181 // Mips64 with o32 ABI did not add "-i128:128".
8182 // Add "-i128:128"
8183 std::string I64 = "-i64:64";
8184 std::string I128 = "-i128:128";
8185 if (!StringRef(Res).contains(I128)) {
8186 size_t Pos = Res.find(I64);
8187 if (Pos != size_t(-1))
8188 Res.insert(Pos + I64.size(), I128);
8189 }
8190 }
8191
8192 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
8193 size_t Pos = Res.find("-S128");
8194 if (Pos == StringRef::npos)
8195 Pos = Res.size();
8196 Res.insert(Pos, "-f64:32:64");
8197 }
8198
8199 // ARM data layout upgrades.
8200 // Add -Fi8 if a -F has not already been specified.
8201 if (T.isARM() && !DL.empty() && !DL.contains("Fi") && !DL.contains("Fn")) {
8202 const StringRef p3232 = "p:32:32";
8203 size_t Pos = Res.find(p3232);
8204 if (Pos != StringRef::npos)
8205 Res.insert(Pos + p3232.size(), "-Fi8");
8206 }
8207
8208 if (!T.isX86())
8209 return Res;
8210
8211 AddPtr32Ptr64AddrSpaces();
8212
8213 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
8214 // for i128 operations prior to this being reflected in the data layout, and
8215 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
8216 // boundaries, so although this is a breaking change, the upgrade is expected
8217 // to fix more IR than it breaks.
8218 // Intel MCU is an exception and uses 4-byte-alignment.
8219 if (!T.isOSIAMCU()) {
8220 std::string I128 = "-i128:128";
8221 if (StringRef Ref = Res; !Ref.contains(I128)) {
8223 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
8224 if (R.match(Res, &Groups))
8225 Res = (Groups[1] + I128 + Groups[3]).str();
8226 }
8227 }
8228
8229 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
8230 // Raising the alignment is safe because Clang did not produce f80 values in
8231 // the MSVC environment before this upgrade was added.
8232 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
8233 StringRef Ref = Res;
8234 auto I = Ref.find("-f80:32-");
8235 if (I != StringRef::npos)
8236 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
8237 }
8238
8239 return Res;
8240}
8241
8242void llvm::UpgradeAttributes(AttrBuilder &B) {
8243 StringRef FramePointer;
8244 Attribute A = B.getAttribute("no-frame-pointer-elim");
8245 if (A.isValid()) {
8246 // The value can be "true" or "false".
8247 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
8248 B.removeAttribute("no-frame-pointer-elim");
8249 }
8250 if (B.contains("no-frame-pointer-elim-non-leaf")) {
8251 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
8252 if (FramePointer != "all")
8253 FramePointer = "non-leaf";
8254 B.removeAttribute("no-frame-pointer-elim-non-leaf");
8255 }
8256 if (!FramePointer.empty())
8257 B.addAttribute("frame-pointer", FramePointer);
8258
8259 A = B.getAttribute("null-pointer-is-valid");
8260 if (A.isValid()) {
8261 // The value can be "true" or "false".
8262 bool NullPointerIsValid = A.getValueAsString() == "true";
8263 B.removeAttribute("null-pointer-is-valid");
8264 if (NullPointerIsValid)
8265 B.addAttribute(Attribute::NullPointerIsValid);
8266 }
8267
8268 A = B.getAttribute("uniform-work-group-size");
8269 if (A.isValid()) {
8270 StringRef Val = A.getValueAsString();
8271 if (!Val.empty()) {
8272 bool IsTrue = Val == "true";
8273 B.removeAttribute("uniform-work-group-size");
8274 if (IsTrue)
8275 B.addAttribute("uniform-work-group-size");
8276 }
8277 }
8278}
8279
8280void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
8281 // clang.arc.attachedcall bundles are now required to have an operand.
8282 // If they don't, it's okay to drop them entirely: when there is an operand,
8283 // the "attachedcall" is meaningful and required, but without an operand,
8284 // it's just a marker NOP. Dropping it merely prevents an optimization.
8285 erase_if(Bundles, [&](OperandBundleDef &OBD) {
8286 return OBD.getTag() == "clang.arc.attachedcall" &&
8287 OBD.inputs().empty();
8288 });
8289}
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
uint64_t uint64_t
uint64_t Imm
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
#define G2S_ID(ID_SUFFIX, NAME)
static Metadata * upgradeLoopArgument(Metadata *MD)
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
static bool isXYZ(StringRef S)
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
static bool isValidLegacyNVVMFPArithIntrinsic(const Intrinsic::ID(&IIDs)[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu)
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
static bool shouldUpgradeVPIntrinsic(StringRef Name)
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
#define G2S_CTA_ID(ID_SUFFIX, NAME)
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
static constexpr Intrinsic::ID NVVMFMAIIDs[2][3]
static constexpr Intrinsic::ID NVVMFMulIIDs[2][3]
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
static unsigned getFunctionalOpcodeForVP(StringRef Name)
static MDNode * rebuildScopeOrDomainNode(MDNode *Old, SmallVectorImpl< Metadata * > &Ops)
Rebuild the alias scope or domain Old with Ops, keeping its identity: a self reference has to point a...
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
static Intrinsic::ID shouldUpgradeNVVMBarrierMulticastIntrinsic(Function *F, StringRef Name)
static bool isOldLoopArgument(Metadata *MD)
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static void ConvertModuleFlag(Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static MDNode * upgradeAliasScope(MDNode *Scope, DenseMap< MDNode *, MDNode * > &Upgraded)
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static MDNode * upgradeAliasScopeDomain(MDNode *Domain, DenseMap< MDNode *, MDNode * > &Upgraded)
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static void rename(GlobalValue *GV)
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
#define NVVM_TMA_G2S_MODES(M)
static Value * upgradeNVVMFPArithCall(IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(Function *F, StringRef Name)
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade(StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool upgradePtrauthInitFiniArrays(Module &M)
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
static constexpr Intrinsic::ID NVVMFAddIIDs[2][3]
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
@ Enable
This file contains constants used for implementing Dwarf debug support.
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define R2(n)
This file contains the declarations for metadata subclasses.
#define T
#define T1
NVPTX address space definition.
uint64_t High
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
int64_t C1
RISCVCC::CondCode CC
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
This file contains some functions that are useful when dealing with strings.
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
static const X86InstrFMA3Group Groups[]
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition APInt.h:78
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
Class to represent array types.
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Type * getElementType() const
an instruction that atomically reads a memory location, combines it with another value,...
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Add
*p = old + v
@ FAdd
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:107
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DWARF expression.
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Base class for non-instruction debug metadata records that have positions within IR.
void setDebugLoc(DebugLoc Loc)
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:793
Diagnostic information for debug metadata version reporting.
Diagnostic information for stripping invalid debug metadata.
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
void setApproxFunc(bool B=true)
Definition FMF.h:93
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
const Function & getFunction() const
Definition Function.h:167
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
size_t arg_size() const
Definition Function.h:895
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
Argument * getArg(unsigned i) const
Definition Function.h:880
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
LinkageTypes getLinkage() const
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:571
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
Base class for instruction visitors.
Definition InstVisitor.h:78
bool isCast() const
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
bool isBinaryOp() const
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
bool isUnaryOp() const
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:92
Metadata node.
Definition Metadata.h:1081
static MDTuple * getDistinct(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1590
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
ArrayRef< MDOperand > operands() const
Definition Metadata.h:1435
op_iterator op_end() const
Definition Metadata.h:1431
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1582
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
bool isDistinct() const
Definition Metadata.h:1264
op_iterator op_begin() const
Definition Metadata.h:1427
LLVMContext & getContext() const
Definition Metadata.h:1245
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
A single uniqued string.
Definition Metadata.h:733
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:613
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:595
Tuple of metadata.
Definition Metadata.h:1499
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1528
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:105
Root of the metadata hierarchy.
Definition Metadata.h:64
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
A tuple of MDNodes.
Definition Metadata.h:1800
LLVM_ABI void setOperand(unsigned I, MDNode *New)
LLVM_ABI MDNode * getOperand(unsigned i) const
LLVM_ABI unsigned getNumOperands() const
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
iterator_range< op_iterator > operands()
Definition Metadata.h:1896
LLVM_ABI void addOperand(MDNode *M)
ArrayRef< InputTy > inputs() const
StringRef getTag() const
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:84
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:860
ArrayRef< int > getShuffleMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:28
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
static constexpr size_t npos
Definition StringRef.h:58
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition StringRef.h:661
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & StartsWith(StringLiteral S, T Value)
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Class to represent struct types.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:462
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:295
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:293
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:271
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:270
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:269
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:707
bool use_empty() const
Definition Value.h:348
bool hasName() const
Definition Value.h:263
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:395
Base class of all SIMD vector types.
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ PRIVATE_ADDRESS
Address space for private memory.
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
static const int NoAliasScopeDeclScopeArg
Definition Intrinsics.h:44
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
constexpr StringLiteral GridConstant("nvvm.grid_constant")
constexpr StringLiteral MaxNTID("nvvm.maxntid")
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
constexpr StringLiteral MinCTASm("nvvm.minctasm")
constexpr StringLiteral ReqNTID("nvvm.reqntid")
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
LLVM_ABI void UpgradeSectionAttributes(Module &M)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
bool isValidAtomicOrdering(Int I)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Load
The value being inserted comes from a load (InsertElement only).
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
LLVM_ABI void UpgradeInstructionMetadata(Module &M)
Discover metadata used by instructions in this module, and apply the suitable upgrade.
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
std::string utostr(uint64_t X, bool isNeg=false)
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
LLVM_ABI MDNode * UpgradeTBAAStructNode(MDNode &TBAAStructNode)
If the given !tbaa.struct node has old-style scalar field tags, return an equivalent node with each f...
LLVM_ABI MDNode * UpgradeAliasScopeList(MDNode &ScopeList, DenseMap< MDNode *, MDNode * > &UpgradedAliasScopes)
Mark any domains from scopes in ScopeList that don't have a disjointness flag as non-disjoint.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:35
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ Dynamic
Denotes mode unknown at compile time.
@ TowardPositive
roundTowardPositive.
@ TowardNegative
roundTowardNegative.
ArrayRef(const T &OneElt) -> ArrayRef< T >
const IROptions & getIROptions(const LLVMContext &C)
Definition IROptions.h:27
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Single-operand tags replacing a removed two-operand form !
StringLiteral Disable
StringLiteral Enable
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getInvalid()
constexpr bool isValid() const
static constexpr DenormalMode getIEEE()
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111