LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/Value.h"
47#include "llvm/IR/Verifier.h"
48#include "llvm/Support/AMDGPUAddrSpace.h"
49#include "llvm/Support/CommandLine.h"
50#include "llvm/Support/ErrorHandling.h"
51#include "llvm/Support/NVPTXAddrSpace.h"
52#include "llvm/Support/NVVMAttributes.h"
53#include "llvm/Support/Regex.h"
54#include "llvm/Support/TimeProfiler.h"
55#include "llvm/TargetParser/Triple.h"
56#include <cstdint>
57#include <cstring>
58#include <numeric>
59
60using namespace llvm;
61
62static cl::opt<bool>
63 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
64 cl::desc("Disable autoupgrade of debug info"));
65
66static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
67
68// Report a fatal error along with the
69// Call Instruction which caused the error
70[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
71 CallBase *CI) {
72 CI->print(llvm::errs());
73 llvm::errs() << "\n";
74 reportFatalUsageError(reason);
75}
76
77// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
78// changed their type from v4f32 to v2i64.
79static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
80 Function *&NewFn) {
81 // Check whether this is an old version of the function, which received
82 // v4f32 arguments.
83 Type *Arg0Type = F->getFunctionType()->getParamType(0);
84 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
85 return false;
86
87 // Yes, it's old, replace it with new version.
88 rename(F);
89 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
90 return true;
91}
92
93// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
94// arguments have changed their type from i32 to i8.
95static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
96 Function *&NewFn) {
97 // Check that the last argument is an i32.
98 Type *LastArgType = F->getFunctionType()->getParamType(
99 F->getFunctionType()->getNumParams() - 1);
100 if (!LastArgType->isIntegerTy(32))
101 return false;
102
103 // Move this function aside and map down.
104 rename(F);
105 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
106 return true;
107}
108
109// Upgrade the declaration of fp compare intrinsics that change return type
110// from scalar to vXi1 mask.
111static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
112 Function *&NewFn) {
113 // Check if the return type is a vector.
114 if (F->getReturnType()->isVectorTy())
115 return false;
116
117 rename(F);
118 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
119 return true;
120}
121
122// Upgrade the declaration of multiply and add bytes intrinsics whose input
123// arguments' types have changed from vectors of i32 to vectors of i8
124static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
125 Function *&NewFn) {
126 // check if input argument type is a vector of i8
127 Type *Arg1Type = F->getFunctionType()->getParamType(1);
128 Type *Arg2Type = F->getFunctionType()->getParamType(2);
129 if (Arg1Type->isVectorTy() &&
130 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
131 Arg2Type->isVectorTy() &&
132 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
133 return false;
134
135 rename(F);
136 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
137 return true;
138}
139
140// Upgrade the declaration of multipy and add words intrinsics whose input
141// arguments' types have changed to vectors of i32 to vectors of i16
142static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
143 Function *&NewFn) {
144 // check if input argument type is a vector of i16
145 Type *Arg1Type = F->getFunctionType()->getParamType(1);
146 Type *Arg2Type = F->getFunctionType()->getParamType(2);
147 if (Arg1Type->isVectorTy() &&
148 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
149 Arg2Type->isVectorTy() &&
150 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
151 return false;
152
153 rename(F);
154 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
155 return true;
156}
157
158static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
159 Function *&NewFn) {
160 if (F->getReturnType()->getScalarType()->isBFloatTy())
161 return false;
162
163 rename(F);
164 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
165 return true;
166}
167
168static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
169 Function *&NewFn) {
170 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
171 return false;
172
173 rename(F);
174 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
175 return true;
176}
177
178static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
179 // All of the intrinsics matches below should be marked with which llvm
180 // version started autoupgrading them. At some point in the future we would
181 // like to use this information to remove upgrade code for some older
182 // intrinsics. It is currently undecided how we will determine that future
183 // point.
184 if (Name.consume_front("avx."))
185 return (Name.starts_with("blend.p") || // Added in 3.7
186 Name == "cvt.ps2.pd.256" || // Added in 3.9
187 Name == "cvtdq2.pd.256" || // Added in 3.9
188 Name == "cvtdq2.ps.256" || // Added in 7.0
189 Name.starts_with("movnt.") || // Added in 3.2
190 Name.starts_with("sqrt.p") || // Added in 7.0
191 Name.starts_with("storeu.") || // Added in 3.9
192 Name.starts_with("vbroadcast.s") || // Added in 3.5
193 Name.starts_with("vbroadcastf128") || // Added in 4.0
194 Name.starts_with("vextractf128.") || // Added in 3.7
195 Name.starts_with("vinsertf128.") || // Added in 3.7
196 Name.starts_with("vperm2f128.") || // Added in 6.0
197 Name.starts_with("vpermil.")); // Added in 3.1
198
199 if (Name.consume_front("avx2."))
200 return (Name == "movntdqa" || // Added in 5.0
201 Name.starts_with("pabs.") || // Added in 6.0
202 Name.starts_with("padds.") || // Added in 8.0
203 Name.starts_with("paddus.") || // Added in 8.0
204 Name.starts_with("pblendd.") || // Added in 3.7
205 Name == "pblendw" || // Added in 3.7
206 Name.starts_with("pbroadcast") || // Added in 3.8
207 Name.starts_with("pcmpeq.") || // Added in 3.1
208 Name.starts_with("pcmpgt.") || // Added in 3.1
209 Name.starts_with("pmax") || // Added in 3.9
210 Name.starts_with("pmin") || // Added in 3.9
211 Name.starts_with("pmovsx") || // Added in 3.9
212 Name.starts_with("pmovzx") || // Added in 3.9
213 Name == "pmul.dq" || // Added in 7.0
214 Name == "pmulu.dq" || // Added in 7.0
215 Name.starts_with("psll.dq") || // Added in 3.7
216 Name.starts_with("psrl.dq") || // Added in 3.7
217 Name.starts_with("psubs.") || // Added in 8.0
218 Name.starts_with("psubus.") || // Added in 8.0
219 Name.starts_with("vbroadcast") || // Added in 3.8
220 Name == "vbroadcasti128" || // Added in 3.7
221 Name == "vextracti128" || // Added in 3.7
222 Name == "vinserti128" || // Added in 3.7
223 Name == "vperm2i128"); // Added in 6.0
224
225 if (Name.consume_front("avx512.")) {
226 if (Name.consume_front("mask."))
227 // 'avx512.mask.*'
228 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
229 Name.starts_with("and.") || // Added in 3.9
230 Name.starts_with("andn.") || // Added in 3.9
231 Name.starts_with("broadcast.s") || // Added in 3.9
232 Name.starts_with("broadcastf32x4.") || // Added in 6.0
233 Name.starts_with("broadcastf32x8.") || // Added in 6.0
234 Name.starts_with("broadcastf64x2.") || // Added in 6.0
235 Name.starts_with("broadcastf64x4.") || // Added in 6.0
236 Name.starts_with("broadcasti32x4.") || // Added in 6.0
237 Name.starts_with("broadcasti32x8.") || // Added in 6.0
238 Name.starts_with("broadcasti64x2.") || // Added in 6.0
239 Name.starts_with("broadcasti64x4.") || // Added in 6.0
240 Name.starts_with("cmp.b") || // Added in 5.0
241 Name.starts_with("cmp.d") || // Added in 5.0
242 Name.starts_with("cmp.q") || // Added in 5.0
243 Name.starts_with("cmp.w") || // Added in 5.0
244 Name.starts_with("compress.b") || // Added in 9.0
245 Name.starts_with("compress.d") || // Added in 9.0
246 Name.starts_with("compress.p") || // Added in 9.0
247 Name.starts_with("compress.q") || // Added in 9.0
248 Name.starts_with("compress.store.") || // Added in 7.0
249 Name.starts_with("compress.w") || // Added in 9.0
250 Name.starts_with("conflict.") || // Added in 9.0
251 Name.starts_with("cvtdq2pd.") || // Added in 4.0
252 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
253 Name == "cvtpd2dq.256" || // Added in 7.0
254 Name == "cvtpd2ps.256" || // Added in 7.0
255 Name == "cvtps2pd.128" || // Added in 7.0
256 Name == "cvtps2pd.256" || // Added in 7.0
257 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
258 Name == "cvtqq2ps.256" || // Added in 9.0
259 Name == "cvtqq2ps.512" || // Added in 9.0
260 Name == "cvttpd2dq.256" || // Added in 7.0
261 Name == "cvttps2dq.128" || // Added in 7.0
262 Name == "cvttps2dq.256" || // Added in 7.0
263 Name.starts_with("cvtudq2pd.") || // Added in 4.0
264 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
265 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
266 Name == "cvtuqq2ps.256" || // Added in 9.0
267 Name == "cvtuqq2ps.512" || // Added in 9.0
268 Name.starts_with("dbpsadbw.") || // Added in 7.0
269 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
270 Name.starts_with("expand.b") || // Added in 9.0
271 Name.starts_with("expand.d") || // Added in 9.0
272 Name.starts_with("expand.load.") || // Added in 7.0
273 Name.starts_with("expand.p") || // Added in 9.0
274 Name.starts_with("expand.q") || // Added in 9.0
275 Name.starts_with("expand.w") || // Added in 9.0
276 Name.starts_with("fpclass.p") || // Added in 7.0
277 Name.starts_with("insert") || // Added in 4.0
278 Name.starts_with("load.") || // Added in 3.9
279 Name.starts_with("loadu.") || // Added in 3.9
280 Name.starts_with("lzcnt.") || // Added in 5.0
281 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
282 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
283 Name.starts_with("movddup") || // Added in 3.9
284 Name.starts_with("move.s") || // Added in 4.0
285 Name.starts_with("movshdup") || // Added in 3.9
286 Name.starts_with("movsldup") || // Added in 3.9
287 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
288 Name.starts_with("or.") || // Added in 3.9
289 Name.starts_with("pabs.") || // Added in 6.0
290 Name.starts_with("packssdw.") || // Added in 5.0
291 Name.starts_with("packsswb.") || // Added in 5.0
292 Name.starts_with("packusdw.") || // Added in 5.0
293 Name.starts_with("packuswb.") || // Added in 5.0
294 Name.starts_with("padd.") || // Added in 4.0
295 Name.starts_with("padds.") || // Added in 8.0
296 Name.starts_with("paddus.") || // Added in 8.0
297 Name.starts_with("palignr.") || // Added in 3.9
298 Name.starts_with("pand.") || // Added in 3.9
299 Name.starts_with("pandn.") || // Added in 3.9
300 Name.starts_with("pavg") || // Added in 6.0
301 Name.starts_with("pbroadcast") || // Added in 6.0
302 Name.starts_with("pcmpeq.") || // Added in 3.9
303 Name.starts_with("pcmpgt.") || // Added in 3.9
304 Name.starts_with("perm.df.") || // Added in 3.9
305 Name.starts_with("perm.di.") || // Added in 3.9
306 Name.starts_with("permvar.") || // Added in 7.0
307 Name.starts_with("pmaddubs.w.") || // Added in 7.0
308 Name.starts_with("pmaddw.d.") || // Added in 7.0
309 Name.starts_with("pmax") || // Added in 4.0
310 Name.starts_with("pmin") || // Added in 4.0
311 Name == "pmov.qd.256" || // Added in 9.0
312 Name == "pmov.qd.512" || // Added in 9.0
313 Name == "pmov.wb.256" || // Added in 9.0
314 Name == "pmov.wb.512" || // Added in 9.0
315 Name.starts_with("pmovsx") || // Added in 4.0
316 Name.starts_with("pmovzx") || // Added in 4.0
317 Name.starts_with("pmul.dq.") || // Added in 4.0
318 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
319 Name.starts_with("pmulh.w.") || // Added in 7.0
320 Name.starts_with("pmulhu.w.") || // Added in 7.0
321 Name.starts_with("pmull.") || // Added in 4.0
322 Name.starts_with("pmultishift.qb.") || // Added in 8.0
323 Name.starts_with("pmulu.dq.") || // Added in 4.0
324 Name.starts_with("por.") || // Added in 3.9
325 Name.starts_with("prol.") || // Added in 8.0
326 Name.starts_with("prolv.") || // Added in 8.0
327 Name.starts_with("pror.") || // Added in 8.0
328 Name.starts_with("prorv.") || // Added in 8.0
329 Name.starts_with("pshuf.b.") || // Added in 4.0
330 Name.starts_with("pshuf.d.") || // Added in 3.9
331 Name.starts_with("pshufh.w.") || // Added in 3.9
332 Name.starts_with("pshufl.w.") || // Added in 3.9
333 Name.starts_with("psll.d") || // Added in 4.0
334 Name.starts_with("psll.q") || // Added in 4.0
335 Name.starts_with("psll.w") || // Added in 4.0
336 Name.starts_with("pslli") || // Added in 4.0
337 Name.starts_with("psllv") || // Added in 4.0
338 Name.starts_with("psra.d") || // Added in 4.0
339 Name.starts_with("psra.q") || // Added in 4.0
340 Name.starts_with("psra.w") || // Added in 4.0
341 Name.starts_with("psrai") || // Added in 4.0
342 Name.starts_with("psrav") || // Added in 4.0
343 Name.starts_with("psrl.d") || // Added in 4.0
344 Name.starts_with("psrl.q") || // Added in 4.0
345 Name.starts_with("psrl.w") || // Added in 4.0
346 Name.starts_with("psrli") || // Added in 4.0
347 Name.starts_with("psrlv") || // Added in 4.0
348 Name.starts_with("psub.") || // Added in 4.0
349 Name.starts_with("psubs.") || // Added in 8.0
350 Name.starts_with("psubus.") || // Added in 8.0
351 Name.starts_with("pternlog.") || // Added in 7.0
352 Name.starts_with("punpckh") || // Added in 3.9
353 Name.starts_with("punpckl") || // Added in 3.9
354 Name.starts_with("pxor.") || // Added in 3.9
355 Name.starts_with("shuf.f") || // Added in 6.0
356 Name.starts_with("shuf.i") || // Added in 6.0
357 Name.starts_with("shuf.p") || // Added in 4.0
358 Name.starts_with("sqrt.p") || // Added in 7.0
359 Name.starts_with("store.b.") || // Added in 3.9
360 Name.starts_with("store.d.") || // Added in 3.9
361 Name.starts_with("store.p") || // Added in 3.9
362 Name.starts_with("store.q.") || // Added in 3.9
363 Name.starts_with("store.w.") || // Added in 3.9
364 Name == "store.ss" || // Added in 7.0
365 Name.starts_with("storeu.") || // Added in 3.9
366 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
367 Name.starts_with("ucmp.") || // Added in 5.0
368 Name.starts_with("unpckh.") || // Added in 3.9
369 Name.starts_with("unpckl.") || // Added in 3.9
370 Name.starts_with("valign.") || // Added in 4.0
371 Name == "vcvtph2ps.128" || // Added in 11.0
372 Name == "vcvtph2ps.256" || // Added in 11.0
373 Name.starts_with("vextract") || // Added in 4.0
374 Name.starts_with("vfmadd.") || // Added in 7.0
375 Name.starts_with("vfmaddsub.") || // Added in 7.0
376 Name.starts_with("vfnmadd.") || // Added in 7.0
377 Name.starts_with("vfnmsub.") || // Added in 7.0
378 Name.starts_with("vpdpbusd.") || // Added in 7.0
379 Name.starts_with("vpdpbusds.") || // Added in 7.0
380 Name.starts_with("vpdpwssd.") || // Added in 7.0
381 Name.starts_with("vpdpwssds.") || // Added in 7.0
382 Name.starts_with("vpermi2var.") || // Added in 7.0
383 Name.starts_with("vpermil.p") || // Added in 3.9
384 Name.starts_with("vpermilvar.") || // Added in 4.0
385 Name.starts_with("vpermt2var.") || // Added in 7.0
386 Name.starts_with("vpmadd52") || // Added in 7.0
387 Name.starts_with("vpshld.") || // Added in 7.0
388 Name.starts_with("vpshldv.") || // Added in 8.0
389 Name.starts_with("vpshrd.") || // Added in 7.0
390 Name.starts_with("vpshrdv.") || // Added in 8.0
391 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
392 Name.starts_with("xor.")); // Added in 3.9
393
394 if (Name.consume_front("mask3."))
395 // 'avx512.mask3.*'
396 return (Name.starts_with("vfmadd.") || // Added in 7.0
397 Name.starts_with("vfmaddsub.") || // Added in 7.0
398 Name.starts_with("vfmsub.") || // Added in 7.0
399 Name.starts_with("vfmsubadd.") || // Added in 7.0
400 Name.starts_with("vfnmsub.")); // Added in 7.0
401
402 if (Name.consume_front("maskz."))
403 // 'avx512.maskz.*'
404 return (Name.starts_with("pternlog.") || // Added in 7.0
405 Name.starts_with("vfmadd.") || // Added in 7.0
406 Name.starts_with("vfmaddsub.") || // Added in 7.0
407 Name.starts_with("vpdpbusd.") || // Added in 7.0
408 Name.starts_with("vpdpbusds.") || // Added in 7.0
409 Name.starts_with("vpdpwssd.") || // Added in 7.0
410 Name.starts_with("vpdpwssds.") || // Added in 7.0
411 Name.starts_with("vpermt2var.") || // Added in 7.0
412 Name.starts_with("vpmadd52") || // Added in 7.0
413 Name.starts_with("vpshldv.") || // Added in 8.0
414 Name.starts_with("vpshrdv.")); // Added in 8.0
415
416 // 'avx512.*'
417 return (Name == "movntdqa" || // Added in 5.0
418 Name == "pmul.dq.512" || // Added in 7.0
419 Name == "pmulu.dq.512" || // Added in 7.0
420 Name.starts_with("broadcastm") || // Added in 6.0
421 Name.starts_with("cmp.p") || // Added in 12.0
422 Name.starts_with("cvtb2mask.") || // Added in 7.0
423 Name.starts_with("cvtd2mask.") || // Added in 7.0
424 Name.starts_with("cvtmask2") || // Added in 5.0
425 Name.starts_with("cvtq2mask.") || // Added in 7.0
426 Name == "cvtusi2sd" || // Added in 7.0
427 Name.starts_with("cvtw2mask.") || // Added in 7.0
428 Name == "kand.w" || // Added in 7.0
429 Name == "kandn.w" || // Added in 7.0
430 Name == "knot.w" || // Added in 7.0
431 Name == "kor.w" || // Added in 7.0
432 Name == "kortestc.w" || // Added in 7.0
433 Name == "kortestz.w" || // Added in 7.0
434 Name.starts_with("kunpck") || // added in 6.0
435 Name == "kxnor.w" || // Added in 7.0
436 Name == "kxor.w" || // Added in 7.0
437 Name.starts_with("padds.") || // Added in 8.0
438 Name.starts_with("pbroadcast") || // Added in 3.9
439 Name.starts_with("prol") || // Added in 8.0
440 Name.starts_with("pror") || // Added in 8.0
441 Name.starts_with("psll.dq") || // Added in 3.9
442 Name.starts_with("psrl.dq") || // Added in 3.9
443 Name.starts_with("psubs.") || // Added in 8.0
444 Name.starts_with("ptestm") || // Added in 6.0
445 Name.starts_with("ptestnm") || // Added in 6.0
446 Name.starts_with("storent.") || // Added in 3.9
447 Name.starts_with("vbroadcast.s") || // Added in 7.0
448 Name.starts_with("vpshld.") || // Added in 8.0
449 Name.starts_with("vpshrd.")); // Added in 8.0
450 }
451
452 if (Name.consume_front("fma."))
453 return (Name.starts_with("vfmadd.") || // Added in 7.0
454 Name.starts_with("vfmsub.") || // Added in 7.0
455 Name.starts_with("vfmsubadd.") || // Added in 7.0
456 Name.starts_with("vfnmadd.") || // Added in 7.0
457 Name.starts_with("vfnmsub.")); // Added in 7.0
458
459 if (Name.consume_front("fma4."))
460 return Name.starts_with("vfmadd.s"); // Added in 7.0
461
462 if (Name.consume_front("sse."))
463 return (Name == "add.ss" || // Added in 4.0
464 Name == "cvtsi2ss" || // Added in 7.0
465 Name == "cvtsi642ss" || // Added in 7.0
466 Name == "div.ss" || // Added in 4.0
467 Name == "mul.ss" || // Added in 4.0
468 Name.starts_with("sqrt.p") || // Added in 7.0
469 Name == "sqrt.ss" || // Added in 7.0
470 Name.starts_with("storeu.") || // Added in 3.9
471 Name == "sub.ss"); // Added in 4.0
472
473 if (Name.consume_front("sse2."))
474 return (Name == "add.sd" || // Added in 4.0
475 Name == "cvtdq2pd" || // Added in 3.9
476 Name == "cvtdq2ps" || // Added in 7.0
477 Name == "cvtps2pd" || // Added in 3.9
478 Name == "cvtsi2sd" || // Added in 7.0
479 Name == "cvtsi642sd" || // Added in 7.0
480 Name == "cvtss2sd" || // Added in 7.0
481 Name == "div.sd" || // Added in 4.0
482 Name == "mul.sd" || // Added in 4.0
483 Name.starts_with("padds.") || // Added in 8.0
484 Name.starts_with("paddus.") || // Added in 8.0
485 Name.starts_with("pcmpeq.") || // Added in 3.1
486 Name.starts_with("pcmpgt.") || // Added in 3.1
487 Name == "pmaxs.w" || // Added in 3.9
488 Name == "pmaxu.b" || // Added in 3.9
489 Name == "pmins.w" || // Added in 3.9
490 Name == "pminu.b" || // Added in 3.9
491 Name == "pmulu.dq" || // Added in 7.0
492 Name.starts_with("pshuf") || // Added in 3.9
493 Name.starts_with("psll.dq") || // Added in 3.7
494 Name.starts_with("psrl.dq") || // Added in 3.7
495 Name.starts_with("psubs.") || // Added in 8.0
496 Name.starts_with("psubus.") || // Added in 8.0
497 Name.starts_with("sqrt.p") || // Added in 7.0
498 Name == "sqrt.sd" || // Added in 7.0
499 Name == "storel.dq" || // Added in 3.9
500 Name.starts_with("storeu.") || // Added in 3.9
501 Name == "sub.sd"); // Added in 4.0
502
503 if (Name.consume_front("sse41."))
504 return (Name.starts_with("blendp") || // Added in 3.7
505 Name == "movntdqa" || // Added in 5.0
506 Name == "pblendw" || // Added in 3.7
507 Name == "pmaxsb" || // Added in 3.9
508 Name == "pmaxsd" || // Added in 3.9
509 Name == "pmaxud" || // Added in 3.9
510 Name == "pmaxuw" || // Added in 3.9
511 Name == "pminsb" || // Added in 3.9
512 Name == "pminsd" || // Added in 3.9
513 Name == "pminud" || // Added in 3.9
514 Name == "pminuw" || // Added in 3.9
515 Name.starts_with("pmovsx") || // Added in 3.8
516 Name.starts_with("pmovzx") || // Added in 3.9
517 Name == "pmuldq"); // Added in 7.0
518
519 if (Name.consume_front("sse42."))
520 return Name == "crc32.64.8"; // Added in 3.4
521
522 if (Name.consume_front("sse4a."))
523 return Name.starts_with("movnt."); // Added in 3.9
524
525 if (Name.consume_front("ssse3."))
526 return (Name == "pabs.b.128" || // Added in 6.0
527 Name == "pabs.d.128" || // Added in 6.0
528 Name == "pabs.w.128"); // Added in 6.0
529
530 if (Name.consume_front("xop."))
531 return (Name == "vpcmov" || // Added in 3.8
532 Name == "vpcmov.256" || // Added in 5.0
533 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
534 Name.starts_with("vprot")); // Added in 8.0
535
536 if (Name.consume_front("bmi."))
537 return (Name.starts_with("pdep.") || // Added in 23.0
538 Name.starts_with("pext.")); // Added in 23.0
539
540 return (Name == "addcarry.u32" || // Added in 8.0
541 Name == "addcarry.u64" || // Added in 8.0
542 Name == "addcarryx.u32" || // Added in 8.0
543 Name == "addcarryx.u64" || // Added in 8.0
544 Name == "subborrow.u32" || // Added in 8.0
545 Name == "subborrow.u64" || // Added in 8.0
546 Name.starts_with("vcvtph2ps.")); // Added in 11.0
547}
548
549static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
550 Function *&NewFn) {
551 // Only handle intrinsics that start with "x86.".
552 if (!Name.consume_front("x86."))
553 return false;
554
555 if (shouldUpgradeX86Intrinsic(F, Name)) {
556 NewFn = nullptr;
557 return true;
558 }
559
560 if (Name == "rdtscp") { // Added in 8.0
561 // If this intrinsic has 0 operands, it's the new version.
562 if (F->getFunctionType()->getNumParams() == 0)
563 return false;
564
565 rename(F);
566 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
567 Intrinsic::x86_rdtscp);
568 return true;
569 }
570
571 Intrinsic::ID ID;
572
573 // SSE4.1 ptest functions may have an old signature.
574 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
575 ID = StringSwitch<Intrinsic::ID>(Name)
576 .Case("c", Intrinsic::x86_sse41_ptestc)
577 .Case("z", Intrinsic::x86_sse41_ptestz)
578 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
579 .Default(Intrinsic::not_intrinsic);
580 if (ID != Intrinsic::not_intrinsic)
581 return upgradePTESTIntrinsic(F, ID, NewFn);
582
583 return false;
584 }
585
586 // Several blend and other instructions with masks used the wrong number of
587 // bits.
588
589 // Added in 3.6
590 ID = StringSwitch<Intrinsic::ID>(Name)
591 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
592 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
593 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
594 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
595 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
596 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
597 .Default(Intrinsic::not_intrinsic);
598 if (ID != Intrinsic::not_intrinsic)
599 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
600
601 if (Name.consume_front("avx512.")) {
602 if (Name.consume_front("mask.cmp.")) {
603 // Added in 7.0
604 ID = StringSwitch<Intrinsic::ID>(Name)
605 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
606 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
607 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
608 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
609 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
610 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
611 .Default(Intrinsic::not_intrinsic);
612 if (ID != Intrinsic::not_intrinsic)
613 return upgradeX86MaskedFPCompare(F, ID, NewFn);
614 } else if (Name.starts_with("vpdpbusd.") ||
615 Name.starts_with("vpdpbusds.")) {
616 // Added in 21.1
617 ID = StringSwitch<Intrinsic::ID>(Name)
618 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
619 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
620 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
621 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
622 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
623 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
624 .Default(Intrinsic::not_intrinsic);
625 if (ID != Intrinsic::not_intrinsic)
626 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
627 } else if (Name.starts_with("vpdpwssd.") ||
628 Name.starts_with("vpdpwssds.")) {
629 // Added in 21.1
630 ID = StringSwitch<Intrinsic::ID>(Name)
631 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
632 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
633 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
634 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
635 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
636 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
637 .Default(Intrinsic::not_intrinsic);
638 if (ID != Intrinsic::not_intrinsic)
639 return upgradeX86MultiplyAddWords(F, ID, NewFn);
640 }
641 return false; // No other 'x86.avx512.*'.
642 }
643
644 if (Name.consume_front("avx2.")) {
645 if (Name.consume_front("vpdpb")) {
646 // Added in 21.1
647 ID = StringSwitch<Intrinsic::ID>(Name)
648 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
649 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
650 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
651 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
652 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
653 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
654 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
655 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
656 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
657 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
658 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
659 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
660 .Default(Intrinsic::not_intrinsic);
661 if (ID != Intrinsic::not_intrinsic)
662 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
663 } else if (Name.consume_front("vpdpw")) {
664 // Added in 21.1
665 ID = StringSwitch<Intrinsic::ID>(Name)
666 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
667 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
668 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
669 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
670 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
671 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
672 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
673 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
674 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
675 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
676 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
677 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
678 .Default(Intrinsic::not_intrinsic);
679 if (ID != Intrinsic::not_intrinsic)
680 return upgradeX86MultiplyAddWords(F, ID, NewFn);
681 }
682 return false; // No other 'x86.avx2.*'
683 }
684
685 if (Name.consume_front("avx10.")) {
686 if (Name.consume_front("vpdpb")) {
687 // Added in 21.1
688 ID = StringSwitch<Intrinsic::ID>(Name)
689 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
690 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
691 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
692 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
693 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
694 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
695 .Default(Intrinsic::not_intrinsic);
696 if (ID != Intrinsic::not_intrinsic)
697 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
698 } else if (Name.consume_front("vpdpw")) {
699 ID = StringSwitch<Intrinsic::ID>(Name)
700 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
701 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
702 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
703 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
704 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
705 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
706 .Default(Intrinsic::not_intrinsic);
707 if (ID != Intrinsic::not_intrinsic)
708 return upgradeX86MultiplyAddWords(F, ID, NewFn);
709 }
710 return false; // No other 'x86.avx10.*'
711 }
712
713 if (Name.consume_front("avx512bf16.")) {
714 // Added in 9.0
715 ID = StringSwitch<Intrinsic::ID>(Name)
716 .Case("cvtne2ps2bf16.128",
717 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
718 .Case("cvtne2ps2bf16.256",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
720 .Case("cvtne2ps2bf16.512",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
722 .Case("mask.cvtneps2bf16.128",
723 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
724 .Case("cvtneps2bf16.256",
725 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
726 .Case("cvtneps2bf16.512",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
728 .Default(Intrinsic::not_intrinsic);
729 if (ID != Intrinsic::not_intrinsic)
730 return upgradeX86BF16Intrinsic(F, ID, NewFn);
731
732 // Added in 9.0
733 ID = StringSwitch<Intrinsic::ID>(Name)
734 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
735 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
736 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
737 .Default(Intrinsic::not_intrinsic);
738 if (ID != Intrinsic::not_intrinsic)
739 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
740 return false; // No other 'x86.avx512bf16.*'.
741 }
742
743 if (Name.consume_front("xop.")) {
744 Intrinsic::ID ID = Intrinsic::not_intrinsic;
745 if (Name.starts_with("vpermil2")) { // Added in 3.9
746 // Upgrade any XOP PERMIL2 index operand still using a float/double
747 // vector.
748 auto Idx = F->getFunctionType()->getParamType(2);
749 if (Idx->isFPOrFPVectorTy()) {
750 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
751 unsigned EltSize = Idx->getScalarSizeInBits();
752 if (EltSize == 64 && IdxSize == 128)
753 ID = Intrinsic::x86_xop_vpermil2pd;
754 else if (EltSize == 32 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2ps;
756 else if (EltSize == 64 && IdxSize == 256)
757 ID = Intrinsic::x86_xop_vpermil2pd_256;
758 else
759 ID = Intrinsic::x86_xop_vpermil2ps_256;
760 }
761 } else if (F->arg_size() == 2)
762 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
763 ID = StringSwitch<Intrinsic::ID>(Name)
764 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
765 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
766 .Default(Intrinsic::not_intrinsic);
767
768 if (ID != Intrinsic::not_intrinsic) {
769 rename(F);
770 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
771 return true;
772 }
773 return false; // No other 'x86.xop.*'
774 }
775
776 if (Name == "seh.recoverfp") {
777 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
778 Intrinsic::eh_recoverfp);
779 return true;
780 }
781
782 return false;
783}
784
785// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
786// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
787static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
788 StringRef Name,
789 Function *&NewFn) {
790 if (Name.starts_with("rbit")) {
791 // '(arm|aarch64).rbit'.
792 NewFn = Intrinsic::getOrInsertDeclaration(
793 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
794 return true;
795 }
796
797 if (Name == "thread.pointer") {
798 // '(arm|aarch64).thread.pointer'.
799 NewFn = Intrinsic::getOrInsertDeclaration(
800 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
801 return true;
802 }
803
804 bool Neon = Name.consume_front("neon.");
805 if (Neon) {
806 // '(arm|aarch64).neon.*'.
807 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
808 // v16i8 respectively.
809 if (Name.consume_front("bfdot.")) {
810 // (arm|aarch64).neon.bfdot.*'.
811 Intrinsic::ID ID =
812 StringSwitch<Intrinsic::ID>(Name)
813 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
814 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
815 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
816 .Default(Intrinsic::not_intrinsic);
817 if (ID != Intrinsic::not_intrinsic) {
818 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
819 assert((OperandWidth == 64 || OperandWidth == 128) &&
820 "Unexpected operand width");
821 LLVMContext &Ctx = F->getParent()->getContext();
822 std::array<Type *, 2> Tys{
823 {F->getReturnType(),
824 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
825 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
826 return true;
827 }
828 return false; // No other '(arm|aarch64).neon.bfdot.*'.
829 }
830
831 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
832 // anymore and accept v8bf16 instead of v16i8.
833 if (Name.consume_front("bfm")) {
834 // (arm|aarch64).neon.bfm*'.
835 if (Name.consume_back(".v4f32.v16i8")) {
836 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
837 Intrinsic::ID ID =
838 StringSwitch<Intrinsic::ID>(Name)
839 .Case("mla",
840 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
841 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
842 .Case("lalb",
843 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
844 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
845 .Case("lalt",
846 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
847 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
848 .Default(Intrinsic::not_intrinsic);
849 if (ID != Intrinsic::not_intrinsic) {
850 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
851 return true;
852 }
853 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.
856 }
857 // Continue on to Aarch64 Neon or Arm Neon.
858 }
859 // Continue on to Arm or Aarch64.
860
861 if (IsArm) {
862 // 'arm.*'.
863 if (Neon) {
864 // 'arm.neon.*'.
865 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
866 .StartsWith("vclz.", Intrinsic::ctlz)
867 .StartsWith("vcnt.", Intrinsic::ctpop)
868 .StartsWith("vqadds.", Intrinsic::sadd_sat)
869 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
870 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
871 .StartsWith("vqsubu.", Intrinsic::usub_sat)
872 .StartsWith("vrinta.", Intrinsic::round)
873 .StartsWith("vrintn.", Intrinsic::roundeven)
874 .StartsWith("vrintm.", Intrinsic::floor)
875 .StartsWith("vrintp.", Intrinsic::ceil)
876 .StartsWith("vrintx.", Intrinsic::rint)
877 .StartsWith("vrintz.", Intrinsic::trunc)
878 .Default(Intrinsic::not_intrinsic);
879 if (ID != Intrinsic::not_intrinsic) {
880 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
881 F->arg_begin()->getType());
882 return true;
883 }
884
885 if (Name.consume_front("vst")) {
886 // 'arm.neon.vst*'.
887 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
888 SmallVector<StringRef, 2> Groups;
889 if (vstRegex.match(Name, &Groups)) {
890 static const Intrinsic::ID StoreInts[] = {
891 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
892 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
893
894 static const Intrinsic::ID StoreLaneInts[] = {
895 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
896 Intrinsic::arm_neon_vst4lane};
897
898 auto fArgs = F->getFunctionType()->params();
899 Type *Tys[] = {fArgs[0], fArgs[1]};
900 if (Groups[1].size() == 1)
901 NewFn = Intrinsic::getOrInsertDeclaration(
902 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
903 else
904 NewFn = Intrinsic::getOrInsertDeclaration(
905 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
906 return true;
907 }
908 return false; // No other 'arm.neon.vst*'.
909 }
910
911 return false; // No other 'arm.neon.*'.
912 }
913
914 if (Name.consume_front("mve.")) {
915 // 'arm.mve.*'.
916 if (Name == "vctp64") {
917 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
918 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
919 // the function and deal with it below in UpgradeIntrinsicCall.
920 rename(F);
921 return true;
922 }
923 return false; // Not 'arm.mve.vctp64'.
924 }
925
926 if (Name.starts_with("vrintn.v")) {
927 NewFn = Intrinsic::getOrInsertDeclaration(
928 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
929 return true;
930 }
931
932 // These too are changed to accept a v2i1 instead of the old v4i1.
933 if (Name.consume_back(".v4i1")) {
934 // 'arm.mve.*.v4i1'.
935 if (Name.consume_back(".predicated.v2i64.v4i32"))
936 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
937 return Name == "mull.int" || Name == "vqdmull";
938
939 if (Name.consume_back(".v2i64")) {
940 // 'arm.mve.*.v2i64.v4i1'
941 bool IsGather = Name.consume_front("vldr.gather.");
942 if (IsGather || Name.consume_front("vstr.scatter.")) {
943 if (Name.consume_front("base.")) {
944 // Optional 'wb.' prefix.
945 Name.consume_front("wb.");
946 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
947 // predicated.v2i64.v2i64.v4i1'.
948 return Name == "predicated.v2i64";
949 }
950
951 if (Name.consume_front("offset.predicated."))
952 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
953 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
954
955 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
956 return false;
957 }
958
959 return false; // No other 'arm.mve.*.v2i64.v4i1'.
960 }
961 return false; // No other 'arm.mve.*.v4i1'.
962 }
963 return false; // No other 'arm.mve.*'.
964 }
965
966 if (Name.consume_front("cde.vcx")) {
967 // 'arm.cde.vcx*'.
968 if (Name.consume_back(".predicated.v2i64.v4i1"))
969 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
970 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
971 Name == "3q" || Name == "3qa";
972
973 return false; // No other 'arm.cde.vcx*'.
974 }
975 } else {
976 // 'aarch64.*'.
977 if (Neon) {
978 // 'aarch64.neon.*'.
979 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
980 .StartsWith("frintn", Intrinsic::roundeven)
981 .StartsWith("rbit", Intrinsic::bitreverse)
982 .Default(Intrinsic::not_intrinsic);
983 if (ID != Intrinsic::not_intrinsic) {
984 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
985 F->arg_begin()->getType());
986 return true;
987 }
988
989 if (Name.starts_with("addp")) {
990 // 'aarch64.neon.addp*'.
991 if (F->arg_size() != 2)
992 return false; // Invalid IR.
993 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
994 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
995 NewFn = Intrinsic::getOrInsertDeclaration(
996 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
997 return true;
998 }
999 }
1000
1001 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1002 if (Name.starts_with("bfcvt")) {
1003 NewFn = nullptr;
1004 return true;
1005 }
1006
1007 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1008 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1009 NewFn = nullptr;
1010 return true;
1011 }
1012
1013 return false; // No other 'aarch64.neon.*'.
1014 }
1015 if (Name.consume_front("sve.")) {
1016 // 'aarch64.sve.*'.
1017 if (Name.consume_front("bf")) {
1018 if (Name == "mmla") {
1019 Type *Tys[] = {F->getReturnType(),
1020 std::next(F->arg_begin())->getType()};
1021 NewFn = Intrinsic::getOrInsertDeclaration(
1022 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1023 return true;
1024 }
1025 if (Name.consume_back(".lane")) {
1026 // 'aarch64.sve.bf*.lane'.
1027 Intrinsic::ID ID =
1028 StringSwitch<Intrinsic::ID>(Name)
1029 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1030 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1031 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1032 .Default(Intrinsic::not_intrinsic);
1033 if (ID != Intrinsic::not_intrinsic) {
1034 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1035 return true;
1036 }
1037 return false; // No other 'aarch64.sve.bf*.lane'.
1038 }
1039 return false; // No other 'aarch64.sve.bf*'.
1040 }
1041
1042 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1043 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1044 NewFn = nullptr;
1045 return true;
1046 }
1047
1048 if (Name.consume_front("addqv")) {
1049 // 'aarch64.sve.addqv'.
1050 if (!F->getReturnType()->isFPOrFPVectorTy())
1051 return false;
1052
1053 auto Args = F->getFunctionType()->params();
1054 Type *Tys[] = {F->getReturnType(), Args[1]};
1055 NewFn = Intrinsic::getOrInsertDeclaration(
1056 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1057 return true;
1058 }
1059
1060 if (Name.consume_front("ld")) {
1061 // 'aarch64.sve.ld*'.
1062 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1063 if (LdRegex.match(Name)) {
1064 Type *ScalarTy =
1065 cast<VectorType>(F->getReturnType())->getElementType();
1066 ElementCount EC =
1067 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1068 assert(F->arg_size() == 2 &&
1069 "Expected 2 arguments for ld* intrinsic.");
1070 Type *PtrTy = F->getArg(1)->getType();
1071 Type *Ty = VectorType::get(ScalarTy, EC);
1072 static const Intrinsic::ID LoadIDs[] = {
1073 Intrinsic::aarch64_sve_ld2_sret,
1074 Intrinsic::aarch64_sve_ld3_sret,
1075 Intrinsic::aarch64_sve_ld4_sret,
1076 };
1077 NewFn = Intrinsic::getOrInsertDeclaration(
1078 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1079 return true;
1080 }
1081 return false; // No other 'aarch64.sve.ld*'.
1082 }
1083
1084 if (Name.consume_front("tuple.")) {
1085 // 'aarch64.sve.tuple.*'.
1086 if (Name.starts_with("get")) {
1087 // 'aarch64.sve.tuple.get*'.
1088 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1089 NewFn = Intrinsic::getOrInsertDeclaration(
1090 F->getParent(), Intrinsic::vector_extract, Tys);
1091 return true;
1092 }
1093
1094 if (Name.starts_with("set")) {
1095 // 'aarch64.sve.tuple.set*'.
1096 auto Args = F->getFunctionType()->params();
1097 Type *Tys[] = {Args[0], Args[2], Args[1]};
1098 NewFn = Intrinsic::getOrInsertDeclaration(
1099 F->getParent(), Intrinsic::vector_insert, Tys);
1100 return true;
1101 }
1102
1103 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1104 if (CreateTupleRegex.match(Name)) {
1105 // 'aarch64.sve.tuple.create*'.
1106 auto Args = F->getFunctionType()->params();
1107 Type *Tys[] = {F->getReturnType(), Args[1]};
1108 NewFn = Intrinsic::getOrInsertDeclaration(
1109 F->getParent(), Intrinsic::vector_insert, Tys);
1110 return true;
1111 }
1112 return false; // No other 'aarch64.sve.tuple.*'.
1113 }
1114
1115 if (Name.starts_with("rev.nxv")) {
1116 // 'aarch64.sve.rev.<Ty>'
1117 NewFn = Intrinsic::getOrInsertDeclaration(
1118 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1119 return true;
1120 }
1121
1122 return false; // No other 'aarch64.sve.*'.
1123 }
1124 if (Name.consume_front("sme.")) {
1125 // 'aarch64.sme.*'.
1126 if (Name.consume_front("ftmopa.")) {
1127 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1128 // intrinsics to model their FPMR dependency.
1129 Intrinsic::ID ID =
1130 StringSwitch<Intrinsic::ID>(Name)
1131 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1132 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1133 .Default(Intrinsic::not_intrinsic);
1134 if (ID != Intrinsic::not_intrinsic) {
1135 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1136 return true;
1137 }
1138 return false; // No other 'aarch64.sme.ftmopa.*'.
1139 }
1140
1141 return false; // No other 'aarch64.sme.*'.
1142 }
1143 }
1144 return false; // No other 'arm.*', 'aarch64.*'.
1145}
1146
1147static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1148 StringRef Name) {
1149 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
1150 Intrinsic::ID ID =
1151 StringSwitch<Intrinsic::ID>(Name)
1152 .Case("im2col.3d",
1153 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1154 .Case("im2col.4d",
1155 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1156 .Case("im2col.5d",
1157 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1158 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1159 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1160 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1161 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1162 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1163 .Default(Intrinsic::not_intrinsic);
1164
1165 if (ID == Intrinsic::not_intrinsic)
1166 return ID;
1167
1168 // These intrinsics may need upgrade for two reasons:
1169 // (1) When the address-space of the first argument is shared[AS=3]
1170 // (and we upgrade it to use shared_cluster address-space[AS=7])
1171 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1172 NVPTXAS::ADDRESS_SPACE_SHARED)
1173 return ID;
1174
1175 // (2) When there are only two boolean flag arguments at the end:
1176 //
1177 // The last three parameters of the older version of these
1178 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1179 //
1180 // The newer version reads as:
1181 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1182 //
1183 // So, when the type of the [N-3]rd argument is "not i1", then
1184 // it is the older version and we need to upgrade.
1185 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1186 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
1187 if (!ArgType->isIntegerTy(1))
1188 return ID;
1189 }
1190
1191 return Intrinsic::not_intrinsic;
1192}
1193
1194static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1195 StringRef Name) {
1196 if (Name.consume_front("mapa.shared.cluster"))
1197 if (F->getReturnType()->getPointerAddressSpace() ==
1198 NVPTXAS::ADDRESS_SPACE_SHARED)
1199 return Intrinsic::nvvm_mapa_shared_cluster;
1200
1201 if (Name.consume_front("cp.async.bulk.")) {
1202 Intrinsic::ID ID =
1203 StringSwitch<Intrinsic::ID>(Name)
1204 .Case("global.to.shared.cluster",
1205 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1206 .Case("shared.cta.to.cluster",
1207 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1208 .Default(Intrinsic::not_intrinsic);
1209
1210 if (ID != Intrinsic::not_intrinsic)
1211 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1212 NVPTXAS::ADDRESS_SPACE_SHARED)
1213 return ID;
1214 }
1215
1216 return Intrinsic::not_intrinsic;
1217}
1218
1219static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1220 if (Name.consume_front("fma.rn."))
1221 return StringSwitch<Intrinsic::ID>(Name)
1222 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1223 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1224 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1225 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1226 .Default(Intrinsic::not_intrinsic);
1227
1228 if (Name.consume_front("fmax."))
1229 return StringSwitch<Intrinsic::ID>(Name)
1230 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1231 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1232 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1233 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1234 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1235 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1236 .Case("ftz.nan.xorsign.abs.bf16",
1237 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1238 .Case("ftz.nan.xorsign.abs.bf16x2",
1239 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1240 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1241 .Case("ftz.xorsign.abs.bf16x2",
1242 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1243 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1244 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1245 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1246 .Case("nan.xorsign.abs.bf16x2",
1247 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1248 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1249 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1250 .Default(Intrinsic::not_intrinsic);
1251
1252 if (Name.consume_front("fmin."))
1253 return StringSwitch<Intrinsic::ID>(Name)
1254 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1255 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1256 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1257 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1258 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1259 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1260 .Case("ftz.nan.xorsign.abs.bf16",
1261 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1262 .Case("ftz.nan.xorsign.abs.bf16x2",
1263 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1264 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1265 .Case("ftz.xorsign.abs.bf16x2",
1266 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1267 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1268 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1269 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1270 .Case("nan.xorsign.abs.bf16x2",
1271 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1272 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1273 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1274 .Default(Intrinsic::not_intrinsic);
1275
1276 if (Name.consume_front("neg."))
1277 return StringSwitch<Intrinsic::ID>(Name)
1278 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1279 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1280 .Default(Intrinsic::not_intrinsic);
1281
1282 return Intrinsic::not_intrinsic;
1283}
1284
1285static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1286 return Name.consume_front("local") || Name.consume_front("shared") ||
1287 Name.consume_front("global") || Name.consume_front("constant") ||
1288 Name.consume_front("param");
1289}
1290
1291static bool convertIntrinsicValidType(StringRef Name,
1292 const FunctionType *FuncTy) {
1293 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1294 if (Name.starts_with("to.fp16")) {
1295 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1296 HalfTy) &&
1297 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1298 FuncTy->getReturnType());
1299 }
1300
1301 if (Name.starts_with("from.fp16")) {
1302 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1303 HalfTy) &&
1304 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1305 FuncTy->getReturnType());
1306 }
1307
1308 return false;
1309}
1310
1311static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1312 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(F->getName());
1313 if (IID == Intrinsic::not_intrinsic)
1314 return false;
1315
1316 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1317 if (Defaults.empty())
1318 return false;
1319
1320 // Overloaded intrinsics are out of scope for the default-arg feature
1321 // and will be supported in a follow-up.
1322 if (Intrinsic::isOverloaded(IID))
1323 return false;
1324
1325 // Get the canonical full declaration for this intrinsic.
1326 Function *FullDecl = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1327
1328 // If the existing declaration already has all args, nothing to upgrade
1329 if (F->arg_size() >= FullDecl->arg_size())
1330 return false;
1331
1332 // Defaults are a contiguous trailing block, so checking the first missing
1333 // argument is enough.
1334 if (F->arg_size() < FirstDefault)
1335 return false;
1336
1337 NewFn = FullDecl;
1338 return true;
1339}
1340
1341static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1342 bool CanUpgradeDebugIntrinsicsToRecords) {
1343 assert(F && "Illegal to upgrade a non-existent Function.");
1344
1345 StringRef Name = F->getName();
1346
1347 // Quickly eliminate it, if it's not a candidate.
1348 if (!Name.consume_front("llvm.") || Name.empty())
1349 return false;
1350
1351 switch (Name[0]) {
1352 default: break;
1353 case 'a': {
1354 bool IsArm = Name.consume_front("arm.");
1355 if (IsArm || Name.consume_front("aarch64.")) {
1356 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1357 return true;
1358 break;
1359 }
1360
1361 if (Name.consume_front("amdgcn.")) {
1362 if (Name == "alignbit") {
1363 // Target specific intrinsic became redundant
1364 NewFn = Intrinsic::getOrInsertDeclaration(
1365 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1366 return true;
1367 }
1368
1369 if (Name.consume_front("atomic.")) {
1370 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1371 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1372 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1373 // and usub_sat so there's no new declaration.
1374 NewFn = nullptr;
1375 return true;
1376 }
1377 break; // No other 'amdgcn.atomic.*'
1378 }
1379
1380 switch (F->getIntrinsicID()) {
1381 default:
1382 break;
1383 // Legacy wmma iu intrinsics without the optional clamp operand.
1384 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1385 if (F->arg_size() == 7) {
1386 NewFn = nullptr;
1387 return true;
1388 }
1389 break;
1390 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1391 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1392 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1393 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1394 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1395 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1396 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1397 if (F->arg_size() == 8) {
1398 NewFn = nullptr;
1399 return true;
1400 }
1401 break;
1402 }
1403
1404 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1405 Name.consume_front("flat.atomic.")) {
1406 if (Name.starts_with("fadd") ||
1407 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1408 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1409 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1410 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1411 // declaration.
1412 NewFn = nullptr;
1413 return true;
1414 }
1415 }
1416
1417 if (Name.starts_with("ldexp.")) {
1418 // Target specific intrinsic became redundant
1419 NewFn = Intrinsic::getOrInsertDeclaration(
1420 F->getParent(), Intrinsic::ldexp,
1421 {F->getReturnType(), F->getArg(1)->getType()});
1422 return true;
1423 }
1424 break; // No other 'amdgcn.*'
1425 }
1426
1427 break;
1428 }
1429 case 'c': {
1430 if (F->arg_size() == 1) {
1431 if (Name.consume_front("convert.")) {
1432 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1433 NewFn = nullptr;
1434 return true;
1435 }
1436 }
1437
1438 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1439 .StartsWith("ctlz.", Intrinsic::ctlz)
1440 .StartsWith("cttz.", Intrinsic::cttz)
1441 .Default(Intrinsic::not_intrinsic);
1442 if (ID != Intrinsic::not_intrinsic) {
1443 rename(F);
1444 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1445 F->arg_begin()->getType());
1446 return true;
1447 }
1448 }
1449
1450 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1451 if (Name == "coro.end" &&
1452 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1453 CoroEndID = Intrinsic::coro_end;
1454 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1455 CoroEndID = Intrinsic::coro_end_async;
1456
1457 if (CoroEndID != Intrinsic::not_intrinsic) {
1458 rename(F);
1459 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1460 return true;
1461 }
1462
1463 break;
1464 }
1465 case 'd':
1466 if (Name.consume_front("dbg.")) {
1467 // Mark debug intrinsics for upgrade to new debug format.
1468 if (CanUpgradeDebugIntrinsicsToRecords) {
1469 if (Name == "addr" || Name == "value" || Name == "assign" ||
1470 Name == "declare" || Name == "label") {
1471 // There's no function to replace these with.
1472 NewFn = nullptr;
1473 // But we do want these to get upgraded.
1474 return true;
1475 }
1476 }
1477 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1478 // converted to DbgVariableRecords later.
1479 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1480 rename(F);
1481 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1482 Intrinsic::dbg_value);
1483 return true;
1484 }
1485 break; // No other 'dbg.*'.
1486 }
1487 break;
1488 case 'e':
1489 if (Name.consume_front("experimental.vector.")) {
1490 Intrinsic::ID ID =
1491 StringSwitch<Intrinsic::ID>(Name)
1492 // Skip over extract.last.active, otherwise it will be 'upgraded'
1493 // to a regular vector extract which is a different operation.
1494 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1495 .StartsWith("extract.", Intrinsic::vector_extract)
1496 .StartsWith("insert.", Intrinsic::vector_insert)
1497 .StartsWith("reverse.", Intrinsic::vector_reverse)
1498 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1499 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1500 .StartsWith("partial.reduce.add",
1501 Intrinsic::vector_partial_reduce_add)
1502 .Default(Intrinsic::not_intrinsic);
1503 if (ID != Intrinsic::not_intrinsic) {
1504 const auto *FT = F->getFunctionType();
1505 SmallVector<Type *, 2> Tys;
1506 if (ID == Intrinsic::vector_extract ||
1507 ID == Intrinsic::vector_interleave2)
1508 // Extracting overloads the return type.
1509 Tys.push_back(FT->getReturnType());
1510 if (ID != Intrinsic::vector_interleave2)
1511 Tys.push_back(FT->getParamType(0));
1512 if (ID == Intrinsic::vector_insert ||
1513 ID == Intrinsic::vector_partial_reduce_add)
1514 // Inserting overloads the inserted type.
1515 Tys.push_back(FT->getParamType(1));
1516 rename(F);
1517 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1518 return true;
1519 }
1520
1521 if (Name.consume_front("reduce.")) {
1522 SmallVector<StringRef, 2> Groups;
1523 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1524 if (R.match(Name, &Groups))
1525 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1526 .Case("add", Intrinsic::vector_reduce_add)
1527 .Case("mul", Intrinsic::vector_reduce_mul)
1528 .Case("and", Intrinsic::vector_reduce_and)
1529 .Case("or", Intrinsic::vector_reduce_or)
1530 .Case("xor", Intrinsic::vector_reduce_xor)
1531 .Case("smax", Intrinsic::vector_reduce_smax)
1532 .Case("smin", Intrinsic::vector_reduce_smin)
1533 .Case("umax", Intrinsic::vector_reduce_umax)
1534 .Case("umin", Intrinsic::vector_reduce_umin)
1535 .Case("fmax", Intrinsic::vector_reduce_fmax)
1536 .Case("fmin", Intrinsic::vector_reduce_fmin)
1537 .Default(Intrinsic::not_intrinsic);
1538
1539 bool V2 = false;
1540 if (ID == Intrinsic::not_intrinsic) {
1541 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1542 Groups.clear();
1543 V2 = true;
1544 if (R2.match(Name, &Groups))
1545 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1546 .Case("fadd", Intrinsic::vector_reduce_fadd)
1547 .Case("fmul", Intrinsic::vector_reduce_fmul)
1548 .Default(Intrinsic::not_intrinsic);
1549 }
1550 if (ID != Intrinsic::not_intrinsic) {
1551 rename(F);
1552 auto Args = F->getFunctionType()->params();
1553 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1554 {Args[V2 ? 1 : 0]});
1555 return true;
1556 }
1557 break; // No other 'expermental.vector.reduce.*'.
1558 }
1559
1560 if (Name.consume_front("splice"))
1561 return true;
1562 break; // No other 'experimental.vector.*'.
1563 }
1564 if (Name.consume_front("experimental.stepvector.")) {
1565 Intrinsic::ID ID = Intrinsic::stepvector;
1566 rename(F);
1567 NewFn = Intrinsic::getOrInsertDeclaration(
1568 F->getParent(), ID, F->getFunctionType()->getReturnType());
1569 return true;
1570 }
1571 break; // No other 'e*'.
1572 case 'f':
1573 if (Name.starts_with("flt.rounds")) {
1574 rename(F);
1575 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1576 Intrinsic::get_rounding);
1577 return true;
1578 }
1579 break;
1580 case 'i':
1581 if (Name.starts_with("invariant.group.barrier")) {
1582 // Rename invariant.group.barrier to launder.invariant.group
1583 auto Args = F->getFunctionType()->params();
1584 Type* ObjectPtr[1] = {Args[0]};
1585 rename(F);
1586 NewFn = Intrinsic::getOrInsertDeclaration(
1587 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1588 return true;
1589 }
1590 break;
1591 case 'l': {
1592 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1593 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1594 if (IsLifetimeStart || IsLifetimeEnd) {
1595 if (F->arg_size() == 2) {
1596 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1597 : Intrinsic::lifetime_end;
1598 rename(F);
1599 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1600 // arguments. Use the Ptr argument to create new declaration.
1601 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1602 F->getArg(1)->getType());
1603 return true;
1604 } else if (F->arg_size() == 1 && Name == ".i64") {
1605 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1606 // Autoupgrade prior to
1607 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1608 // intrinsic with no expected calls. To allow auto-upgrade process to
1609 // delete such invalid intrinsic declaration, set NewFn = nullptr
1610 // and return true here. If there are actual calls to this intrinsic
1611 // (which is not expected), they will be deleted in
1612 // UpgradeIntrinsicCall.
1613 NewFn = nullptr;
1614 return true;
1615 }
1616 }
1617 break;
1618 }
1619 case 'm': {
1620 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1621 // alignment parameter to embedding the alignment as an attribute of
1622 // the pointer args.
1623 if (unsigned ID = StringSwitch<unsigned>(Name)
1624 .StartsWith("memcpy.", Intrinsic::memcpy)
1625 .StartsWith("memmove.", Intrinsic::memmove)
1626 .Default(0)) {
1627 if (F->arg_size() == 5) {
1628 rename(F);
1629 // Get the types of dest, src, and len
1630 ArrayRef<Type *> ParamTypes =
1631 F->getFunctionType()->params().slice(0, 3);
1632 NewFn =
1633 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1634 return true;
1635 }
1636 }
1637 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1638 rename(F);
1639 // Get the types of dest, and len
1640 const auto *FT = F->getFunctionType();
1641 Type *ParamTypes[2] = {
1642 FT->getParamType(0), // Dest
1643 FT->getParamType(2) // len
1644 };
1645 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1646 Intrinsic::memset, ParamTypes);
1647 return true;
1648 }
1649
1650 unsigned MaskedID =
1651 StringSwitch<unsigned>(Name)
1652 .StartsWith("masked.load", Intrinsic::masked_load)
1653 .StartsWith("masked.gather", Intrinsic::masked_gather)
1654 .StartsWith("masked.store", Intrinsic::masked_store)
1655 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1656 .Default(0);
1657 if (MaskedID && F->arg_size() == 4) {
1658 rename(F);
1659 if (MaskedID == Intrinsic::masked_load ||
1660 MaskedID == Intrinsic::masked_gather) {
1661 NewFn = Intrinsic::getOrInsertDeclaration(
1662 F->getParent(), MaskedID,
1663 {F->getReturnType(), F->getArg(0)->getType()});
1664 return true;
1665 }
1666 NewFn = Intrinsic::getOrInsertDeclaration(
1667 F->getParent(), MaskedID,
1668 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1669 return true;
1670 }
1671 break;
1672 }
1673 case 'n': {
1674 if (Name.consume_front("nvvm.")) {
1675 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1676 if (F->arg_size() == 1) {
1677 Intrinsic::ID IID =
1678 StringSwitch<Intrinsic::ID>(Name)
1679 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1680 .Case("clz.i", Intrinsic::ctlz)
1681 .Case("popc.i", Intrinsic::ctpop)
1682 .Default(Intrinsic::not_intrinsic);
1683 if (IID != Intrinsic::not_intrinsic) {
1684 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1685 {F->getReturnType()});
1686 return true;
1687 }
1688 } else if (F->arg_size() == 2) {
1689 Intrinsic::ID IID =
1690 StringSwitch<Intrinsic::ID>(Name)
1691 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
1692 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
1693 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
1694 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
1695 .Default(Intrinsic::not_intrinsic);
1696 if (IID != Intrinsic::not_intrinsic) {
1697 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1698 {F->getReturnType()});
1699 return true;
1700 }
1701 }
1702
1703 // Check for nvvm intrinsics that need a return type adjustment.
1704 if (!F->getReturnType()->getScalarType()->isBFloatTy()) {
1705 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1706 if (IID != Intrinsic::not_intrinsic) {
1707 NewFn = nullptr;
1708 return true;
1709 }
1710 }
1711
1712 // Upgrade Distributed Shared Memory Intrinsics
1713 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1714 if (IID != Intrinsic::not_intrinsic) {
1715 rename(F);
1716 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1717 return true;
1718 }
1719
1720 // Upgrade TMA copy G2S Intrinsics
1721 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1722 if (IID != Intrinsic::not_intrinsic) {
1723 rename(F);
1724 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1725 return true;
1726 }
1727
1728 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1729 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1730 //
1731 // TODO: We could add lohi.i2d.
1732 bool Expand = false;
1733 if (Name.consume_front("abs."))
1734 // nvvm.abs.{i,ii}
1735 Expand =
1736 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
1737 else if (Name.consume_front("fabs."))
1738 // nvvm.fabs.{f,ftz.f,d}
1739 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
1740 else if (Name.consume_front("ex2.approx."))
1741 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
1742 Expand =
1743 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
1744 else if (Name.consume_front("atomic.load."))
1745 // nvvm.atomic.load.add.{f32,f64}.p
1746 // nvvm.atomic.load.{inc,dec}.32.p
1747 Expand = StringSwitch<bool>(Name)
1748 .StartsWith("add.f32.p", true)
1749 .StartsWith("add.f64.p", true)
1750 .StartsWith("inc.32.p", true)
1751 .StartsWith("dec.32.p", true)
1752 .Default(false);
1753 else if (Name.consume_front("atomic."))
1754 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
1755 // nvvm.atomic.cas.gen.i.{cta,sys}
1756 Expand = StringSwitch<bool>(Name)
1757 .StartsWith("add.gen.", true)
1758 .StartsWith("exch.gen.", true)
1759 .StartsWith("max.gen.", true)
1760 .StartsWith("min.gen.", true)
1761 .StartsWith("inc.gen.", true)
1762 .StartsWith("dec.gen.", true)
1763 .StartsWith("and.gen.", true)
1764 .StartsWith("or.gen.", true)
1765 .StartsWith("xor.gen.", true)
1766 .StartsWith("cas.gen.", true)
1767 .Default(false);
1768 else if (Name.consume_front("bitcast."))
1769 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
1770 Expand =
1771 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
1772 else if (Name.consume_front("rotate."))
1773 // nvvm.rotate.{b32,b64,right.b64}
1774 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
1775 else if (Name.consume_front("ptr.gen.to."))
1776 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
1777 Expand = consumeNVVMPtrAddrSpace(Name);
1778 else if (Name.consume_front("ptr."))
1779 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
1780 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
1781 else if (Name.consume_front("ldg.global."))
1782 // nvvm.ldg.global.{i,p,f}
1783 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
1784 Name.starts_with("p."));
1785 else
1786 Expand = StringSwitch<bool>(Name)
1787 .Case("barrier0", true)
1788 .Case("barrier.n", true)
1789 .Case("barrier.sync.cnt", true)
1790 .Case("barrier.sync", true)
1791 .Case("barrier", true)
1792 .Case("bar.sync", true)
1793 .Case("barrier0.popc", true)
1794 .Case("barrier0.and", true)
1795 .Case("barrier0.or", true)
1796 .Case("clz.ll", true)
1797 .Case("popc.ll", true)
1798 .Case("h2f", true)
1799 .Case("swap.lo.hi.b64", true)
1800 .Case("tanh.approx.f32", true)
1801 .Default(false);
1802
1803 if (Expand) {
1804 NewFn = nullptr;
1805 return true;
1806 }
1807 break; // No other 'nvvm.*'.
1808 }
1809 break;
1810 }
1811 case 'o':
1812 if (Name.starts_with("objectsize.")) {
1813 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
1814 if (F->arg_size() == 2 || F->arg_size() == 3) {
1815 rename(F);
1816 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1817 Intrinsic::objectsize, Tys);
1818 return true;
1819 }
1820 }
1821 break;
1822
1823 case 'p':
1824 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
1825 rename(F);
1826 NewFn = Intrinsic::getOrInsertDeclaration(
1827 F->getParent(), Intrinsic::ptr_annotation,
1828 {F->arg_begin()->getType(), F->getArg(1)->getType()});
1829 return true;
1830 }
1831 break;
1832
1833 case 'r': {
1834 if (Name.consume_front("riscv.")) {
1835 Intrinsic::ID ID;
1836 ID = StringSwitch<Intrinsic::ID>(Name)
1837 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
1838 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
1839 .Case("aes32esi", Intrinsic::riscv_aes32esi)
1840 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
1841 .Default(Intrinsic::not_intrinsic);
1842 if (ID != Intrinsic::not_intrinsic) {
1843 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
1844 rename(F);
1845 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1846 return true;
1847 }
1848 break; // No other applicable upgrades.
1849 }
1850
1851 ID = StringSwitch<Intrinsic::ID>(Name)
1852 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
1853 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
1854 .Default(Intrinsic::not_intrinsic);
1855 if (ID != Intrinsic::not_intrinsic) {
1856 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
1857 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1858 rename(F);
1859 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1860 return true;
1861 }
1862 break; // No other applicable upgrades.
1863 }
1864
1865 ID = StringSwitch<Intrinsic::ID>(Name)
1866 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
1867 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
1868 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
1869 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
1870 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
1871 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
1872 .Default(Intrinsic::not_intrinsic);
1873 if (ID != Intrinsic::not_intrinsic) {
1874 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
1875 rename(F);
1876 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1877 return true;
1878 }
1879 break; // No other applicable upgrades.
1880 }
1881
1882 // Replace llvm.riscv.clmul with llvm.clmul.
1883 if (Name == "clmul.i32" || Name == "clmul.i64") {
1884 NewFn = Intrinsic::getOrInsertDeclaration(
1885 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
1886 return true;
1887 }
1888
1889 break; // No other 'riscv.*' intrinsics
1890 }
1891 } break;
1892
1893 case 's':
1894 if (Name == "stackprotectorcheck") {
1895 NewFn = nullptr;
1896 return true;
1897 }
1898 break;
1899
1900 case 't':
1901 if (Name == "thread.pointer") {
1902 NewFn = Intrinsic::getOrInsertDeclaration(
1903 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
1904 return true;
1905 }
1906 break;
1907
1908 case 'v': {
1909 if (Name == "var.annotation" && F->arg_size() == 4) {
1910 rename(F);
1911 NewFn = Intrinsic::getOrInsertDeclaration(
1912 F->getParent(), Intrinsic::var_annotation,
1913 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
1914 return true;
1915 }
1916 if (Name.consume_front("vector.splice")) {
1917 if (Name.starts_with(".left") || Name.starts_with(".right"))
1918 break;
1919 return true;
1920 }
1921 break;
1922 }
1923
1924 case 'w':
1925 if (Name.consume_front("wasm.")) {
1926 Intrinsic::ID ID =
1927 StringSwitch<Intrinsic::ID>(Name)
1928 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
1929 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
1930 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
1931 .Default(Intrinsic::not_intrinsic);
1932 if (ID != Intrinsic::not_intrinsic) {
1933 rename(F);
1934 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1935 F->getReturnType());
1936 return true;
1937 }
1938
1939 if (Name.consume_front("dot.i8x16.i7x16.")) {
1940 ID = StringSwitch<Intrinsic::ID>(Name)
1941 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
1942 .Case("add.signed",
1943 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
1944 .Default(Intrinsic::not_intrinsic);
1945 if (ID != Intrinsic::not_intrinsic) {
1946 rename(F);
1947 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1948 return true;
1949 }
1950 break; // No other 'wasm.dot.i8x16.i7x16.*'.
1951 }
1952 break; // No other 'wasm.*'.
1953 }
1954 break;
1955
1956 case 'x':
1957 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
1958 return true;
1959 }
1960
1961 auto *ST = dyn_cast<StructType>(F->getReturnType());
1962 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
1963 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
1964 // Replace return type with literal non-packed struct. Only do this for
1965 // intrinsics declared to return a struct, not for intrinsics with
1966 // overloaded return type, in which case the exact struct type will be
1967 // mangled into the name.
1968 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
1969 FunctionType *FT = F->getFunctionType();
1970 auto *NewST = StructType::get(ST->getContext(), ST->elements());
1971 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
1972 std::string Name = F->getName().str();
1973 rename(F);
1974 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
1975 Name, F->getParent());
1976
1977 // The new function may also need remangling.
1978 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
1979 NewFn = *Result;
1980 return true;
1981 }
1982 }
1983
1984 // Remangle our intrinsic since we upgrade the mangling
1985 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
1986 if (Result != std::nullopt) {
1987 NewFn = *Result;
1988 return true;
1989 }
1990
1991 // This may not belong here. This function is effectively being overloaded
1992 // to both detect an intrinsic which needs upgrading, and to provide the
1993 // upgraded form of the intrinsic. We should perhaps have two separate
1994 // functions for this.
1995 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
1996 return true;
1997
1998 return false;
1999}
2000
2001bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2002 bool CanUpgradeDebugIntrinsicsToRecords) {
2003 NewFn = nullptr;
2004 bool Upgraded =
2005 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2006
2007 // Upgrade intrinsic attributes. This does not change the function.
2008 if (NewFn)
2009 F = NewFn;
2010 if (Intrinsic::ID id = F->getIntrinsicID()) {
2011 // Only do this if the intrinsic signature is valid.
2012 SmallVector<Type *> OverloadTys;
2013 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2014 F->setAttributes(
2015 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2016 }
2017 return Upgraded;
2018}
2019
2020GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2021 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2022 GV->getName() == "llvm.global_dtors")) ||
2023 !GV->hasInitializer())
2024 return nullptr;
2025 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2026 if (!ATy)
2027 return nullptr;
2028 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2029 if (!STy || STy->getNumElements() != 2)
2030 return nullptr;
2031
2032 LLVMContext &C = GV->getContext();
2033 IRBuilder<> IRB(C);
2034 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2035 IRB.getPtrTy());
2036 Constant *Init = GV->getInitializer();
2037 unsigned N = Init->getNumOperands();
2038 std::vector<Constant *> NewCtors(N);
2039 for (unsigned i = 0; i != N; ++i) {
2040 auto Ctor = cast<Constant>(Init->getOperand(i));
2041 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2042 Ctor->getAggregateElement(1),
2043 ConstantPointerNull::get(IRB.getPtrTy()));
2044 }
2045 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2046
2047 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2048 NewInit, GV->getName());
2049}
2050
2051// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2052// to byte shuffles.
2053static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2054 unsigned Shift) {
2055 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2056 unsigned NumElts = ResultTy->getNumElements() * 8;
2057
2058 // Bitcast from a 64-bit element type to a byte element type.
2059 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2060 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2061
2062 // We'll be shuffling in zeroes.
2063 Value *Res = Constant::getNullValue(VecTy);
2064
2065 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2066 // we'll just return the zero vector.
2067 if (Shift < 16) {
2068 int Idxs[64];
2069 // 256/512-bit version is split into 2/4 16-byte lanes.
2070 for (unsigned l = 0; l != NumElts; l += 16)
2071 for (unsigned i = 0; i != 16; ++i) {
2072 unsigned Idx = NumElts + i - Shift;
2073 if (Idx < NumElts)
2074 Idx -= NumElts - 16; // end of lane, switch operand.
2075 Idxs[l + i] = Idx + l;
2076 }
2077
2078 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2079 }
2080
2081 // Bitcast back to a 64-bit element type.
2082 return Builder.CreateBitCast(Res, ResultTy, "cast");
2083}
2084
2085// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2086// to byte shuffles.
2087static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2088 unsigned Shift) {
2089 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2090 unsigned NumElts = ResultTy->getNumElements() * 8;
2091
2092 // Bitcast from a 64-bit element type to a byte element type.
2093 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2094 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2095
2096 // We'll be shuffling in zeroes.
2097 Value *Res = Constant::getNullValue(VecTy);
2098
2099 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2100 // we'll just return the zero vector.
2101 if (Shift < 16) {
2102 int Idxs[64];
2103 // 256/512-bit version is split into 2/4 16-byte lanes.
2104 for (unsigned l = 0; l != NumElts; l += 16)
2105 for (unsigned i = 0; i != 16; ++i) {
2106 unsigned Idx = i + Shift;
2107 if (Idx >= 16)
2108 Idx += NumElts - 16; // end of lane, switch operand.
2109 Idxs[l + i] = Idx + l;
2110 }
2111
2112 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2113 }
2114
2115 // Bitcast back to a 64-bit element type.
2116 return Builder.CreateBitCast(Res, ResultTy, "cast");
2117}
2118
2119static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2120 unsigned NumElts) {
2121 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2122 llvm::VectorType *MaskTy = FixedVectorType::get(
2123 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2124 Mask = Builder.CreateBitCast(Mask, MaskTy);
2125
2126 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2127 // i8 and we need to extract down to the right number of elements.
2128 if (NumElts <= 4) {
2129 int Indices[4];
2130 for (unsigned i = 0; i != NumElts; ++i)
2131 Indices[i] = i;
2132 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2133 "extract");
2134 }
2135
2136 return Mask;
2137}
2138
2139static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2140 Value *Op1) {
2141 // If the mask is all ones just emit the first operation.
2142 if (const auto *C = dyn_cast<Constant>(Mask))
2143 if (C->isAllOnesValue())
2144 return Op0;
2145
2146 Mask = getX86MaskVec(Builder, Mask,
2147 cast<FixedVectorType>(Op0->getType())->getNumElements());
2148 return Builder.CreateSelect(Mask, Op0, Op1);
2149}
2150
2151static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2152 Value *Op1) {
2153 // If the mask is all ones just emit the first operation.
2154 if (const auto *C = dyn_cast<Constant>(Mask))
2155 if (C->isAllOnesValue())
2156 return Op0;
2157
2158 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2159 Mask->getType()->getIntegerBitWidth());
2160 Mask = Builder.CreateBitCast(Mask, MaskTy);
2161 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2162 return Builder.CreateSelect(Mask, Op0, Op1);
2163}
2164
2165// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2166// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2167// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2168static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2169 Value *Op1, Value *Shift,
2170 Value *Passthru, Value *Mask,
2171 bool IsVALIGN) {
2172 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2173
2174 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2175 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2176 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2177 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2178
2179 // Mask the immediate for VALIGN.
2180 if (IsVALIGN)
2181 ShiftVal &= (NumElts - 1);
2182
2183 // If palignr is shifting the pair of vectors more than the size of two
2184 // lanes, emit zero.
2185 if (ShiftVal >= 32)
2186 return llvm::Constant::getNullValue(Op0->getType());
2187
2188 // If palignr is shifting the pair of input vectors more than one lane,
2189 // but less than two lanes, convert to shifting in zeroes.
2190 if (ShiftVal > 16) {
2191 ShiftVal -= 16;
2192 Op1 = Op0;
2193 Op0 = llvm::Constant::getNullValue(Op0->getType());
2194 }
2195
2196 int Indices[64];
2197 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2198 for (unsigned l = 0; l < NumElts; l += 16) {
2199 for (unsigned i = 0; i != 16; ++i) {
2200 unsigned Idx = ShiftVal + i;
2201 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2202 Idx += NumElts - 16; // End of lane, switch operand.
2203 Indices[l + i] = Idx + l;
2204 }
2205 }
2206
2207 Value *Align = Builder.CreateShuffleVector(
2208 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2209
2210 return emitX86Select(Builder, Mask, Align, Passthru);
2211}
2212
2213static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2214 bool ZeroMask, bool IndexForm) {
2215 Type *Ty = CI.getType();
2216 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2217 unsigned EltWidth = Ty->getScalarSizeInBits();
2218 bool IsFloat = Ty->isFPOrFPVectorTy();
2219 Intrinsic::ID IID;
2220 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2221 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2222 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2223 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2224 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2225 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2226 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2227 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2228 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2229 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2230 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2231 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2232 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2233 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2234 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2235 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2236 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2237 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2238 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2239 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2240 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2241 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2242 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2243 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2244 else if (VecWidth == 128 && EltWidth == 16)
2245 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2246 else if (VecWidth == 256 && EltWidth == 16)
2247 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2248 else if (VecWidth == 512 && EltWidth == 16)
2249 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2250 else if (VecWidth == 128 && EltWidth == 8)
2251 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2252 else if (VecWidth == 256 && EltWidth == 8)
2253 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2254 else if (VecWidth == 512 && EltWidth == 8)
2255 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2256 else
2257 llvm_unreachable("Unexpected intrinsic");
2258
2259 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2260 CI.getArgOperand(2) };
2261
2262 // If this isn't index form we need to swap operand 0 and 1.
2263 if (!IndexForm)
2264 std::swap(Args[0], Args[1]);
2265
2266 Value *V = Builder.CreateIntrinsic(IID, Args);
2267 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2268 : Builder.CreateBitCast(CI.getArgOperand(1),
2269 Ty);
2270 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2271}
2272
2273static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2274 Intrinsic::ID IID) {
2275 Type *Ty = CI.getType();
2276 Value *Op0 = CI.getOperand(0);
2277 Value *Op1 = CI.getOperand(1);
2278 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2279
2280 if (CI.arg_size() == 4) { // For masked intrinsics.
2281 Value *VecSrc = CI.getOperand(2);
2282 Value *Mask = CI.getOperand(3);
2283 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2284 }
2285 return Res;
2286}
2287
2288static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2289 bool IsRotateRight) {
2290 Type *Ty = CI.getType();
2291 Value *Src = CI.getArgOperand(0);
2292 Value *Amt = CI.getArgOperand(1);
2293
2294 // Amount may be scalar immediate, in which case create a splat vector.
2295 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2296 // we only care about the lowest log2 bits anyway.
2297 if (Amt->getType() != Ty) {
2298 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2299 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2300 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2301 }
2302
2303 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2304 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2305
2306 if (CI.arg_size() == 4) { // For masked intrinsics.
2307 Value *VecSrc = CI.getOperand(2);
2308 Value *Mask = CI.getOperand(3);
2309 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2310 }
2311 return Res;
2312}
2313
2314static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2315 bool IsSigned) {
2316 Type *Ty = CI.getType();
2317 Value *LHS = CI.getArgOperand(0);
2318 Value *RHS = CI.getArgOperand(1);
2319
2320 CmpInst::Predicate Pred;
2321 switch (Imm) {
2322 case 0x0:
2323 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2324 break;
2325 case 0x1:
2326 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2327 break;
2328 case 0x2:
2329 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2330 break;
2331 case 0x3:
2332 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2333 break;
2334 case 0x4:
2335 Pred = ICmpInst::ICMP_EQ;
2336 break;
2337 case 0x5:
2338 Pred = ICmpInst::ICMP_NE;
2339 break;
2340 case 0x6:
2341 return Constant::getNullValue(Ty); // FALSE
2342 case 0x7:
2343 return Constant::getAllOnesValue(Ty); // TRUE
2344 default:
2345 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2346 }
2347
2348 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2349 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2350 return Ext;
2351}
2352
2353static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2354 bool IsShiftRight, bool ZeroMask) {
2355 Type *Ty = CI.getType();
2356 Value *Op0 = CI.getArgOperand(0);
2357 Value *Op1 = CI.getArgOperand(1);
2358 Value *Amt = CI.getArgOperand(2);
2359
2360 if (IsShiftRight)
2361 std::swap(Op0, Op1);
2362
2363 // Amount may be scalar immediate, in which case create a splat vector.
2364 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2365 // we only care about the lowest log2 bits anyway.
2366 if (Amt->getType() != Ty) {
2367 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2368 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2369 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2370 }
2371
2372 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2373 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2374
2375 unsigned NumArgs = CI.arg_size();
2376 if (NumArgs >= 4) { // For masked intrinsics.
2377 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2378 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2379 CI.getArgOperand(0);
2380 Value *Mask = CI.getOperand(NumArgs - 1);
2381 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2382 }
2383 return Res;
2384}
2385
2386static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2387 Value *Mask, bool Aligned) {
2388 const Align Alignment =
2389 Aligned
2390 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2391 : Align(1);
2392
2393 // If the mask is all ones just emit a regular store.
2394 if (const auto *C = dyn_cast<Constant>(Mask))
2395 if (C->isAllOnesValue())
2396 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2397
2398 // Convert the mask from an integer type to a vector of i1.
2399 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2400 Mask = getX86MaskVec(Builder, Mask, NumElts);
2401 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2402}
2403
2404static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2405 Value *Passthru, Value *Mask, bool Aligned) {
2406 Type *ValTy = Passthru->getType();
2407 const Align Alignment =
2408 Aligned
2409 ? Align(
2410 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2411 8)
2412 : Align(1);
2413
2414 // If the mask is all ones just emit a regular store.
2415 if (const auto *C = dyn_cast<Constant>(Mask))
2416 if (C->isAllOnesValue())
2417 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2418
2419 // Convert the mask from an integer type to a vector of i1.
2420 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2421 Mask = getX86MaskVec(Builder, Mask, NumElts);
2422 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2423}
2424
2425static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2426 Type *Ty = CI.getType();
2427 Value *Op0 = CI.getArgOperand(0);
2428 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2429 {Op0, Builder.getInt1(false)});
2430 if (CI.arg_size() == 3)
2431 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2432 return Res;
2433}
2434
2435static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2436 Type *Ty = CI.getType();
2437
2438 // Arguments have a vXi32 type so cast to vXi64.
2439 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2440 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2441
2442 if (IsSigned) {
2443 // Shift left then arithmetic shift right.
2444 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2445 LHS = Builder.CreateShl(LHS, ShiftAmt);
2446 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2447 RHS = Builder.CreateShl(RHS, ShiftAmt);
2448 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2449 } else {
2450 // Clear the upper bits.
2451 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2452 LHS = Builder.CreateAnd(LHS, Mask);
2453 RHS = Builder.CreateAnd(RHS, Mask);
2454 }
2455
2456 Value *Res = Builder.CreateMul(LHS, RHS);
2457
2458 if (CI.arg_size() == 4)
2459 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2460
2461 return Res;
2462}
2463
2464// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2465static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2466 Value *Mask) {
2467 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2468 if (Mask) {
2469 const auto *C = dyn_cast<Constant>(Mask);
2470 if (!C || !C->isAllOnesValue())
2471 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2472 }
2473
2474 if (NumElts < 8) {
2475 int Indices[8];
2476 for (unsigned i = 0; i != NumElts; ++i)
2477 Indices[i] = i;
2478 for (unsigned i = NumElts; i != 8; ++i)
2479 Indices[i] = NumElts + i % NumElts;
2480 Vec = Builder.CreateShuffleVector(Vec,
2481 Constant::getNullValue(Vec->getType()),
2482 Indices);
2483 }
2484 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2485}
2486
2487static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2488 unsigned CC, bool Signed) {
2489 Value *Op0 = CI.getArgOperand(0);
2490 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2491
2492 Value *Cmp;
2493 if (CC == 3) {
2494 Cmp = Constant::getNullValue(
2495 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2496 } else if (CC == 7) {
2497 Cmp = Constant::getAllOnesValue(
2498 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2499 } else {
2500 ICmpInst::Predicate Pred;
2501 switch (CC) {
2502 default: llvm_unreachable("Unknown condition code");
2503 case 0: Pred = ICmpInst::ICMP_EQ; break;
2504 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2505 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2506 case 4: Pred = ICmpInst::ICMP_NE; break;
2507 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2508 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2509 }
2510 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2511 }
2512
2513 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2514
2515 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2516}
2517
2518// Replace a masked intrinsic with an older unmasked intrinsic.
2519static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2520 Intrinsic::ID IID) {
2521 Value *Rep =
2522 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2523 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2524}
2525
2526static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2527 Value* A = CI.getArgOperand(0);
2528 Value* B = CI.getArgOperand(1);
2529 Value* Src = CI.getArgOperand(2);
2530 Value* Mask = CI.getArgOperand(3);
2531
2532 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2533 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2534 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2535 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2536 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2537 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2538}
2539
2540static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2541 Value* Op = CI.getArgOperand(0);
2542 Type* ReturnOp = CI.getType();
2543 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2544 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2545 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2546}
2547
2548// Replace intrinsic with unmasked version and a select.
2549static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2550 CallBase &CI, Value *&Rep) {
2551 Name = Name.substr(12); // Remove avx512.mask.
2552
2553 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2554 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2555 Intrinsic::ID IID;
2556 if (Name.starts_with("max.p")) {
2557 if (VecWidth == 128 && EltWidth == 32)
2558 IID = Intrinsic::x86_sse_max_ps;
2559 else if (VecWidth == 128 && EltWidth == 64)
2560 IID = Intrinsic::x86_sse2_max_pd;
2561 else if (VecWidth == 256 && EltWidth == 32)
2562 IID = Intrinsic::x86_avx_max_ps_256;
2563 else if (VecWidth == 256 && EltWidth == 64)
2564 IID = Intrinsic::x86_avx_max_pd_256;
2565 else
2566 llvm_unreachable("Unexpected intrinsic");
2567 } else if (Name.starts_with("min.p")) {
2568 if (VecWidth == 128 && EltWidth == 32)
2569 IID = Intrinsic::x86_sse_min_ps;
2570 else if (VecWidth == 128 && EltWidth == 64)
2571 IID = Intrinsic::x86_sse2_min_pd;
2572 else if (VecWidth == 256 && EltWidth == 32)
2573 IID = Intrinsic::x86_avx_min_ps_256;
2574 else if (VecWidth == 256 && EltWidth == 64)
2575 IID = Intrinsic::x86_avx_min_pd_256;
2576 else
2577 llvm_unreachable("Unexpected intrinsic");
2578 } else if (Name.starts_with("pshuf.b.")) {
2579 if (VecWidth == 128)
2580 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2581 else if (VecWidth == 256)
2582 IID = Intrinsic::x86_avx2_pshuf_b;
2583 else if (VecWidth == 512)
2584 IID = Intrinsic::x86_avx512_pshuf_b_512;
2585 else
2586 llvm_unreachable("Unexpected intrinsic");
2587 } else if (Name.starts_with("pmul.hr.sw.")) {
2588 if (VecWidth == 128)
2589 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2590 else if (VecWidth == 256)
2591 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2592 else if (VecWidth == 512)
2593 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2594 else
2595 llvm_unreachable("Unexpected intrinsic");
2596 } else if (Name.starts_with("pmulh.w.")) {
2597 if (VecWidth == 128)
2598 IID = Intrinsic::x86_sse2_pmulh_w;
2599 else if (VecWidth == 256)
2600 IID = Intrinsic::x86_avx2_pmulh_w;
2601 else if (VecWidth == 512)
2602 IID = Intrinsic::x86_avx512_pmulh_w_512;
2603 else
2604 llvm_unreachable("Unexpected intrinsic");
2605 } else if (Name.starts_with("pmulhu.w.")) {
2606 if (VecWidth == 128)
2607 IID = Intrinsic::x86_sse2_pmulhu_w;
2608 else if (VecWidth == 256)
2609 IID = Intrinsic::x86_avx2_pmulhu_w;
2610 else if (VecWidth == 512)
2611 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2612 else
2613 llvm_unreachable("Unexpected intrinsic");
2614 } else if (Name.starts_with("pmaddw.d.")) {
2615 if (VecWidth == 128)
2616 IID = Intrinsic::x86_sse2_pmadd_wd;
2617 else if (VecWidth == 256)
2618 IID = Intrinsic::x86_avx2_pmadd_wd;
2619 else if (VecWidth == 512)
2620 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2621 else
2622 llvm_unreachable("Unexpected intrinsic");
2623 } else if (Name.starts_with("pmaddubs.w.")) {
2624 if (VecWidth == 128)
2625 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2626 else if (VecWidth == 256)
2627 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2628 else if (VecWidth == 512)
2629 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2630 else
2631 llvm_unreachable("Unexpected intrinsic");
2632 } else if (Name.starts_with("packsswb.")) {
2633 if (VecWidth == 128)
2634 IID = Intrinsic::x86_sse2_packsswb_128;
2635 else if (VecWidth == 256)
2636 IID = Intrinsic::x86_avx2_packsswb;
2637 else if (VecWidth == 512)
2638 IID = Intrinsic::x86_avx512_packsswb_512;
2639 else
2640 llvm_unreachable("Unexpected intrinsic");
2641 } else if (Name.starts_with("packssdw.")) {
2642 if (VecWidth == 128)
2643 IID = Intrinsic::x86_sse2_packssdw_128;
2644 else if (VecWidth == 256)
2645 IID = Intrinsic::x86_avx2_packssdw;
2646 else if (VecWidth == 512)
2647 IID = Intrinsic::x86_avx512_packssdw_512;
2648 else
2649 llvm_unreachable("Unexpected intrinsic");
2650 } else if (Name.starts_with("packuswb.")) {
2651 if (VecWidth == 128)
2652 IID = Intrinsic::x86_sse2_packuswb_128;
2653 else if (VecWidth == 256)
2654 IID = Intrinsic::x86_avx2_packuswb;
2655 else if (VecWidth == 512)
2656 IID = Intrinsic::x86_avx512_packuswb_512;
2657 else
2658 llvm_unreachable("Unexpected intrinsic");
2659 } else if (Name.starts_with("packusdw.")) {
2660 if (VecWidth == 128)
2661 IID = Intrinsic::x86_sse41_packusdw;
2662 else if (VecWidth == 256)
2663 IID = Intrinsic::x86_avx2_packusdw;
2664 else if (VecWidth == 512)
2665 IID = Intrinsic::x86_avx512_packusdw_512;
2666 else
2667 llvm_unreachable("Unexpected intrinsic");
2668 } else if (Name.starts_with("vpermilvar.")) {
2669 if (VecWidth == 128 && EltWidth == 32)
2670 IID = Intrinsic::x86_avx_vpermilvar_ps;
2671 else if (VecWidth == 128 && EltWidth == 64)
2672 IID = Intrinsic::x86_avx_vpermilvar_pd;
2673 else if (VecWidth == 256 && EltWidth == 32)
2674 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2675 else if (VecWidth == 256 && EltWidth == 64)
2676 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2677 else if (VecWidth == 512 && EltWidth == 32)
2678 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2679 else if (VecWidth == 512 && EltWidth == 64)
2680 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2681 else
2682 llvm_unreachable("Unexpected intrinsic");
2683 } else if (Name == "cvtpd2dq.256") {
2684 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2685 } else if (Name == "cvtpd2ps.256") {
2686 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2687 } else if (Name == "cvttpd2dq.256") {
2688 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2689 } else if (Name == "cvttps2dq.128") {
2690 IID = Intrinsic::x86_sse2_cvttps2dq;
2691 } else if (Name == "cvttps2dq.256") {
2692 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2693 } else if (Name.starts_with("permvar.")) {
2694 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2695 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2696 IID = Intrinsic::x86_avx2_permps;
2697 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2698 IID = Intrinsic::x86_avx2_permd;
2699 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2700 IID = Intrinsic::x86_avx512_permvar_df_256;
2701 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2702 IID = Intrinsic::x86_avx512_permvar_di_256;
2703 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2704 IID = Intrinsic::x86_avx512_permvar_sf_512;
2705 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2706 IID = Intrinsic::x86_avx512_permvar_si_512;
2707 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2708 IID = Intrinsic::x86_avx512_permvar_df_512;
2709 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2710 IID = Intrinsic::x86_avx512_permvar_di_512;
2711 else if (VecWidth == 128 && EltWidth == 16)
2712 IID = Intrinsic::x86_avx512_permvar_hi_128;
2713 else if (VecWidth == 256 && EltWidth == 16)
2714 IID = Intrinsic::x86_avx512_permvar_hi_256;
2715 else if (VecWidth == 512 && EltWidth == 16)
2716 IID = Intrinsic::x86_avx512_permvar_hi_512;
2717 else if (VecWidth == 128 && EltWidth == 8)
2718 IID = Intrinsic::x86_avx512_permvar_qi_128;
2719 else if (VecWidth == 256 && EltWidth == 8)
2720 IID = Intrinsic::x86_avx512_permvar_qi_256;
2721 else if (VecWidth == 512 && EltWidth == 8)
2722 IID = Intrinsic::x86_avx512_permvar_qi_512;
2723 else
2724 llvm_unreachable("Unexpected intrinsic");
2725 } else if (Name.starts_with("dbpsadbw.")) {
2726 if (VecWidth == 128)
2727 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2728 else if (VecWidth == 256)
2729 IID = Intrinsic::x86_avx512_dbpsadbw_256;
2730 else if (VecWidth == 512)
2731 IID = Intrinsic::x86_avx512_dbpsadbw_512;
2732 else
2733 llvm_unreachable("Unexpected intrinsic");
2734 } else if (Name.starts_with("pmultishift.qb.")) {
2735 if (VecWidth == 128)
2736 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
2737 else if (VecWidth == 256)
2738 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
2739 else if (VecWidth == 512)
2740 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
2741 else
2742 llvm_unreachable("Unexpected intrinsic");
2743 } else if (Name.starts_with("conflict.")) {
2744 if (Name[9] == 'd' && VecWidth == 128)
2745 IID = Intrinsic::x86_avx512_conflict_d_128;
2746 else if (Name[9] == 'd' && VecWidth == 256)
2747 IID = Intrinsic::x86_avx512_conflict_d_256;
2748 else if (Name[9] == 'd' && VecWidth == 512)
2749 IID = Intrinsic::x86_avx512_conflict_d_512;
2750 else if (Name[9] == 'q' && VecWidth == 128)
2751 IID = Intrinsic::x86_avx512_conflict_q_128;
2752 else if (Name[9] == 'q' && VecWidth == 256)
2753 IID = Intrinsic::x86_avx512_conflict_q_256;
2754 else if (Name[9] == 'q' && VecWidth == 512)
2755 IID = Intrinsic::x86_avx512_conflict_q_512;
2756 else
2757 llvm_unreachable("Unexpected intrinsic");
2758 } else if (Name.starts_with("pavg.")) {
2759 if (Name[5] == 'b' && VecWidth == 128)
2760 IID = Intrinsic::x86_sse2_pavg_b;
2761 else if (Name[5] == 'b' && VecWidth == 256)
2762 IID = Intrinsic::x86_avx2_pavg_b;
2763 else if (Name[5] == 'b' && VecWidth == 512)
2764 IID = Intrinsic::x86_avx512_pavg_b_512;
2765 else if (Name[5] == 'w' && VecWidth == 128)
2766 IID = Intrinsic::x86_sse2_pavg_w;
2767 else if (Name[5] == 'w' && VecWidth == 256)
2768 IID = Intrinsic::x86_avx2_pavg_w;
2769 else if (Name[5] == 'w' && VecWidth == 512)
2770 IID = Intrinsic::x86_avx512_pavg_w_512;
2771 else
2772 llvm_unreachable("Unexpected intrinsic");
2773 } else
2774 return false;
2775
2776 SmallVector<Value *, 4> Args(CI.args());
2777 Args.pop_back();
2778 Args.pop_back();
2779 Rep = Builder.CreateIntrinsic(IID, Args);
2780 unsigned NumArgs = CI.arg_size();
2781 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
2782 CI.getArgOperand(NumArgs - 2));
2783 return true;
2784}
2785
2786/// Upgrade comment in call to inline asm that represents an objc retain release
2787/// marker.
2788void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
2789 size_t Pos;
2790 if (AsmStr->find("mov\tfp") == 0 &&
2791 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
2792 (Pos = AsmStr->find("# marker")) != std::string::npos) {
2793 AsmStr->replace(Pos, 1, ";");
2794 }
2795}
2796
2797static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
2798 Function *F, IRBuilder<> &Builder) {
2799 Value *Rep = nullptr;
2800
2801 if (Name == "abs.i" || Name == "abs.ll") {
2802 Value *Arg = CI->getArgOperand(0);
2803 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
2804 {Arg, Builder.getTrue()},
2805 /*FMFSource=*/nullptr, "abs");
2806 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
2807 Type *Ty = (Name == "abs.bf16")
2808 ? Builder.getBFloatTy()
2809 : FixedVectorType::get(Builder.getBFloatTy(), 2);
2810 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
2811 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
2812 Rep = Builder.CreateBitCast(Abs, CI->getType());
2813 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
2814 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
2815 : Intrinsic::nvvm_fabs;
2816 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
2817 } else if (Name.consume_front("ex2.approx.")) {
2818 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2819 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
2820 : Intrinsic::nvvm_ex2_approx;
2821 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
2822 } else if (Name.starts_with("atomic.load.add.f32.p") ||
2823 Name.starts_with("atomic.load.add.f64.p")) {
2824 Value *Ptr = CI->getArgOperand(0);
2825 Value *Val = CI->getArgOperand(1);
2826 Rep = Builder.CreateAtomicRMW(
2827 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
2828 CI->getContext().getOrInsertSyncScopeID("device"));
2829 // The default scope for atomic.load.* intrinsics is device
2830 // (= gpu scope in ptx), but the default LLVM atomic scope is
2831 // "system"
2832 } else if (Name.starts_with("atomic.load.inc.32.p") ||
2833 Name.starts_with("atomic.load.dec.32.p")) {
2834 Value *Ptr = CI->getArgOperand(0);
2835 Value *Val = CI->getArgOperand(1);
2836 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
2837 : AtomicRMWInst::UDecWrap;
2838 Rep = Builder.CreateAtomicRMW(
2839 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
2840 CI->getContext().getOrInsertSyncScopeID("device"));
2841 // See comment above.
2842 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
2843 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
2844 StringRef Op = Name.substr(StringRef("atomic.").size());
2845 Value *Ptr = CI->getArgOperand(0);
2846 Value *Val = CI->getArgOperand(1);
2847 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
2848 Op.contains(".cta.") ? "block" : "");
2849 if (Op.starts_with("cas.")) {
2850 Value *New = CI->getArgOperand(2);
2851 Value *Pair = Builder.CreateAtomicCmpXchg(
2852 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
2853 AtomicOrdering::Monotonic, SSID);
2854 Rep = Builder.CreateExtractValue(Pair, 0);
2855 } else {
2856 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
2857 // because we were actually missing those intrinsics!
2858 AtomicRMWInst::BinOp BinOp =
2859 StringSwitch<AtomicRMWInst::BinOp>(Op)
2860 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
2861 .StartsWith("add.gen.i", AtomicRMWInst::Add)
2862 .StartsWith("exch.", AtomicRMWInst::Xchg)
2863 .StartsWith("max.", AtomicRMWInst::Max)
2864 .StartsWith("min.", AtomicRMWInst::Min)
2865 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
2866 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
2867 .StartsWith("and.", AtomicRMWInst::And)
2868 .StartsWith("or.", AtomicRMWInst::Or)
2869 .StartsWith("xor.", AtomicRMWInst::Xor)
2870 .Default(AtomicRMWInst::BAD_BINOP);
2871 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
2872 "unexpected nvvm scoped atomic intrinsic");
2873 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
2874 AtomicOrdering::Monotonic, SSID);
2875 }
2876 } else if (Name == "clz.ll") {
2877 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
2878 Value *Arg = CI->getArgOperand(0);
2879 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
2880 {Arg, Builder.getFalse()},
2881 /*FMFSource=*/nullptr, "ctlz");
2882 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
2883 } else if (Name == "popc.ll") {
2884 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
2885 // i64.
2886 Value *Arg = CI->getArgOperand(0);
2887 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
2888 Arg, /*FMFSource=*/nullptr, "ctpop");
2889 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
2890 } else if (Name == "h2f") {
2891 Value *Cast =
2892 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
2893 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
2894 } else if (Name.consume_front("bitcast.") &&
2895 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
2896 Name == "d2ll")) {
2897 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
2898 } else if (Name == "rotate.b32") {
2899 Value *Arg = CI->getOperand(0);
2900 Value *ShiftAmt = CI->getOperand(1);
2901 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
2902 {Arg, Arg, ShiftAmt});
2903 } else if (Name == "rotate.b64") {
2904 Type *Int64Ty = Builder.getInt64Ty();
2905 Value *Arg = CI->getOperand(0);
2906 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
2907 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
2908 {Arg, Arg, ZExtShiftAmt});
2909 } else if (Name == "rotate.right.b64") {
2910 Type *Int64Ty = Builder.getInt64Ty();
2911 Value *Arg = CI->getOperand(0);
2912 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
2913 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
2914 {Arg, Arg, ZExtShiftAmt});
2915 } else if (Name == "swap.lo.hi.b64") {
2916 Type *Int64Ty = Builder.getInt64Ty();
2917 Value *Arg = CI->getOperand(0);
2918 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
2919 {Arg, Arg, Builder.getInt64(32)});
2920 } else if ((Name.consume_front("ptr.gen.to.") &&
2921 consumeNVVMPtrAddrSpace(Name)) ||
2922 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
2923 Name.starts_with(".to.gen"))) {
2924 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
2925 } else if (Name.consume_front("ldg.global")) {
2926 Value *Ptr = CI->getArgOperand(0);
2927 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
2928 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
2929 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
2930 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
2931 MDNode *MD = MDNode::get(Builder.getContext(), {});
2932 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
2933 return LD;
2934 } else if (Name == "tanh.approx.f32") {
2935 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
2936 FastMathFlags FMF;
2937 FMF.setApproxFunc();
2938 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
2939 FMF);
2940 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
2941 Value *Arg =
2942 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
2943 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
2944 {}, {Arg});
2945 } else if (Name == "barrier") {
2946 Rep = Builder.CreateIntrinsic(
2947 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
2948 {CI->getArgOperand(0), CI->getArgOperand(1)});
2949 } else if (Name == "barrier.sync") {
2950 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
2951 {CI->getArgOperand(0)});
2952 } else if (Name == "barrier.sync.cnt") {
2953 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
2954 {CI->getArgOperand(0), CI->getArgOperand(1)});
2955 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
2956 Name == "barrier0.or") {
2957 Value *C = CI->getArgOperand(0);
2958 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
2959
2960 Intrinsic::ID IID =
2961 StringSwitch<Intrinsic::ID>(Name)
2962 .Case("barrier0.popc",
2963 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
2964 .Case("barrier0.and",
2965 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
2966 .Case("barrier0.or",
2967 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
2968 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
2969 Rep = Builder.CreateZExt(Bar, CI->getType());
2970 } else {
2971 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2972 if (IID != Intrinsic::not_intrinsic &&
2973 !F->getReturnType()->getScalarType()->isBFloatTy()) {
2974 rename(F);
2975 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2976 SmallVector<Value *, 2> Args;
2977 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
2978 Value *Arg = CI->getArgOperand(I);
2979 Type *OldType = Arg->getType();
2980 Type *NewType = NewFn->getArg(I)->getType();
2981 Args.push_back(
2982 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
2983 ? Builder.CreateBitCast(Arg, NewType)
2984 : Arg);
2985 }
2986 Rep = Builder.CreateCall(NewFn, Args);
2987 if (F->getReturnType()->isIntegerTy())
2988 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
2989 }
2990 }
2991
2992 return Rep;
2993}
2994
2995static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
2996 IRBuilder<> &Builder) {
2997 LLVMContext &C = F->getContext();
2998 Value *Rep = nullptr;
2999
3000 if (Name.starts_with("sse4a.movnt.")) {
3001 SmallVector<Metadata *, 1> Elts;
3002 Elts.push_back(
3003 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3004 MDNode *Node = MDNode::get(C, Elts);
3005
3006 Value *Arg0 = CI->getArgOperand(0);
3007 Value *Arg1 = CI->getArgOperand(1);
3008
3009 // Nontemporal (unaligned) store of the 0'th element of the float/double
3010 // vector.
3011 Value *Extract =
3012 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3013
3014 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3015 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3016 } else if (Name.starts_with("avx.movnt.") ||
3017 Name.starts_with("avx512.storent.")) {
3018 SmallVector<Metadata *, 1> Elts;
3019 Elts.push_back(
3020 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3021 MDNode *Node = MDNode::get(C, Elts);
3022
3023 Value *Arg0 = CI->getArgOperand(0);
3024 Value *Arg1 = CI->getArgOperand(1);
3025
3026 StoreInst *SI = Builder.CreateAlignedStore(
3027 Arg1, Arg0,
3028 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3029 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3030 } else if (Name == "sse2.storel.dq") {
3031 Value *Arg0 = CI->getArgOperand(0);
3032 Value *Arg1 = CI->getArgOperand(1);
3033
3034 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3035 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3036 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3037 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3038 } else if (Name.starts_with("sse.storeu.") ||
3039 Name.starts_with("sse2.storeu.") ||
3040 Name.starts_with("avx.storeu.")) {
3041 Value *Arg0 = CI->getArgOperand(0);
3042 Value *Arg1 = CI->getArgOperand(1);
3043 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3044 } else if (Name == "avx512.mask.store.ss") {
3045 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3046 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3047 Mask, false);
3048 } else if (Name.starts_with("avx512.mask.store")) {
3049 // "avx512.mask.storeu." or "avx512.mask.store."
3050 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3051 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3052 CI->getArgOperand(2), Aligned);
3053 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3054 // Upgrade packed integer vector compare intrinsics to compare instructions.
3055 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3056 bool CmpEq = Name[9] == 'e';
3057 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3058 CI->getArgOperand(0), CI->getArgOperand(1));
3059 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3060 } else if (Name.starts_with("avx512.broadcastm")) {
3061 Type *ExtTy = Type::getInt32Ty(C);
3062 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3063 ExtTy = Type::getInt64Ty(C);
3064 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3065 ExtTy->getPrimitiveSizeInBits();
3066 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3067 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3068 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3069 Value *Vec = CI->getArgOperand(0);
3070 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3071 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3072 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3073 } else if (Name.starts_with("avx.sqrt.p") ||
3074 Name.starts_with("sse2.sqrt.p") ||
3075 Name.starts_with("sse.sqrt.p")) {
3076 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3077 {CI->getArgOperand(0)});
3078 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3079 if (CI->arg_size() == 4 &&
3080 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3081 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3082 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3083 : Intrinsic::x86_avx512_sqrt_pd_512;
3084
3085 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3086 Rep = Builder.CreateIntrinsic(IID, Args);
3087 } else {
3088 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3089 {CI->getArgOperand(0)});
3090 }
3091 Rep =
3092 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3093 } else if (Name.starts_with("avx512.ptestm") ||
3094 Name.starts_with("avx512.ptestnm")) {
3095 Value *Op0 = CI->getArgOperand(0);
3096 Value *Op1 = CI->getArgOperand(1);
3097 Value *Mask = CI->getArgOperand(2);
3098 Rep = Builder.CreateAnd(Op0, Op1);
3099 llvm::Type *Ty = Op0->getType();
3100 Value *Zero = llvm::Constant::getNullValue(Ty);
3101 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3102 ? ICmpInst::ICMP_NE
3103 : ICmpInst::ICMP_EQ;
3104 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3105 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3106 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3107 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3108 ->getNumElements();
3109 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3110 Rep =
3111 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3112 } else if (Name.starts_with("avx512.kunpck")) {
3113 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3114 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3115 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3116 int Indices[64];
3117 for (unsigned i = 0; i != NumElts; ++i)
3118 Indices[i] = i;
3119
3120 // First extract half of each vector. This gives better codegen than
3121 // doing it in a single shuffle.
3122 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3123 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3124 // Concat the vectors.
3125 // NOTE: Operands have to be swapped to match intrinsic definition.
3126 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3127 Rep = Builder.CreateBitCast(Rep, CI->getType());
3128 } else if (Name == "avx512.kand.w") {
3129 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3130 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3131 Rep = Builder.CreateAnd(LHS, RHS);
3132 Rep = Builder.CreateBitCast(Rep, CI->getType());
3133 } else if (Name == "avx512.kandn.w") {
3134 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3135 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3136 LHS = Builder.CreateNot(LHS);
3137 Rep = Builder.CreateAnd(LHS, RHS);
3138 Rep = Builder.CreateBitCast(Rep, CI->getType());
3139 } else if (Name == "avx512.kor.w") {
3140 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3141 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3142 Rep = Builder.CreateOr(LHS, RHS);
3143 Rep = Builder.CreateBitCast(Rep, CI->getType());
3144 } else if (Name == "avx512.kxor.w") {
3145 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3146 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3147 Rep = Builder.CreateXor(LHS, RHS);
3148 Rep = Builder.CreateBitCast(Rep, CI->getType());
3149 } else if (Name == "avx512.kxnor.w") {
3150 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3151 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3152 LHS = Builder.CreateNot(LHS);
3153 Rep = Builder.CreateXor(LHS, RHS);
3154 Rep = Builder.CreateBitCast(Rep, CI->getType());
3155 } else if (Name == "avx512.knot.w") {
3156 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3157 Rep = Builder.CreateNot(Rep);
3158 Rep = Builder.CreateBitCast(Rep, CI->getType());
3159 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3160 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3161 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3162 Rep = Builder.CreateOr(LHS, RHS);
3163 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3164 Value *C;
3165 if (Name[14] == 'c')
3166 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3167 else
3168 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3169 Rep = Builder.CreateICmpEQ(Rep, C);
3170 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3171 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3172 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3173 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3174 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3175 Type *I32Ty = Type::getInt32Ty(C);
3176 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3177 ConstantInt::get(I32Ty, 0));
3178 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3179 ConstantInt::get(I32Ty, 0));
3180 Value *EltOp;
3181 if (Name.contains(".add."))
3182 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3183 else if (Name.contains(".sub."))
3184 EltOp = Builder.CreateFSub(Elt0, Elt1);
3185 else if (Name.contains(".mul."))
3186 EltOp = Builder.CreateFMul(Elt0, Elt1);
3187 else
3188 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3189 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3190 ConstantInt::get(I32Ty, 0));
3191 } else if (Name.starts_with("avx512.mask.pcmp")) {
3192 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3193 bool CmpEq = Name[16] == 'e';
3194 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3195 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3196 Type *OpTy = CI->getArgOperand(0)->getType();
3197 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3198 Intrinsic::ID IID;
3199 switch (VecWidth) {
3200 default:
3201 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3202 break;
3203 case 128:
3204 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3205 break;
3206 case 256:
3207 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3208 break;
3209 case 512:
3210 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3211 break;
3212 }
3213
3214 Rep =
3215 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3216 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3217 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3218 Type *OpTy = CI->getArgOperand(0)->getType();
3219 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3220 unsigned EltWidth = OpTy->getScalarSizeInBits();
3221 Intrinsic::ID IID;
3222 if (VecWidth == 128 && EltWidth == 32)
3223 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3224 else if (VecWidth == 256 && EltWidth == 32)
3225 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3226 else if (VecWidth == 512 && EltWidth == 32)
3227 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3228 else if (VecWidth == 128 && EltWidth == 64)
3229 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3230 else if (VecWidth == 256 && EltWidth == 64)
3231 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3232 else if (VecWidth == 512 && EltWidth == 64)
3233 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3234 else
3235 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3236
3237 Rep =
3238 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3239 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3240 } else if (Name.starts_with("avx512.cmp.p")) {
3241 SmallVector<Value *, 4> Args(CI->args());
3242 Type *OpTy = Args[0]->getType();
3243 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3244 unsigned EltWidth = OpTy->getScalarSizeInBits();
3245 Intrinsic::ID IID;
3246 if (VecWidth == 128 && EltWidth == 32)
3247 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3248 else if (VecWidth == 256 && EltWidth == 32)
3249 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3250 else if (VecWidth == 512 && EltWidth == 32)
3251 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3252 else if (VecWidth == 128 && EltWidth == 64)
3253 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3254 else if (VecWidth == 256 && EltWidth == 64)
3255 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3256 else if (VecWidth == 512 && EltWidth == 64)
3257 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3258 else
3259 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3260
3261 Value *Mask = Constant::getAllOnesValue(CI->getType());
3262 if (VecWidth == 512)
3263 std::swap(Mask, Args.back());
3264 Args.push_back(Mask);
3265
3266 Rep = Builder.CreateIntrinsic(IID, Args);
3267 } else if (Name.starts_with("avx512.mask.cmp.")) {
3268 // Integer compare intrinsics.
3269 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3270 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3271 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3272 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3273 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3274 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3275 Name.starts_with("avx512.cvtw2mask.") ||
3276 Name.starts_with("avx512.cvtd2mask.") ||
3277 Name.starts_with("avx512.cvtq2mask.")) {
3278 Value *Op = CI->getArgOperand(0);
3279 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3280 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3281 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3282 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3283 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3284 Name.starts_with("avx512.mask.pabs")) {
3285 Rep = upgradeAbs(Builder, *CI);
3286 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3287 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3288 Name.starts_with("avx512.mask.pmaxs")) {
3289 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3290 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3291 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3292 Name.starts_with("avx512.mask.pmaxu")) {
3293 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3294 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3295 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3296 Name.starts_with("avx512.mask.pmins")) {
3297 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3298 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3299 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3300 Name.starts_with("avx512.mask.pminu")) {
3301 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3302 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3303 Name == "avx512.pmulu.dq.512" ||
3304 Name.starts_with("avx512.mask.pmulu.dq.")) {
3305 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3306 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3307 Name == "avx512.pmul.dq.512" ||
3308 Name.starts_with("avx512.mask.pmul.dq.")) {
3309 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3310 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3311 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3312 Rep =
3313 Builder.CreateSIToFP(CI->getArgOperand(1),
3314 cast<VectorType>(CI->getType())->getElementType());
3315 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3316 } else if (Name == "avx512.cvtusi2sd") {
3317 Rep =
3318 Builder.CreateUIToFP(CI->getArgOperand(1),
3319 cast<VectorType>(CI->getType())->getElementType());
3320 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3321 } else if (Name == "sse2.cvtss2sd") {
3322 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3323 Rep = Builder.CreateFPExt(
3324 Rep, cast<VectorType>(CI->getType())->getElementType());
3325 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3326 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3327 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3328 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3329 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3330 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3331 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3332 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3333 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3334 Name == "avx512.mask.cvtqq2ps.256" ||
3335 Name == "avx512.mask.cvtqq2ps.512" ||
3336 Name == "avx512.mask.cvtuqq2ps.256" ||
3337 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3338 Name == "avx.cvt.ps2.pd.256" ||
3339 Name == "avx512.mask.cvtps2pd.128" ||
3340 Name == "avx512.mask.cvtps2pd.256") {
3341 auto *DstTy = cast<FixedVectorType>(CI->getType());
3342 Rep = CI->getArgOperand(0);
3343 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3344
3345 unsigned NumDstElts = DstTy->getNumElements();
3346 if (NumDstElts < SrcTy->getNumElements()) {
3347 assert(NumDstElts == 2 && "Unexpected vector size");
3348 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3349 }
3350
3351 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3352 bool IsUnsigned = Name.contains("cvtu");
3353 if (IsPS2PD)
3354 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3355 else if (CI->arg_size() == 4 &&
3356 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3357 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3358 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3359 : Intrinsic::x86_avx512_sitofp_round;
3360 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3361 {Rep, CI->getArgOperand(3)});
3362 } else {
3363 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3364 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3365 }
3366
3367 if (CI->arg_size() >= 3)
3368 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3369 CI->getArgOperand(1));
3370 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3371 Name.starts_with("vcvtph2ps.")) {
3372 auto *DstTy = cast<FixedVectorType>(CI->getType());
3373 Rep = CI->getArgOperand(0);
3374 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3375 unsigned NumDstElts = DstTy->getNumElements();
3376 if (NumDstElts != SrcTy->getNumElements()) {
3377 assert(NumDstElts == 4 && "Unexpected vector size");
3378 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3379 }
3380 Rep = Builder.CreateBitCast(
3381 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3382 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3383 if (CI->arg_size() >= 3)
3384 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3385 CI->getArgOperand(1));
3386 } else if (Name.starts_with("avx512.mask.load")) {
3387 // "avx512.mask.loadu." or "avx512.mask.load."
3388 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3389 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3390 CI->getArgOperand(2), Aligned);
3391 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3392 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3393 auto *PtrTy = CI->getOperand(0)->getType();
3394 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3395 ResultTy->getNumElements());
3396 Rep = Builder.CreateIntrinsic(
3397 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3398 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3399 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3400 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3401 auto *PtrTy = CI->getArgOperand(0)->getType();
3402 Value *MaskVec =
3403 getX86MaskVec(Builder, CI->getArgOperand(2),
3404 cast<FixedVectorType>(ResultTy)->getNumElements());
3405 Rep = Builder.CreateIntrinsic(
3406 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3407 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3408 } else if (Name.starts_with("avx512.mask.compress.") ||
3409 Name.starts_with("avx512.mask.expand.")) {
3410 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3411
3412 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3413 ResultTy->getNumElements());
3414
3415 bool IsCompress = Name[12] == 'c';
3416 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3417 : Intrinsic::x86_avx512_mask_expand;
3418 Rep = Builder.CreateIntrinsic(
3419 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3420 } else if (Name.starts_with("xop.vpcom")) {
3421 bool IsSigned;
3422 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3423 Name.ends_with("uq"))
3424 IsSigned = false;
3425 else if (Name.ends_with("b") || Name.ends_with("w") ||
3426 Name.ends_with("d") || Name.ends_with("q"))
3427 IsSigned = true;
3428 else
3429 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3430
3431 unsigned Imm;
3432 if (CI->arg_size() == 3) {
3433 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3434 } else {
3435 Name = Name.substr(9); // strip off "xop.vpcom"
3436 if (Name.starts_with("lt"))
3437 Imm = 0;
3438 else if (Name.starts_with("le"))
3439 Imm = 1;
3440 else if (Name.starts_with("gt"))
3441 Imm = 2;
3442 else if (Name.starts_with("ge"))
3443 Imm = 3;
3444 else if (Name.starts_with("eq"))
3445 Imm = 4;
3446 else if (Name.starts_with("ne"))
3447 Imm = 5;
3448 else if (Name.starts_with("false"))
3449 Imm = 6;
3450 else if (Name.starts_with("true"))
3451 Imm = 7;
3452 else
3453 llvm_unreachable("Unknown condition");
3454 }
3455
3456 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3457 } else if (Name.starts_with("xop.vpcmov")) {
3458 Value *Sel = CI->getArgOperand(2);
3459 Value *NotSel = Builder.CreateNot(Sel);
3460 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3461 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3462 Rep = Builder.CreateOr(Sel0, Sel1);
3463 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3464 Name.starts_with("avx512.mask.prol")) {
3465 Rep = upgradeX86Rotate(Builder, *CI, false);
3466 } else if (Name.starts_with("avx512.pror") ||
3467 Name.starts_with("avx512.mask.pror")) {
3468 Rep = upgradeX86Rotate(Builder, *CI, true);
3469 } else if (Name.starts_with("avx512.vpshld.") ||
3470 Name.starts_with("avx512.mask.vpshld") ||
3471 Name.starts_with("avx512.maskz.vpshld")) {
3472 bool ZeroMask = Name[11] == 'z';
3473 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3474 } else if (Name.starts_with("avx512.vpshrd.") ||
3475 Name.starts_with("avx512.mask.vpshrd") ||
3476 Name.starts_with("avx512.maskz.vpshrd")) {
3477 bool ZeroMask = Name[11] == 'z';
3478 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3479 } else if (Name == "sse42.crc32.64.8") {
3480 Value *Trunc0 =
3481 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3482 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3483 {Trunc0, CI->getArgOperand(1)});
3484 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3485 } else if (Name.starts_with("avx.vbroadcast.s") ||
3486 Name.starts_with("avx512.vbroadcast.s")) {
3487 // Replace broadcasts with a series of insertelements.
3488 auto *VecTy = cast<FixedVectorType>(CI->getType());
3489 Type *EltTy = VecTy->getElementType();
3490 unsigned EltNum = VecTy->getNumElements();
3491 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3492 Type *I32Ty = Type::getInt32Ty(C);
3493 Rep = PoisonValue::get(VecTy);
3494 for (unsigned I = 0; I < EltNum; ++I)
3495 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3496 } else if (Name.starts_with("sse41.pmovsx") ||
3497 Name.starts_with("sse41.pmovzx") ||
3498 Name.starts_with("avx2.pmovsx") ||
3499 Name.starts_with("avx2.pmovzx") ||
3500 Name.starts_with("avx512.mask.pmovsx") ||
3501 Name.starts_with("avx512.mask.pmovzx")) {
3502 auto *DstTy = cast<FixedVectorType>(CI->getType());
3503 unsigned NumDstElts = DstTy->getNumElements();
3504
3505 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3506 SmallVector<int, 8> ShuffleMask(NumDstElts);
3507 for (unsigned i = 0; i != NumDstElts; ++i)
3508 ShuffleMask[i] = i;
3509
3510 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3511
3512 bool DoSext = Name.contains("pmovsx");
3513 Rep =
3514 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3515 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3516 if (CI->arg_size() == 3)
3517 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3518 CI->getArgOperand(1));
3519 } else if (Name == "avx512.mask.pmov.qd.256" ||
3520 Name == "avx512.mask.pmov.qd.512" ||
3521 Name == "avx512.mask.pmov.wb.256" ||
3522 Name == "avx512.mask.pmov.wb.512") {
3523 Type *Ty = CI->getArgOperand(1)->getType();
3524 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3525 Rep =
3526 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3527 } else if (Name.starts_with("avx.vbroadcastf128") ||
3528 Name == "avx2.vbroadcasti128") {
3529 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3530 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3531 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3532 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3533 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3534 if (NumSrcElts == 2)
3535 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3536 else
3537 Rep = Builder.CreateShuffleVector(Load,
3538 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3539 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3540 Name.starts_with("avx512.mask.shuf.f")) {
3541 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3542 Type *VT = CI->getType();
3543 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3544 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3545 unsigned ControlBitsMask = NumLanes - 1;
3546 unsigned NumControlBits = NumLanes / 2;
3547 SmallVector<int, 8> ShuffleMask(0);
3548
3549 for (unsigned l = 0; l != NumLanes; ++l) {
3550 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3551 // We actually need the other source.
3552 if (l >= NumLanes / 2)
3553 LaneMask += NumLanes;
3554 for (unsigned i = 0; i != NumElementsInLane; ++i)
3555 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3556 }
3557 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3558 CI->getArgOperand(1), ShuffleMask);
3559 Rep =
3560 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3561 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3562 Name.starts_with("avx512.mask.broadcasti")) {
3563 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3564 ->getNumElements();
3565 unsigned NumDstElts =
3566 cast<FixedVectorType>(CI->getType())->getNumElements();
3567
3568 SmallVector<int, 8> ShuffleMask(NumDstElts);
3569 for (unsigned i = 0; i != NumDstElts; ++i)
3570 ShuffleMask[i] = i % NumSrcElts;
3571
3572 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3573 CI->getArgOperand(0), ShuffleMask);
3574 Rep =
3575 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3576 } else if (Name.starts_with("avx2.pbroadcast") ||
3577 Name.starts_with("avx2.vbroadcast") ||
3578 Name.starts_with("avx512.pbroadcast") ||
3579 Name.starts_with("avx512.mask.broadcast.s")) {
3580 // Replace vp?broadcasts with a vector shuffle.
3581 Value *Op = CI->getArgOperand(0);
3582 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3583 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3584 SmallVector<int, 8> M;
3585 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3586 Rep = Builder.CreateShuffleVector(Op, M);
3587
3588 if (CI->arg_size() == 3)
3589 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3590 CI->getArgOperand(1));
3591 } else if (Name.starts_with("sse2.padds.") ||
3592 Name.starts_with("avx2.padds.") ||
3593 Name.starts_with("avx512.padds.") ||
3594 Name.starts_with("avx512.mask.padds.")) {
3595 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3596 } else if (Name.starts_with("sse2.psubs.") ||
3597 Name.starts_with("avx2.psubs.") ||
3598 Name.starts_with("avx512.psubs.") ||
3599 Name.starts_with("avx512.mask.psubs.")) {
3600 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3601 } else if (Name.starts_with("sse2.paddus.") ||
3602 Name.starts_with("avx2.paddus.") ||
3603 Name.starts_with("avx512.mask.paddus.")) {
3604 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3605 } else if (Name.starts_with("sse2.psubus.") ||
3606 Name.starts_with("avx2.psubus.") ||
3607 Name.starts_with("avx512.mask.psubus.")) {
3608 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3609 } else if (Name.starts_with("avx512.mask.palignr.")) {
3610 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3611 CI->getArgOperand(1), CI->getArgOperand(2),
3612 CI->getArgOperand(3), CI->getArgOperand(4),
3613 false);
3614 } else if (Name.starts_with("avx512.mask.valign.")) {
3615 Rep = upgradeX86ALIGNIntrinsics(
3616 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3617 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3618 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3619 // 128/256-bit shift left specified in bits.
3620 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3621 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3622 Shift / 8); // Shift is in bits.
3623 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3624 // 128/256-bit shift right specified in bits.
3625 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3626 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
3627 Shift / 8); // Shift is in bits.
3628 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3629 Name == "avx512.psll.dq.512") {
3630 // 128/256/512-bit shift left specified in bytes.
3631 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3632 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3633 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3634 Name == "avx512.psrl.dq.512") {
3635 // 128/256/512-bit shift right specified in bytes.
3636 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3637 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3638 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3639 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3640 Name.starts_with("avx2.pblendd.")) {
3641 Value *Op0 = CI->getArgOperand(0);
3642 Value *Op1 = CI->getArgOperand(1);
3643 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3644 auto *VecTy = cast<FixedVectorType>(CI->getType());
3645 unsigned NumElts = VecTy->getNumElements();
3646
3647 SmallVector<int, 16> Idxs(NumElts);
3648 for (unsigned i = 0; i != NumElts; ++i)
3649 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3650
3651 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3652 } else if (Name.starts_with("avx.vinsertf128.") ||
3653 Name == "avx2.vinserti128" ||
3654 Name.starts_with("avx512.mask.insert")) {
3655 Value *Op0 = CI->getArgOperand(0);
3656 Value *Op1 = CI->getArgOperand(1);
3657 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3658 unsigned DstNumElts =
3659 cast<FixedVectorType>(CI->getType())->getNumElements();
3660 unsigned SrcNumElts =
3661 cast<FixedVectorType>(Op1->getType())->getNumElements();
3662 unsigned Scale = DstNumElts / SrcNumElts;
3663
3664 // Mask off the high bits of the immediate value; hardware ignores those.
3665 Imm = Imm % Scale;
3666
3667 // Extend the second operand into a vector the size of the destination.
3668 SmallVector<int, 8> Idxs(DstNumElts);
3669 for (unsigned i = 0; i != SrcNumElts; ++i)
3670 Idxs[i] = i;
3671 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3672 Idxs[i] = SrcNumElts;
3673 Rep = Builder.CreateShuffleVector(Op1, Idxs);
3674
3675 // Insert the second operand into the first operand.
3676
3677 // Note that there is no guarantee that instruction lowering will actually
3678 // produce a vinsertf128 instruction for the created shuffles. In
3679 // particular, the 0 immediate case involves no lane changes, so it can
3680 // be handled as a blend.
3681
3682 // Example of shuffle mask for 32-bit elements:
3683 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3684 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3685
3686 // First fill with identify mask.
3687 for (unsigned i = 0; i != DstNumElts; ++i)
3688 Idxs[i] = i;
3689 // Then replace the elements where we need to insert.
3690 for (unsigned i = 0; i != SrcNumElts; ++i)
3691 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3692 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
3693
3694 // If the intrinsic has a mask operand, handle that.
3695 if (CI->arg_size() == 5)
3696 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
3697 CI->getArgOperand(3));
3698 } else if (Name.starts_with("avx.vextractf128.") ||
3699 Name == "avx2.vextracti128" ||
3700 Name.starts_with("avx512.mask.vextract")) {
3701 Value *Op0 = CI->getArgOperand(0);
3702 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3703 unsigned DstNumElts =
3704 cast<FixedVectorType>(CI->getType())->getNumElements();
3705 unsigned SrcNumElts =
3706 cast<FixedVectorType>(Op0->getType())->getNumElements();
3707 unsigned Scale = SrcNumElts / DstNumElts;
3708
3709 // Mask off the high bits of the immediate value; hardware ignores those.
3710 Imm = Imm % Scale;
3711
3712 // Get indexes for the subvector of the input vector.
3713 SmallVector<int, 8> Idxs(DstNumElts);
3714 for (unsigned i = 0; i != DstNumElts; ++i) {
3715 Idxs[i] = i + (Imm * DstNumElts);
3716 }
3717 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3718
3719 // If the intrinsic has a mask operand, handle that.
3720 if (CI->arg_size() == 4)
3721 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3722 CI->getArgOperand(2));
3723 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3724 Name.starts_with("avx512.mask.perm.di.")) {
3725 Value *Op0 = CI->getArgOperand(0);
3726 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3727 auto *VecTy = cast<FixedVectorType>(CI->getType());
3728 unsigned NumElts = VecTy->getNumElements();
3729
3730 SmallVector<int, 8> Idxs(NumElts);
3731 for (unsigned i = 0; i != NumElts; ++i)
3732 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
3733
3734 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3735
3736 if (CI->arg_size() == 4)
3737 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3738 CI->getArgOperand(2));
3739 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
3740 // The immediate permute control byte looks like this:
3741 // [1:0] - select 128 bits from sources for low half of destination
3742 // [2] - ignore
3743 // [3] - zero low half of destination
3744 // [5:4] - select 128 bits from sources for high half of destination
3745 // [6] - ignore
3746 // [7] - zero high half of destination
3747
3748 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3749
3750 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3751 unsigned HalfSize = NumElts / 2;
3752 SmallVector<int, 8> ShuffleMask(NumElts);
3753
3754 // Determine which operand(s) are actually in use for this instruction.
3755 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3756 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3757
3758 // If needed, replace operands based on zero mask.
3759 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3760 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3761
3762 // Permute low half of result.
3763 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
3764 for (unsigned i = 0; i < HalfSize; ++i)
3765 ShuffleMask[i] = StartIndex + i;
3766
3767 // Permute high half of result.
3768 StartIndex = (Imm & 0x10) ? HalfSize : 0;
3769 for (unsigned i = 0; i < HalfSize; ++i)
3770 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
3771
3772 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3773
3774 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
3775 Name.starts_with("avx512.mask.vpermil.p") ||
3776 Name.starts_with("avx512.mask.pshuf.d.")) {
3777 Value *Op0 = CI->getArgOperand(0);
3778 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3779 auto *VecTy = cast<FixedVectorType>(CI->getType());
3780 unsigned NumElts = VecTy->getNumElements();
3781 // Calculate the size of each index in the immediate.
3782 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
3783 unsigned IdxMask = ((1 << IdxSize) - 1);
3784
3785 SmallVector<int, 8> Idxs(NumElts);
3786 // Lookup the bits for this element, wrapping around the immediate every
3787 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
3788 // to offset by the first index of each group.
3789 for (unsigned i = 0; i != NumElts; ++i)
3790 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
3791
3792 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3793
3794 if (CI->arg_size() == 4)
3795 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3796 CI->getArgOperand(2));
3797 } else if (Name == "sse2.pshufl.w" ||
3798 Name.starts_with("avx512.mask.pshufl.w.")) {
3799 Value *Op0 = CI->getArgOperand(0);
3800 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3801 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3802
3803 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
3804 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
3805
3806 SmallVector<int, 16> Idxs(NumElts);
3807 for (unsigned l = 0; l != NumElts; l += 8) {
3808 for (unsigned i = 0; i != 4; ++i)
3809 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
3810 for (unsigned i = 4; i != 8; ++i)
3811 Idxs[i + l] = i + l;
3812 }
3813
3814 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3815
3816 if (CI->arg_size() == 4)
3817 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3818 CI->getArgOperand(2));
3819 } else if (Name == "sse2.pshufh.w" ||
3820 Name.starts_with("avx512.mask.pshufh.w.")) {
3821 Value *Op0 = CI->getArgOperand(0);
3822 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3823 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3824
3825 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
3826 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
3827
3828 SmallVector<int, 16> Idxs(NumElts);
3829 for (unsigned l = 0; l != NumElts; l += 8) {
3830 for (unsigned i = 0; i != 4; ++i)
3831 Idxs[i + l] = i + l;
3832 for (unsigned i = 0; i != 4; ++i)
3833 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
3834 }
3835
3836 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3837
3838 if (CI->arg_size() == 4)
3839 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3840 CI->getArgOperand(2));
3841 } else if (Name.starts_with("avx512.mask.shuf.p")) {
3842 Value *Op0 = CI->getArgOperand(0);
3843 Value *Op1 = CI->getArgOperand(1);
3844 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3845 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3846
3847 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3848 unsigned HalfLaneElts = NumLaneElts / 2;
3849
3850 SmallVector<int, 16> Idxs(NumElts);
3851 for (unsigned i = 0; i != NumElts; ++i) {
3852 // Base index is the starting element of the lane.
3853 Idxs[i] = i - (i % NumLaneElts);
3854 // If we are half way through the lane switch to the other source.
3855 if ((i % NumLaneElts) >= HalfLaneElts)
3856 Idxs[i] += NumElts;
3857 // Now select the specific element. By adding HalfLaneElts bits from
3858 // the immediate. Wrapping around the immediate every 8-bits.
3859 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
3860 }
3861
3862 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3863
3864 Rep =
3865 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3866 } else if (Name.starts_with("avx512.mask.movddup") ||
3867 Name.starts_with("avx512.mask.movshdup") ||
3868 Name.starts_with("avx512.mask.movsldup")) {
3869 Value *Op0 = CI->getArgOperand(0);
3870 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3871 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3872
3873 unsigned Offset = 0;
3874 if (Name.starts_with("avx512.mask.movshdup."))
3875 Offset = 1;
3876
3877 SmallVector<int, 16> Idxs(NumElts);
3878 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
3879 for (unsigned i = 0; i != NumLaneElts; i += 2) {
3880 Idxs[i + l + 0] = i + l + Offset;
3881 Idxs[i + l + 1] = i + l + Offset;
3882 }
3883
3884 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3885
3886 Rep =
3887 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3888 } else if (Name.starts_with("avx512.mask.punpckl") ||
3889 Name.starts_with("avx512.mask.unpckl.")) {
3890 Value *Op0 = CI->getArgOperand(0);
3891 Value *Op1 = CI->getArgOperand(1);
3892 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3893 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3894
3895 SmallVector<int, 64> Idxs(NumElts);
3896 for (int l = 0; l != NumElts; l += NumLaneElts)
3897 for (int i = 0; i != NumLaneElts; ++i)
3898 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
3899
3900 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3901
3902 Rep =
3903 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3904 } else if (Name.starts_with("avx512.mask.punpckh") ||
3905 Name.starts_with("avx512.mask.unpckh.")) {
3906 Value *Op0 = CI->getArgOperand(0);
3907 Value *Op1 = CI->getArgOperand(1);
3908 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3909 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3910
3911 SmallVector<int, 64> Idxs(NumElts);
3912 for (int l = 0; l != NumElts; l += NumLaneElts)
3913 for (int i = 0; i != NumLaneElts; ++i)
3914 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
3915
3916 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3917
3918 Rep =
3919 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3920 } else if (Name.starts_with("avx512.mask.and.") ||
3921 Name.starts_with("avx512.mask.pand.")) {
3922 VectorType *FTy = cast<VectorType>(CI->getType());
3923 VectorType *ITy = VectorType::getInteger(FTy);
3924 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3925 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3926 Rep = Builder.CreateBitCast(Rep, FTy);
3927 Rep =
3928 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3929 } else if (Name.starts_with("avx512.mask.andn.") ||
3930 Name.starts_with("avx512.mask.pandn.")) {
3931 VectorType *FTy = cast<VectorType>(CI->getType());
3932 VectorType *ITy = VectorType::getInteger(FTy);
3933 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
3934 Rep = Builder.CreateAnd(Rep,
3935 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3936 Rep = Builder.CreateBitCast(Rep, FTy);
3937 Rep =
3938 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3939 } else if (Name.starts_with("avx512.mask.or.") ||
3940 Name.starts_with("avx512.mask.por.")) {
3941 VectorType *FTy = cast<VectorType>(CI->getType());
3942 VectorType *ITy = VectorType::getInteger(FTy);
3943 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3944 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3945 Rep = Builder.CreateBitCast(Rep, FTy);
3946 Rep =
3947 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3948 } else if (Name.starts_with("avx512.mask.xor.") ||
3949 Name.starts_with("avx512.mask.pxor.")) {
3950 VectorType *FTy = cast<VectorType>(CI->getType());
3951 VectorType *ITy = VectorType::getInteger(FTy);
3952 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3953 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3954 Rep = Builder.CreateBitCast(Rep, FTy);
3955 Rep =
3956 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3957 } else if (Name.starts_with("avx512.mask.padd.")) {
3958 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3959 Rep =
3960 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3961 } else if (Name.starts_with("avx512.mask.psub.")) {
3962 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
3963 Rep =
3964 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3965 } else if (Name.starts_with("avx512.mask.pmull.")) {
3966 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
3967 Rep =
3968 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3969 } else if (Name.starts_with("avx512.mask.add.p")) {
3970 if (Name.ends_with(".512")) {
3971 Intrinsic::ID IID;
3972 if (Name[17] == 's')
3973 IID = Intrinsic::x86_avx512_add_ps_512;
3974 else
3975 IID = Intrinsic::x86_avx512_add_pd_512;
3976
3977 Rep = Builder.CreateIntrinsic(
3978 IID,
3979 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3980 } else {
3981 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3982 }
3983 Rep =
3984 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3985 } else if (Name.starts_with("avx512.mask.div.p")) {
3986 if (Name.ends_with(".512")) {
3987 Intrinsic::ID IID;
3988 if (Name[17] == 's')
3989 IID = Intrinsic::x86_avx512_div_ps_512;
3990 else
3991 IID = Intrinsic::x86_avx512_div_pd_512;
3992
3993 Rep = Builder.CreateIntrinsic(
3994 IID,
3995 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3996 } else {
3997 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
3998 }
3999 Rep =
4000 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4001 } else if (Name.starts_with("avx512.mask.mul.p")) {
4002 if (Name.ends_with(".512")) {
4003 Intrinsic::ID IID;
4004 if (Name[17] == 's')
4005 IID = Intrinsic::x86_avx512_mul_ps_512;
4006 else
4007 IID = Intrinsic::x86_avx512_mul_pd_512;
4008
4009 Rep = Builder.CreateIntrinsic(
4010 IID,
4011 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4012 } else {
4013 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4014 }
4015 Rep =
4016 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4017 } else if (Name.starts_with("avx512.mask.sub.p")) {
4018 if (Name.ends_with(".512")) {
4019 Intrinsic::ID IID;
4020 if (Name[17] == 's')
4021 IID = Intrinsic::x86_avx512_sub_ps_512;
4022 else
4023 IID = Intrinsic::x86_avx512_sub_pd_512;
4024
4025 Rep = Builder.CreateIntrinsic(
4026 IID,
4027 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4028 } else {
4029 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4030 }
4031 Rep =
4032 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4033 } else if ((Name.starts_with("avx512.mask.max.p") ||
4034 Name.starts_with("avx512.mask.min.p")) &&
4035 Name.drop_front(18) == ".512") {
4036 bool IsDouble = Name[17] == 'd';
4037 bool IsMin = Name[13] == 'i';
4038 static const Intrinsic::ID MinMaxTbl[2][2] = {
4039 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4040 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4041 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4042
4043 Rep = Builder.CreateIntrinsic(
4044 IID,
4045 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4046 Rep =
4047 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4048 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4049 Rep =
4050 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4051 {CI->getArgOperand(0), Builder.getInt1(false)});
4052 Rep =
4053 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4054 } else if (Name.starts_with("avx512.mask.psll")) {
4055 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4056 bool IsVariable = Name[16] == 'v';
4057 char Size = Name[16] == '.' ? Name[17]
4058 : Name[17] == '.' ? Name[18]
4059 : Name[18] == '.' ? Name[19]
4060 : Name[20];
4061
4062 Intrinsic::ID IID;
4063 if (IsVariable && Name[17] != '.') {
4064 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4065 IID = Intrinsic::x86_avx2_psllv_q;
4066 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4067 IID = Intrinsic::x86_avx2_psllv_q_256;
4068 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4069 IID = Intrinsic::x86_avx2_psllv_d;
4070 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4071 IID = Intrinsic::x86_avx2_psllv_d_256;
4072 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4073 IID = Intrinsic::x86_avx512_psllv_w_128;
4074 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4075 IID = Intrinsic::x86_avx512_psllv_w_256;
4076 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4077 IID = Intrinsic::x86_avx512_psllv_w_512;
4078 else
4079 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4080 } else if (Name.ends_with(".128")) {
4081 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4082 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4083 : Intrinsic::x86_sse2_psll_d;
4084 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4085 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4086 : Intrinsic::x86_sse2_psll_q;
4087 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4088 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4089 : Intrinsic::x86_sse2_psll_w;
4090 else
4091 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4092 } else if (Name.ends_with(".256")) {
4093 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4094 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4095 : Intrinsic::x86_avx2_psll_d;
4096 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4097 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4098 : Intrinsic::x86_avx2_psll_q;
4099 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4100 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4101 : Intrinsic::x86_avx2_psll_w;
4102 else
4103 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4104 } else {
4105 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4106 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4107 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4108 : Intrinsic::x86_avx512_psll_d_512;
4109 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4110 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4111 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4112 : Intrinsic::x86_avx512_psll_q_512;
4113 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4114 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4115 : Intrinsic::x86_avx512_psll_w_512;
4116 else
4117 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4118 }
4119
4120 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4121 } else if (Name.starts_with("avx512.mask.psrl")) {
4122 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4123 bool IsVariable = Name[16] == 'v';
4124 char Size = Name[16] == '.' ? Name[17]
4125 : Name[17] == '.' ? Name[18]
4126 : Name[18] == '.' ? Name[19]
4127 : Name[20];
4128
4129 Intrinsic::ID IID;
4130 if (IsVariable && Name[17] != '.') {
4131 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4132 IID = Intrinsic::x86_avx2_psrlv_q;
4133 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4134 IID = Intrinsic::x86_avx2_psrlv_q_256;
4135 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4136 IID = Intrinsic::x86_avx2_psrlv_d;
4137 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4138 IID = Intrinsic::x86_avx2_psrlv_d_256;
4139 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4140 IID = Intrinsic::x86_avx512_psrlv_w_128;
4141 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4142 IID = Intrinsic::x86_avx512_psrlv_w_256;
4143 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4144 IID = Intrinsic::x86_avx512_psrlv_w_512;
4145 else
4146 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4147 } else if (Name.ends_with(".128")) {
4148 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4149 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4150 : Intrinsic::x86_sse2_psrl_d;
4151 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4152 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4153 : Intrinsic::x86_sse2_psrl_q;
4154 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4155 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4156 : Intrinsic::x86_sse2_psrl_w;
4157 else
4158 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4159 } else if (Name.ends_with(".256")) {
4160 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4161 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4162 : Intrinsic::x86_avx2_psrl_d;
4163 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4164 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4165 : Intrinsic::x86_avx2_psrl_q;
4166 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4167 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4168 : Intrinsic::x86_avx2_psrl_w;
4169 else
4170 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4171 } else {
4172 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4173 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4174 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4175 : Intrinsic::x86_avx512_psrl_d_512;
4176 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4177 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4178 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4179 : Intrinsic::x86_avx512_psrl_q_512;
4180 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4181 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4182 : Intrinsic::x86_avx512_psrl_w_512;
4183 else
4184 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4185 }
4186
4187 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4188 } else if (Name.starts_with("avx512.mask.psra")) {
4189 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4190 bool IsVariable = Name[16] == 'v';
4191 char Size = Name[16] == '.' ? Name[17]
4192 : Name[17] == '.' ? Name[18]
4193 : Name[18] == '.' ? Name[19]
4194 : Name[20];
4195
4196 Intrinsic::ID IID;
4197 if (IsVariable && Name[17] != '.') {
4198 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4199 IID = Intrinsic::x86_avx2_psrav_d;
4200 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4201 IID = Intrinsic::x86_avx2_psrav_d_256;
4202 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4203 IID = Intrinsic::x86_avx512_psrav_w_128;
4204 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4205 IID = Intrinsic::x86_avx512_psrav_w_256;
4206 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4207 IID = Intrinsic::x86_avx512_psrav_w_512;
4208 else
4209 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4210 } else if (Name.ends_with(".128")) {
4211 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4212 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4213 : Intrinsic::x86_sse2_psra_d;
4214 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4215 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4216 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4217 : Intrinsic::x86_avx512_psra_q_128;
4218 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4219 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4220 : Intrinsic::x86_sse2_psra_w;
4221 else
4222 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4223 } else if (Name.ends_with(".256")) {
4224 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4225 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4226 : Intrinsic::x86_avx2_psra_d;
4227 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4228 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4229 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4230 : Intrinsic::x86_avx512_psra_q_256;
4231 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4232 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4233 : Intrinsic::x86_avx2_psra_w;
4234 else
4235 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4236 } else {
4237 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4238 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4239 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4240 : Intrinsic::x86_avx512_psra_d_512;
4241 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4242 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4243 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4244 : Intrinsic::x86_avx512_psra_q_512;
4245 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4246 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4247 : Intrinsic::x86_avx512_psra_w_512;
4248 else
4249 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4250 }
4251
4252 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4253 } else if (Name.starts_with("avx512.mask.move.s")) {
4254 Rep = upgradeMaskedMove(Builder, *CI);
4255 } else if (Name.starts_with("avx512.cvtmask2")) {
4256 Rep = upgradeMaskToInt(Builder, *CI);
4257 } else if (Name.ends_with(".movntdqa")) {
4258 MDNode *Node = MDNode::get(
4259 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4260
4261 LoadInst *LI = Builder.CreateAlignedLoad(
4262 CI->getType(), CI->getArgOperand(0),
4263 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4264 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4265 Rep = LI;
4266 } else if (Name.starts_with("fma.vfmadd.") ||
4267 Name.starts_with("fma.vfmsub.") ||
4268 Name.starts_with("fma.vfnmadd.") ||
4269 Name.starts_with("fma.vfnmsub.")) {
4270 bool NegMul = Name[6] == 'n';
4271 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4272 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4273
4274 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4275 CI->getArgOperand(2)};
4276
4277 if (IsScalar) {
4278 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4279 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4280 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4281 }
4282
4283 if (NegMul && !IsScalar)
4284 Ops[0] = Builder.CreateFNeg(Ops[0]);
4285 if (NegMul && IsScalar)
4286 Ops[1] = Builder.CreateFNeg(Ops[1]);
4287 if (NegAcc)
4288 Ops[2] = Builder.CreateFNeg(Ops[2]);
4289
4290 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4291
4292 if (IsScalar)
4293 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4294 } else if (Name.starts_with("fma4.vfmadd.s")) {
4295 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4296 CI->getArgOperand(2)};
4297
4298 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4299 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4300 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4301
4302 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4303
4304 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4305 Rep, (uint64_t)0);
4306 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4307 Name.starts_with("avx512.maskz.vfmadd.s") ||
4308 Name.starts_with("avx512.mask3.vfmadd.s") ||
4309 Name.starts_with("avx512.mask3.vfmsub.s") ||
4310 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4311 bool IsMask3 = Name[11] == '3';
4312 bool IsMaskZ = Name[11] == 'z';
4313 // Drop the "avx512.mask." to make it easier.
4314 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4315 bool NegMul = Name[2] == 'n';
4316 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4317
4318 Value *A = CI->getArgOperand(0);
4319 Value *B = CI->getArgOperand(1);
4320 Value *C = CI->getArgOperand(2);
4321
4322 if (NegMul && (IsMask3 || IsMaskZ))
4323 A = Builder.CreateFNeg(A);
4324 if (NegMul && !(IsMask3 || IsMaskZ))
4325 B = Builder.CreateFNeg(B);
4326 if (NegAcc)
4327 C = Builder.CreateFNeg(C);
4328
4329 A = Builder.CreateExtractElement(A, (uint64_t)0);
4330 B = Builder.CreateExtractElement(B, (uint64_t)0);
4331 C = Builder.CreateExtractElement(C, (uint64_t)0);
4332
4333 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4334 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4335 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4336
4337 Intrinsic::ID IID;
4338 if (Name.back() == 'd')
4339 IID = Intrinsic::x86_avx512_vfmadd_f64;
4340 else
4341 IID = Intrinsic::x86_avx512_vfmadd_f32;
4342 Rep = Builder.CreateIntrinsic(IID, Ops);
4343 } else {
4344 Rep = Builder.CreateFMA(A, B, C);
4345 }
4346
4347 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4348 : IsMask3 ? C
4349 : A;
4350
4351 // For Mask3 with NegAcc, we need to create a new extractelement that
4352 // avoids the negation above.
4353 if (NegAcc && IsMask3)
4354 PassThru =
4355 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4356
4357 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4358 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4359 (uint64_t)0);
4360 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4361 Name.starts_with("avx512.mask.vfnmadd.p") ||
4362 Name.starts_with("avx512.mask.vfnmsub.p") ||
4363 Name.starts_with("avx512.mask3.vfmadd.p") ||
4364 Name.starts_with("avx512.mask3.vfmsub.p") ||
4365 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4366 Name.starts_with("avx512.maskz.vfmadd.p")) {
4367 bool IsMask3 = Name[11] == '3';
4368 bool IsMaskZ = Name[11] == 'z';
4369 // Drop the "avx512.mask." to make it easier.
4370 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4371 bool NegMul = Name[2] == 'n';
4372 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4373
4374 Value *A = CI->getArgOperand(0);
4375 Value *B = CI->getArgOperand(1);
4376 Value *C = CI->getArgOperand(2);
4377
4378 if (NegMul && (IsMask3 || IsMaskZ))
4379 A = Builder.CreateFNeg(A);
4380 if (NegMul && !(IsMask3 || IsMaskZ))
4381 B = Builder.CreateFNeg(B);
4382 if (NegAcc)
4383 C = Builder.CreateFNeg(C);
4384
4385 if (CI->arg_size() == 5 &&
4386 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4387 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4388 Intrinsic::ID IID;
4389 // Check the character before ".512" in string.
4390 if (Name[Name.size() - 5] == 's')
4391 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4392 else
4393 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4394
4395 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4396 } else {
4397 Rep = Builder.CreateFMA(A, B, C);
4398 }
4399
4400 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4401 : IsMask3 ? CI->getArgOperand(2)
4402 : CI->getArgOperand(0);
4403
4404 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4405 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4406 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4407 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4408 Intrinsic::ID IID;
4409 if (VecWidth == 128 && EltWidth == 32)
4410 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4411 else if (VecWidth == 256 && EltWidth == 32)
4412 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4413 else if (VecWidth == 128 && EltWidth == 64)
4414 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4415 else if (VecWidth == 256 && EltWidth == 64)
4416 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4417 else
4418 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4419
4420 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4421 CI->getArgOperand(2)};
4422 Ops[2] = Builder.CreateFNeg(Ops[2]);
4423 Rep = Builder.CreateIntrinsic(IID, Ops);
4424 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4425 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4426 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4427 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4428 bool IsMask3 = Name[11] == '3';
4429 bool IsMaskZ = Name[11] == 'z';
4430 // Drop the "avx512.mask." to make it easier.
4431 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4432 bool IsSubAdd = Name[3] == 's';
4433 if (CI->arg_size() == 5) {
4434 Intrinsic::ID IID;
4435 // Check the character before ".512" in string.
4436 if (Name[Name.size() - 5] == 's')
4437 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4438 else
4439 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4440
4441 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4442 CI->getArgOperand(2), CI->getArgOperand(4)};
4443 if (IsSubAdd)
4444 Ops[2] = Builder.CreateFNeg(Ops[2]);
4445
4446 Rep = Builder.CreateIntrinsic(IID, Ops);
4447 } else {
4448 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4449
4450 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4451 CI->getArgOperand(2)};
4452
4453 Function *FMA = Intrinsic::getOrInsertDeclaration(
4454 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4455 Value *Odd = Builder.CreateCall(FMA, Ops);
4456 Ops[2] = Builder.CreateFNeg(Ops[2]);
4457 Value *Even = Builder.CreateCall(FMA, Ops);
4458
4459 if (IsSubAdd)
4460 std::swap(Even, Odd);
4461
4462 SmallVector<int, 32> Idxs(NumElts);
4463 for (int i = 0; i != NumElts; ++i)
4464 Idxs[i] = i + (i % 2) * NumElts;
4465
4466 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4467 }
4468
4469 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4470 : IsMask3 ? CI->getArgOperand(2)
4471 : CI->getArgOperand(0);
4472
4473 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4474 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4475 Name.starts_with("avx512.maskz.pternlog.")) {
4476 bool ZeroMask = Name[11] == 'z';
4477 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4478 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4479 Intrinsic::ID IID;
4480 if (VecWidth == 128 && EltWidth == 32)
4481 IID = Intrinsic::x86_avx512_pternlog_d_128;
4482 else if (VecWidth == 256 && EltWidth == 32)
4483 IID = Intrinsic::x86_avx512_pternlog_d_256;
4484 else if (VecWidth == 512 && EltWidth == 32)
4485 IID = Intrinsic::x86_avx512_pternlog_d_512;
4486 else if (VecWidth == 128 && EltWidth == 64)
4487 IID = Intrinsic::x86_avx512_pternlog_q_128;
4488 else if (VecWidth == 256 && EltWidth == 64)
4489 IID = Intrinsic::x86_avx512_pternlog_q_256;
4490 else if (VecWidth == 512 && EltWidth == 64)
4491 IID = Intrinsic::x86_avx512_pternlog_q_512;
4492 else
4493 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4494
4495 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4496 CI->getArgOperand(2), CI->getArgOperand(3)};
4497 Rep = Builder.CreateIntrinsic(IID, Args);
4498 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4499 : CI->getArgOperand(0);
4500 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4501 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4502 Name.starts_with("avx512.maskz.vpmadd52")) {
4503 bool ZeroMask = Name[11] == 'z';
4504 bool High = Name[20] == 'h' || Name[21] == 'h';
4505 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4506 Intrinsic::ID IID;
4507 if (VecWidth == 128 && !High)
4508 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4509 else if (VecWidth == 256 && !High)
4510 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4511 else if (VecWidth == 512 && !High)
4512 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4513 else if (VecWidth == 128 && High)
4514 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4515 else if (VecWidth == 256 && High)
4516 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4517 else if (VecWidth == 512 && High)
4518 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4519 else
4520 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4521
4522 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4523 CI->getArgOperand(2)};
4524 Rep = Builder.CreateIntrinsic(IID, Args);
4525 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4526 : CI->getArgOperand(0);
4527 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4528 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4529 Name.starts_with("avx512.mask.vpermt2var.") ||
4530 Name.starts_with("avx512.maskz.vpermt2var.")) {
4531 bool ZeroMask = Name[11] == 'z';
4532 bool IndexForm = Name[17] == 'i';
4533 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4534 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4535 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4536 Name.starts_with("avx512.mask.vpdpbusds.") ||
4537 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4538 bool ZeroMask = Name[11] == 'z';
4539 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4540 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4541 Intrinsic::ID IID;
4542 if (VecWidth == 128 && !IsSaturating)
4543 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4544 else if (VecWidth == 256 && !IsSaturating)
4545 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4546 else if (VecWidth == 512 && !IsSaturating)
4547 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4548 else if (VecWidth == 128 && IsSaturating)
4549 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4550 else if (VecWidth == 256 && IsSaturating)
4551 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4552 else if (VecWidth == 512 && IsSaturating)
4553 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4554 else
4555 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4556
4557 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4558 CI->getArgOperand(2)};
4559
4560 // Input arguments types were incorrectly set to vectors of i32 before but
4561 // they should be vectors of i8. Insert bit cast when encountering the old
4562 // types
4563 if (Args[1]->getType()->isVectorTy() &&
4564 cast<VectorType>(Args[1]->getType())
4565 ->getElementType()
4566 ->isIntegerTy(32) &&
4567 Args[2]->getType()->isVectorTy() &&
4568 cast<VectorType>(Args[2]->getType())
4569 ->getElementType()
4570 ->isIntegerTy(32)) {
4571 Type *NewArgType = nullptr;
4572 if (VecWidth == 128)
4573 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4574 else if (VecWidth == 256)
4575 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4576 else if (VecWidth == 512)
4577 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4578 else
4579 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4580 CI);
4581
4582 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4583 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4584 }
4585
4586 Rep = Builder.CreateIntrinsic(IID, Args);
4587 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4588 : CI->getArgOperand(0);
4589 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4590 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4591 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4592 Name.starts_with("avx512.mask.vpdpwssds.") ||
4593 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4594 bool ZeroMask = Name[11] == 'z';
4595 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4596 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4597 Intrinsic::ID IID;
4598 if (VecWidth == 128 && !IsSaturating)
4599 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4600 else if (VecWidth == 256 && !IsSaturating)
4601 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4602 else if (VecWidth == 512 && !IsSaturating)
4603 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4604 else if (VecWidth == 128 && IsSaturating)
4605 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4606 else if (VecWidth == 256 && IsSaturating)
4607 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4608 else if (VecWidth == 512 && IsSaturating)
4609 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4610 else
4611 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4612
4613 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4614 CI->getArgOperand(2)};
4615
4616 // Input arguments types were incorrectly set to vectors of i32 before but
4617 // they should be vectors of i16. Insert bit cast when encountering the old
4618 // types
4619 if (Args[1]->getType()->isVectorTy() &&
4620 cast<VectorType>(Args[1]->getType())
4621 ->getElementType()
4622 ->isIntegerTy(32) &&
4623 Args[2]->getType()->isVectorTy() &&
4624 cast<VectorType>(Args[2]->getType())
4625 ->getElementType()
4626 ->isIntegerTy(32)) {
4627 Type *NewArgType = nullptr;
4628 if (VecWidth == 128)
4629 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
4630 else if (VecWidth == 256)
4631 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
4632 else if (VecWidth == 512)
4633 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
4634 else
4635 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4636 CI);
4637
4638 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4639 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4640 }
4641
4642 Rep = Builder.CreateIntrinsic(IID, Args);
4643 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4644 : CI->getArgOperand(0);
4645 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4646 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4647 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4648 Name == "subborrow.u32" || Name == "subborrow.u64") {
4649 Intrinsic::ID IID;
4650 if (Name[0] == 'a' && Name.back() == '2')
4651 IID = Intrinsic::x86_addcarry_32;
4652 else if (Name[0] == 'a' && Name.back() == '4')
4653 IID = Intrinsic::x86_addcarry_64;
4654 else if (Name[0] == 's' && Name.back() == '2')
4655 IID = Intrinsic::x86_subborrow_32;
4656 else if (Name[0] == 's' && Name.back() == '4')
4657 IID = Intrinsic::x86_subborrow_64;
4658 else
4659 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4660
4661 // Make a call with 3 operands.
4662 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4663 CI->getArgOperand(2)};
4664 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
4665
4666 // Extract the second result and store it.
4667 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4668 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
4669 // Replace the original call result with the first result of the new call.
4670 Value *CF = Builder.CreateExtractValue(NewCall, 0);
4671
4672 CI->replaceAllUsesWith(CF);
4673 Rep = nullptr;
4674 } else if (Name.starts_with("avx512.mask.") &&
4675 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
4676 // Rep will be updated by the call in the condition.
4677 } else if (Name.starts_with("bmi.pdep.")) {
4678 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
4679 } else if (Name.starts_with("bmi.pext.")) {
4680 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
4681 } else
4682 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4683
4684 return Rep;
4685}
4686
4687static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4688 Function *F, IRBuilder<> &Builder) {
4689 if (Name.starts_with("neon.bfcvt")) {
4690 if (Name.starts_with("neon.bfcvtn2")) {
4691 SmallVector<int, 32> LoMask(4);
4692 std::iota(LoMask.begin(), LoMask.end(), 0);
4693 SmallVector<int, 32> ConcatMask(8);
4694 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4695 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
4696 Value *Trunc =
4697 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
4698 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
4699 } else if (Name.starts_with("neon.bfcvtn")) {
4700 SmallVector<int, 32> ConcatMask(8);
4701 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4702 Type *V4BF16 =
4703 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
4704 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
4705 dbgs() << "Trunc: " << *Trunc << "\n";
4706 return Builder.CreateShuffleVector(
4707 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
4708 } else {
4709 return Builder.CreateFPTrunc(CI->getOperand(0),
4710 Type::getBFloatTy(F->getContext()));
4711 }
4712 } else if (Name.starts_with("sve.fcvt")) {
4713 Intrinsic::ID NewID =
4714 StringSwitch<Intrinsic::ID>(Name)
4715 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4716 .Case("sve.fcvtnt.bf16f32",
4717 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4718 .Default(Intrinsic::not_intrinsic);
4719 if (NewID == Intrinsic::not_intrinsic)
4720 llvm_unreachable("Unhandled Intrinsic!");
4721
4722 SmallVector<Value *, 3> Args(CI->args());
4723
4724 // The original intrinsics incorrectly used a predicate based on the
4725 // smallest element type rather than the largest.
4726 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
4727 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
4728
4729 if (Args[1]->getType() != BadPredTy)
4730 llvm_unreachable("Unexpected predicate type!");
4731
4732 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
4733 BadPredTy, Args[1]);
4734 Args[1] = Builder.CreateIntrinsic(
4735 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
4736
4737 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
4738 CI->getName());
4739 }
4740
4741 if (Name == "neon.vcvtfp2hf")
4742 return Builder.CreateBitCast(
4743 Builder.CreateFPTrunc(
4744 CI->getOperand(0),
4745 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4746 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
4747 if (Name == "neon.vcvthf2fp")
4748 return Builder.CreateFPExt(
4749 Builder.CreateBitCast(
4750 CI->getOperand(0),
4751 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
4752 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
4753
4754 llvm_unreachable("Unhandled Intrinsic!");
4755}
4756
4757static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
4758 IRBuilder<> &Builder) {
4759 if (Name == "mve.vctp64.old") {
4760 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
4761 // correct type.
4762 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
4763 CI->getArgOperand(0),
4764 /*FMFSource=*/nullptr, CI->getName());
4765 Value *C1 = Builder.CreateIntrinsic(
4766 Intrinsic::arm_mve_pred_v2i,
4767 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
4768 return Builder.CreateIntrinsic(
4769 Intrinsic::arm_mve_pred_i2v,
4770 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
4771 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
4772 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
4773 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
4774 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
4775 Name ==
4776 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
4777 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
4778 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
4779 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
4780 Name ==
4781 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
4782 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
4783 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
4784 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
4785 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
4786 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
4787 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
4788 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
4789 std::vector<Type *> Tys;
4790 unsigned ID = CI->getIntrinsicID();
4791 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
4792 switch (ID) {
4793 case Intrinsic::arm_mve_mull_int_predicated:
4794 case Intrinsic::arm_mve_vqdmull_predicated:
4795 case Intrinsic::arm_mve_vldr_gather_base_predicated:
4796 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
4797 break;
4798 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
4799 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
4800 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
4801 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
4802 V2I1Ty};
4803 break;
4804 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
4805 Tys = {CI->getType(), CI->getOperand(0)->getType(),
4806 CI->getOperand(1)->getType(), V2I1Ty};
4807 break;
4808 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
4809 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
4810 CI->getOperand(2)->getType(), V2I1Ty};
4811 break;
4812 case Intrinsic::arm_cde_vcx1q_predicated:
4813 case Intrinsic::arm_cde_vcx1qa_predicated:
4814 case Intrinsic::arm_cde_vcx2q_predicated:
4815 case Intrinsic::arm_cde_vcx2qa_predicated:
4816 case Intrinsic::arm_cde_vcx3q_predicated:
4817 case Intrinsic::arm_cde_vcx3qa_predicated:
4818 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
4819 break;
4820 default:
4821 llvm_unreachable("Unhandled Intrinsic!");
4822 }
4823
4824 std::vector<Value *> Ops;
4825 for (Value *Op : CI->args()) {
4826 Type *Ty = Op->getType();
4827 if (Ty->getScalarSizeInBits() == 1) {
4828 Value *C1 = Builder.CreateIntrinsic(
4829 Intrinsic::arm_mve_pred_v2i,
4830 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
4831 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
4832 }
4833 Ops.push_back(Op);
4834 }
4835
4836 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
4837 CI->getName());
4838 }
4839 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
4840}
4841
4842// These are expected to have the arguments:
4843// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
4844//
4845// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
4846//
4847static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
4848 Function *F, IRBuilder<> &Builder) {
4849 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
4850 // for compatibility.
4851 auto UpgradeLegacyWMMAIUIntrinsicCall =
4852 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
4853 ArrayRef<Type *> OverloadTys) -> Value * {
4854 // Prepare arguments, append clamp=0 for compatibility
4855 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
4856 Args.push_back(Builder.getFalse());
4857
4858 // Insert the declaration for the right overload types
4859 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4860 F->getParent(), F->getIntrinsicID(), OverloadTys);
4861
4862 // Copy operand bundles if any
4863 SmallVector<OperandBundleDef, 1> Bundles;
4864 CI->getOperandBundlesAsDefs(Bundles);
4865
4866 // Create the new call and copy calling properties
4867 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
4868 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4869 NewCall->setCallingConv(CI->getCallingConv());
4870 NewCall->setAttributes(CI->getAttributes());
4871 NewCall->setDebugLoc(CI->getDebugLoc());
4872 NewCall->copyMetadata(*CI);
4873 return NewCall;
4874 };
4875
4876 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
4877 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
4878 "intrinsic should have 7 arguments");
4879 Type *T1 = CI->getArgOperand(4)->getType();
4880 Type *T2 = CI->getArgOperand(1)->getType();
4881 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
4882 }
4883 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
4884 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
4885 "intrinsic should have 8 arguments");
4886 Type *T1 = CI->getArgOperand(4)->getType();
4887 Type *T2 = CI->getArgOperand(1)->getType();
4888 Type *T3 = CI->getArgOperand(3)->getType();
4889 Type *T4 = CI->getArgOperand(5)->getType();
4890 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
4891 }
4892
4893 switch (F->getIntrinsicID()) {
4894 default:
4895 break;
4896 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4897 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4898 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4899 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4900 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4901 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
4902 // Drop src0 and src1 modifiers.
4903 const Value *Op0 = CI->getArgOperand(0);
4904 const Value *Op2 = CI->getArgOperand(2);
4905 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
4906 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
4907 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
4908 if (!ModA->isZero() || !ModB->isZero())
4909 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
4910
4911 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
4912 for (int I = 4, E = CI->arg_size(); I < E; ++I)
4913 Args.push_back(CI->getArgOperand(I));
4914
4915 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
4916 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
4917 Overloads.push_back(Args[3]->getType());
4918 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
4919 F->getParent(), F->getIntrinsicID(), Overloads);
4920
4921 SmallVector<OperandBundleDef, 1> Bundles;
4922 CI->getOperandBundlesAsDefs(Bundles);
4923
4924 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
4925 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4926 NewCall->setCallingConv(CI->getCallingConv());
4927 NewCall->setAttributes(CI->getAttributes());
4928 NewCall->setDebugLoc(CI->getDebugLoc());
4929 NewCall->copyMetadata(*CI);
4930 NewCall->takeName(CI);
4931 return NewCall;
4932 }
4933 }
4934
4935 AtomicRMWInst::BinOp RMWOp =
4936 StringSwitch<AtomicRMWInst::BinOp>(Name)
4937 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
4938 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
4939 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
4940 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
4941 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
4942 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
4943 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
4944 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
4945 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
4946 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
4947 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
4948 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
4949 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
4950
4951 unsigned NumOperands = CI->getNumOperands();
4952 if (NumOperands < 3) // Malformed bitcode.
4953 return nullptr;
4954
4955 Value *Ptr = CI->getArgOperand(0);
4956 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
4957 if (!PtrTy) // Malformed.
4958 return nullptr;
4959
4960 Value *Val = CI->getArgOperand(1);
4961 if (Val->getType() != CI->getType()) // Malformed.
4962 return nullptr;
4963
4964 ConstantInt *OrderArg = nullptr;
4965 bool IsVolatile = false;
4966
4967 // These should have 5 arguments (plus the callee). A separate version of the
4968 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
4969 if (NumOperands > 3)
4970 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
4971
4972 // Ignore scope argument at 3
4973
4974 if (NumOperands > 5) {
4975 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
4976 IsVolatile = !VolatileArg || !VolatileArg->isZero();
4977 }
4978
4979 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
4980 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
4981 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
4982 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
4983 Order = AtomicOrdering::SequentiallyConsistent;
4984
4985 LLVMContext &Ctx = F->getContext();
4986
4987 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
4988 Type *RetTy = CI->getType();
4989 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
4990 if (VT->getElementType()->isIntegerTy(16)) {
4991 VectorType *AsBF16 =
4992 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
4993 Val = Builder.CreateBitCast(Val, AsBF16);
4994 }
4995 }
4996
4997 // The scope argument never really worked correctly. Use agent as the most
4998 // conservative option which should still always produce the instruction.
4999 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5000 AtomicRMWInst *RMW =
5001 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5002
5003 unsigned AddrSpace = PtrTy->getAddressSpace();
5004 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5005 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5006 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5007 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5008 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5009 }
5010
5011 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5012 MDBuilder MDB(F->getContext());
5013 MDNode *RangeNotPrivate =
5014 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5015 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5016 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5017 }
5018
5019 if (IsVolatile)
5020 RMW->setVolatile(true);
5021
5022 return Builder.CreateBitCast(RMW, RetTy);
5023}
5024
5025/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5026/// plain MDNode, as it's the verifier's job to check these are the correct
5027/// types later.
5028static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5029 if (Op < CI->arg_size()) {
5030 if (MetadataAsValue *MAV =
5031 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5032 Metadata *MD = MAV->getMetadata();
5033 return dyn_cast_if_present<MDNode>(MD);
5034 }
5035 }
5036 return nullptr;
5037}
5038
5039/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5040static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5041 if (Op < CI->arg_size())
5042 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5043 return MAV->getMetadata();
5044 return nullptr;
5045}
5046
5047/// Convert debug intrinsic calls to non-instruction debug records.
5048/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5049/// \p CI - The debug intrinsic call.
5050static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5051 DbgRecord *DR = nullptr;
5052 if (Name == "label") {
5053 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5054 } else if (Name == "assign") {
5055 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5056 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5057 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5058 unwrapMAVMetadataOp(CI, 4),
5059 /*The address is a Value ref, it will be stored as a Metadata */
5060 unwrapMAVOp(CI, 5));
5061 } else if (Name == "declare") {
5062 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5063 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5064 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5065 } else if (Name == "addr") {
5066 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5067 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5068 // Don't try to add something to the expression if it's not an expression.
5069 // Instead, allow the verifier to fail later.
5070 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5071 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5072 }
5073 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5074 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5075 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5076 } else if (Name == "value") {
5077 // An old version of dbg.value had an extra offset argument.
5078 unsigned VarOp = 1;
5079 unsigned ExprOp = 2;
5080 if (CI->arg_size() == 4) {
5081 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5082 // Nonzero offset dbg.values get dropped without a replacement.
5083 if (!Offset || !Offset->isNullValue())
5084 return;
5085 VarOp = 2;
5086 ExprOp = 3;
5087 }
5088 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5089 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5090 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5091 nullptr);
5092 }
5093 DR->setDebugLoc(CI->getDebugLoc());
5094 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5095 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5096}
5097
5098static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5099 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5100 if (!Offset)
5101 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5102 int64_t OffsetVal = Offset->getSExtValue();
5103 return Builder.CreateIntrinsic(OffsetVal >= 0
5104 ? Intrinsic::vector_splice_left
5105 : Intrinsic::vector_splice_right,
5106 CI->getType(),
5107 {CI->getArgOperand(0), CI->getArgOperand(1),
5108 Builder.getInt32(std::abs(OffsetVal))});
5109}
5110
5111static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5112 Function *F, IRBuilder<> &Builder) {
5113 if (Name.starts_with("to.fp16")) {
5114 Value *Cast =
5115 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5116 return Builder.CreateBitCast(Cast, CI->getType());
5117 }
5118
5119 if (Name.starts_with("from.fp16")) {
5120 Value *Cast =
5121 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5122 return Builder.CreateFPExt(Cast, CI->getType());
5123 }
5124
5125 return nullptr;
5126}
5127
5128static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5129 IRBuilder<> &Builder) {
5130 Intrinsic::ID IID = NewFn->getIntrinsicID();
5131
5132 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5133 if (Defaults.empty())
5134 return false;
5135
5136 unsigned OldArgCount = CI->arg_size();
5137 unsigned NewArgCount = NewFn->arg_size();
5138
5139 // If the caller already supplied all arguments (or more), nothing to do.
5140 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5141 if (OldArgCount >= NewArgCount)
5142 return false;
5143
5144 // Start with the existing arguments from the old call.
5145 SmallVector<Value *, 8> NewArgs(CI->args());
5146
5147 // Defaults are a contiguous trailing block, so checking the first missing
5148 // argument is enough.
5149 if (OldArgCount < FirstDefault)
5150 return false;
5151
5152 // Fill in each missing trailing argument from the table.
5153 FunctionType *NewFT = NewFn->getFunctionType();
5154 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5155 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5156 "missing argument outside the default range");
5157 Type *ParamTy = NewFT->getParamType(Idx);
5158
5159 // Only integer types are supported (i1, i8, i16, i32, i64).
5160 if (!ParamTy->isIntegerTy())
5161 return false;
5162 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5163 }
5164
5165 // Preserve operand bundles by creating the call with them.
5166 SmallVector<OperandBundleDef, 1> OpBundles;
5167 CI->getOperandBundlesAsDefs(OpBundles);
5168 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5169
5170 NewCall->takeName(CI);
5171 NewCall->setCallingConv(CI->getCallingConv());
5172 NewCall->copyMetadata(*CI);
5173 if (auto *OldCI = dyn_cast<CallInst>(CI))
5174 NewCall->setTailCallKind(OldCI->getTailCallKind());
5175
5176 CI->replaceAllUsesWith(NewCall);
5177 CI->eraseFromParent();
5178 return true;
5179}
5180
5181/// Upgrade a call to an old intrinsic. All argument and return casting must be
5182/// provided to seamlessly integrate with existing context.
5183void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5184 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5185 // checks the callee's function type matches. It's likely we need to handle
5186 // type changes here.
5187 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5188 if (!F)
5189 return;
5190
5191 LLVMContext &C = CI->getContext();
5192 IRBuilder<> Builder(C);
5193 if (isa<FPMathOperator>(CI))
5194 Builder.setFastMathFlags(CI->getFastMathFlags());
5195 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5196
5197 if (!NewFn) {
5198 // Get the Function's name.
5199 StringRef Name = F->getName();
5200 if (!Name.consume_front("llvm."))
5201 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5202
5203 bool IsX86 = Name.consume_front("x86.");
5204 bool IsNVVM = Name.consume_front("nvvm.");
5205 bool IsAArch64 = Name.consume_front("aarch64.");
5206 bool IsARM = Name.consume_front("arm.");
5207 bool IsAMDGCN = Name.consume_front("amdgcn.");
5208 bool IsDbg = Name.consume_front("dbg.");
5209 bool IsOldSplice =
5210 (Name.consume_front("experimental.vector.splice") ||
5211 Name.consume_front("vector.splice")) &&
5212 !(Name.starts_with(".left") || Name.starts_with(".right"));
5213 Value *Rep = nullptr;
5214
5215 if (!IsX86 && Name == "stackprotectorcheck") {
5216 Rep = nullptr;
5217 } else if (IsNVVM) {
5218 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5219 } else if (IsX86) {
5220 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5221 } else if (IsAArch64) {
5222 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5223 } else if (IsARM) {
5224 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5225 } else if (IsAMDGCN) {
5226 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5227 } else if (IsDbg) {
5228 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5229 } else if (IsOldSplice) {
5230 Rep = upgradeVectorSplice(CI, Builder);
5231 } else if (Name.consume_front("convert.")) {
5232 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5233 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5234 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5235 Rep = nullptr;
5236 } else {
5237 llvm_unreachable("Unknown function for CallBase upgrade.");
5238 }
5239
5240 if (Rep)
5241 CI->replaceAllUsesWith(Rep);
5242 CI->eraseFromParent();
5243 return;
5244 }
5245
5246 const auto &DefaultCase = [&]() -> void {
5247 if (F == NewFn)
5248 return;
5249
5250 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5251 // Handle generic mangling change.
5252 assert(
5253 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5254 "Unknown function for CallBase upgrade and isn't just a name change");
5255 CI->setCalledFunction(NewFn);
5256 return;
5257 }
5258
5259 // This must be an upgrade from a named to a literal struct.
5260 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5261 assert(OldST != NewFn->getReturnType() &&
5262 "Return type must have changed");
5263 assert(OldST->getNumElements() ==
5264 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5265 "Must have same number of elements");
5266
5267 SmallVector<Value *> Args(CI->args());
5268 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5269 NewCI->setAttributes(CI->getAttributes());
5270 Value *Res = PoisonValue::get(OldST);
5271 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5272 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5273 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5274 }
5275 CI->replaceAllUsesWith(Res);
5276 CI->eraseFromParent();
5277 return;
5278 }
5279
5280 // We're probably about to produce something invalid. Let the verifier catch
5281 // it instead of dying here.
5282 CI->setCalledOperand(
5283 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5284 return;
5285 };
5286 CallInst *NewCall = nullptr;
5287 switch (NewFn->getIntrinsicID()) {
5288 default: {
5289 // Last resort: try the data-driven default-arg upgrade.
5290 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5291 // in its .td definition, without needing a dedicated case.
5292 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5293 return;
5294 DefaultCase();
5295 return;
5296 }
5297 case Intrinsic::arm_neon_vst1:
5298 case Intrinsic::arm_neon_vst2:
5299 case Intrinsic::arm_neon_vst3:
5300 case Intrinsic::arm_neon_vst4:
5301 case Intrinsic::arm_neon_vst2lane:
5302 case Intrinsic::arm_neon_vst3lane:
5303 case Intrinsic::arm_neon_vst4lane: {
5304 SmallVector<Value *, 4> Args(CI->args());
5305 NewCall = Builder.CreateCall(NewFn, Args);
5306 break;
5307 }
5308 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5309 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5310 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5311 LLVMContext &Ctx = F->getParent()->getContext();
5312 SmallVector<Value *, 4> Args(CI->args());
5313 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5314 cast<ConstantInt>(Args[3])->getZExtValue());
5315 NewCall = Builder.CreateCall(NewFn, Args);
5316 break;
5317 }
5318 case Intrinsic::aarch64_sve_ld3_sret:
5319 case Intrinsic::aarch64_sve_ld4_sret:
5320 case Intrinsic::aarch64_sve_ld2_sret: {
5321 // Is this a trivial remangle of the name to support ptr address spaces?
5322 if (isa<StructType>(F->getReturnType())) {
5323 DefaultCase();
5324 return;
5325 }
5326
5327 StringRef Name = F->getName();
5328 Name = Name.substr(5);
5329 unsigned N = StringSwitch<unsigned>(Name)
5330 .StartsWith("aarch64.sve.ld2", 2)
5331 .StartsWith("aarch64.sve.ld3", 3)
5332 .StartsWith("aarch64.sve.ld4", 4)
5333 .Default(0);
5334 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5335 unsigned MinElts = RetTy->getMinNumElements() / N;
5336 SmallVector<Value *, 2> Args(CI->args());
5337 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5338 Value *Ret = llvm::PoisonValue::get(RetTy);
5339 for (unsigned I = 0; I < N; I++) {
5340 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5341 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5342 }
5343 NewCall = dyn_cast<CallInst>(Ret);
5344 break;
5345 }
5346
5347 case Intrinsic::coro_end_async:
5348 case Intrinsic::coro_end: {
5349 SmallVector<Value *, 3> Args(CI->args());
5350 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5351 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5352 NewCall = Builder.CreateCall(NewFn, Args);
5353
5354 if (!CI->getType()->isVoidTy()) {
5355 if (!CI->use_empty()) {
5356 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5357 CI->getModule(), Intrinsic::coro_is_in_ramp);
5358 Value *InRamp = Builder.CreateCall(IsInRamp);
5359 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5360 }
5361 CI->eraseFromParent();
5362 return;
5363 }
5364
5365 break;
5366 }
5367
5368 case Intrinsic::vector_extract: {
5369 StringRef Name = F->getName();
5370 Name = Name.substr(5); // Strip llvm
5371 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5372 DefaultCase();
5373 return;
5374 }
5375 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5376 unsigned MinElts = RetTy->getMinNumElements();
5377 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5378 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5379 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5380 break;
5381 }
5382
5383 case Intrinsic::vector_insert: {
5384 StringRef Name = F->getName();
5385 Name = Name.substr(5);
5386 if (!Name.starts_with("aarch64.sve.tuple")) {
5387 DefaultCase();
5388 return;
5389 }
5390 if (Name.starts_with("aarch64.sve.tuple.set")) {
5391 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5392 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5393 Value *NewIdx =
5394 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5395 NewCall = Builder.CreateCall(
5396 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5397 break;
5398 }
5399 if (Name.starts_with("aarch64.sve.tuple.create")) {
5400 unsigned N = StringSwitch<unsigned>(Name)
5401 .StartsWith("aarch64.sve.tuple.create2", 2)
5402 .StartsWith("aarch64.sve.tuple.create3", 3)
5403 .StartsWith("aarch64.sve.tuple.create4", 4)
5404 .Default(0);
5405 assert(N > 1 && "Create is expected to be between 2-4");
5406 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5407 Value *Ret = llvm::PoisonValue::get(RetTy);
5408 unsigned MinElts = RetTy->getMinNumElements() / N;
5409 for (unsigned I = 0; I < N; I++) {
5410 Value *V = CI->getArgOperand(I);
5411 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5412 }
5413 NewCall = dyn_cast<CallInst>(Ret);
5414 }
5415 break;
5416 }
5417
5418 case Intrinsic::arm_neon_bfdot:
5419 case Intrinsic::arm_neon_bfmmla:
5420 case Intrinsic::arm_neon_bfmlalb:
5421 case Intrinsic::arm_neon_bfmlalt:
5422 case Intrinsic::aarch64_neon_bfdot:
5423 case Intrinsic::aarch64_neon_bfmmla:
5424 case Intrinsic::aarch64_neon_bfmlalb:
5425 case Intrinsic::aarch64_neon_bfmlalt: {
5426 SmallVector<Value *, 3> Args;
5427 assert(CI->arg_size() == 3 &&
5428 "Mismatch between function args and call args");
5429 size_t OperandWidth =
5430 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5431 assert((OperandWidth == 64 || OperandWidth == 128) &&
5432 "Unexpected operand width");
5433 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5434 auto Iter = CI->args().begin();
5435 Args.push_back(*Iter++);
5436 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5437 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5438 NewCall = Builder.CreateCall(NewFn, Args);
5439 break;
5440 }
5441
5442 case Intrinsic::bitreverse:
5443 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5444 break;
5445
5446 case Intrinsic::ctlz:
5447 case Intrinsic::cttz: {
5448 if (CI->arg_size() != 1) {
5449 DefaultCase();
5450 return;
5451 }
5452
5453 NewCall =
5454 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5455 break;
5456 }
5457
5458 case Intrinsic::objectsize: {
5459 Value *NullIsUnknownSize =
5460 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5461 Value *Dynamic =
5462 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5463 NewCall = Builder.CreateCall(
5464 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5465 break;
5466 }
5467
5468 case Intrinsic::ctpop:
5469 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5470 break;
5471 case Intrinsic::dbg_value: {
5472 StringRef Name = F->getName();
5473 Name = Name.substr(5); // Strip llvm.
5474 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5475 if (Name.starts_with("dbg.addr")) {
5476 DIExpression *Expr = cast<DIExpression>(
5477 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5478 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5479 NewCall =
5480 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5481 MetadataAsValue::get(C, Expr)});
5482 break;
5483 }
5484
5485 // Upgrade from the old version that had an extra offset argument.
5486 assert(CI->arg_size() == 4);
5487 // Drop nonzero offsets instead of attempting to upgrade them.
5488 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5489 if (Offset->isNullValue()) {
5490 NewCall = Builder.CreateCall(
5491 NewFn,
5492 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5493 break;
5494 }
5495 CI->eraseFromParent();
5496 return;
5497 }
5498
5499 case Intrinsic::ptr_annotation:
5500 // Upgrade from versions that lacked the annotation attribute argument.
5501 if (CI->arg_size() != 4) {
5502 DefaultCase();
5503 return;
5504 }
5505
5506 // Create a new call with an added null annotation attribute argument.
5507 NewCall = Builder.CreateCall(
5508 NewFn,
5509 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5510 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5511 NewCall->takeName(CI);
5512 CI->replaceAllUsesWith(NewCall);
5513 CI->eraseFromParent();
5514 return;
5515
5516 case Intrinsic::var_annotation:
5517 // Upgrade from versions that lacked the annotation attribute argument.
5518 if (CI->arg_size() != 4) {
5519 DefaultCase();
5520 return;
5521 }
5522 // Create a new call with an added null annotation attribute argument.
5523 NewCall = Builder.CreateCall(
5524 NewFn,
5525 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5526 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5527 NewCall->takeName(CI);
5528 CI->replaceAllUsesWith(NewCall);
5529 CI->eraseFromParent();
5530 return;
5531
5532 case Intrinsic::riscv_aes32dsi:
5533 case Intrinsic::riscv_aes32dsmi:
5534 case Intrinsic::riscv_aes32esi:
5535 case Intrinsic::riscv_aes32esmi:
5536 case Intrinsic::riscv_sm4ks:
5537 case Intrinsic::riscv_sm4ed: {
5538 // The last argument to these intrinsics used to be i8 and changed to i32.
5539 // The type overload for sm4ks and sm4ed was removed.
5540 Value *Arg2 = CI->getArgOperand(2);
5541 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
5542 return;
5543
5544 Value *Arg0 = CI->getArgOperand(0);
5545 Value *Arg1 = CI->getArgOperand(1);
5546 if (CI->getType()->isIntegerTy(64)) {
5547 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
5548 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
5549 }
5550
5551 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
5552 cast<ConstantInt>(Arg2)->getZExtValue());
5553
5554 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
5555 Value *Res = NewCall;
5556 if (Res->getType() != CI->getType())
5557 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5558 NewCall->takeName(CI);
5559 CI->replaceAllUsesWith(Res);
5560 CI->eraseFromParent();
5561 return;
5562 }
5563 case Intrinsic::nvvm_mapa_shared_cluster: {
5564 // Create a new call with the correct address space.
5565 NewCall =
5566 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
5567 Value *Res = NewCall;
5568 Res = Builder.CreateAddrSpaceCast(
5569 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
5570 NewCall->takeName(CI);
5571 CI->replaceAllUsesWith(Res);
5572 CI->eraseFromParent();
5573 return;
5574 }
5575 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5576 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5577 // Create a new call with the correct address space.
5578 SmallVector<Value *, 4> Args(CI->args());
5579 Args[0] = Builder.CreateAddrSpaceCast(
5580 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5581
5582 NewCall = Builder.CreateCall(NewFn, Args);
5583 NewCall->takeName(CI);
5584 CI->replaceAllUsesWith(NewCall);
5585 CI->eraseFromParent();
5586 return;
5587 }
5588 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5589 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5590 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5591 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5592 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5593 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5594 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5595 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5596 SmallVector<Value *, 16> Args(CI->args());
5597
5598 // Create AddrSpaceCast to shared_cluster if needed.
5599 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5600 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
5601 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5602 Args[0] = Builder.CreateAddrSpaceCast(
5603 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5604
5605 // Attach the flag argument for cta_group, with a
5606 // default value of 0. This handles case (2) in
5607 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5608 size_t NumArgs = CI->arg_size();
5609 Value *FlagArg = CI->getArgOperand(NumArgs - 3);
5610 if (!FlagArg->getType()->isIntegerTy(1))
5611 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
5612
5613 NewCall = Builder.CreateCall(NewFn, Args);
5614 NewCall->takeName(CI);
5615 CI->replaceAllUsesWith(NewCall);
5616 CI->eraseFromParent();
5617 return;
5618 }
5619 case Intrinsic::riscv_sha256sig0:
5620 case Intrinsic::riscv_sha256sig1:
5621 case Intrinsic::riscv_sha256sum0:
5622 case Intrinsic::riscv_sha256sum1:
5623 case Intrinsic::riscv_sm3p0:
5624 case Intrinsic::riscv_sm3p1: {
5625 // The last argument to these intrinsics used to be i8 and changed to i32.
5626 // The type overload for sm4ks and sm4ed was removed.
5627 if (!CI->getType()->isIntegerTy(64))
5628 return;
5629
5630 Value *Arg =
5631 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
5632
5633 NewCall = Builder.CreateCall(NewFn, Arg);
5634 Value *Res =
5635 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5636 NewCall->takeName(CI);
5637 CI->replaceAllUsesWith(Res);
5638 CI->eraseFromParent();
5639 return;
5640 }
5641
5642 case Intrinsic::x86_xop_vfrcz_ss:
5643 case Intrinsic::x86_xop_vfrcz_sd:
5644 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
5645 break;
5646
5647 case Intrinsic::x86_xop_vpermil2pd:
5648 case Intrinsic::x86_xop_vpermil2ps:
5649 case Intrinsic::x86_xop_vpermil2pd_256:
5650 case Intrinsic::x86_xop_vpermil2ps_256: {
5651 SmallVector<Value *, 4> Args(CI->args());
5652 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
5653 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
5654 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
5655 NewCall = Builder.CreateCall(NewFn, Args);
5656 break;
5657 }
5658
5659 case Intrinsic::x86_sse41_ptestc:
5660 case Intrinsic::x86_sse41_ptestz:
5661 case Intrinsic::x86_sse41_ptestnzc: {
5662 // The arguments for these intrinsics used to be v4f32, and changed
5663 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
5664 // So, the only thing required is a bitcast for both arguments.
5665 // First, check the arguments have the old type.
5666 Value *Arg0 = CI->getArgOperand(0);
5667 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
5668 return;
5669
5670 // Old intrinsic, add bitcasts
5671 Value *Arg1 = CI->getArgOperand(1);
5672
5673 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
5674
5675 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
5676 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
5677
5678 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
5679 break;
5680 }
5681
5682 case Intrinsic::x86_rdtscp: {
5683 // This used to take 1 arguments. If we have no arguments, it is already
5684 // upgraded.
5685 if (CI->getNumOperands() == 0)
5686 return;
5687
5688 NewCall = Builder.CreateCall(NewFn);
5689 // Extract the second result and store it.
5690 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5691 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
5692 // Replace the original call result with the first result of the new call.
5693 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
5694
5695 NewCall->takeName(CI);
5696 CI->replaceAllUsesWith(TSC);
5697 CI->eraseFromParent();
5698 return;
5699 }
5700
5701 case Intrinsic::x86_sse41_insertps:
5702 case Intrinsic::x86_sse41_dppd:
5703 case Intrinsic::x86_sse41_dpps:
5704 case Intrinsic::x86_sse41_mpsadbw:
5705 case Intrinsic::x86_avx_dp_ps_256:
5706 case Intrinsic::x86_avx2_mpsadbw: {
5707 // Need to truncate the last argument from i32 to i8 -- this argument models
5708 // an inherently 8-bit immediate operand to these x86 instructions.
5709 SmallVector<Value *, 4> Args(CI->args());
5710
5711 // Replace the last argument with a trunc.
5712 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
5713 NewCall = Builder.CreateCall(NewFn, Args);
5714 break;
5715 }
5716
5717 case Intrinsic::x86_avx512_mask_cmp_pd_128:
5718 case Intrinsic::x86_avx512_mask_cmp_pd_256:
5719 case Intrinsic::x86_avx512_mask_cmp_pd_512:
5720 case Intrinsic::x86_avx512_mask_cmp_ps_128:
5721 case Intrinsic::x86_avx512_mask_cmp_ps_256:
5722 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
5723 SmallVector<Value *, 4> Args(CI->args());
5724 unsigned NumElts =
5725 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
5726 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
5727
5728 NewCall = Builder.CreateCall(NewFn, Args);
5729 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
5730
5731 NewCall->takeName(CI);
5732 CI->replaceAllUsesWith(Res);
5733 CI->eraseFromParent();
5734 return;
5735 }
5736
5737 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
5738 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
5739 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
5740 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
5741 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
5742 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
5743 SmallVector<Value *, 4> Args(CI->args());
5744 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
5745 if (NewFn->getIntrinsicID() ==
5746 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
5747 Args[1] = Builder.CreateBitCast(
5748 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5749
5750 NewCall = Builder.CreateCall(NewFn, Args);
5751 Value *Res = Builder.CreateBitCast(
5752 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
5753
5754 NewCall->takeName(CI);
5755 CI->replaceAllUsesWith(Res);
5756 CI->eraseFromParent();
5757 return;
5758 }
5759 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
5760 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
5761 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
5762 SmallVector<Value *, 4> Args(CI->args());
5763 unsigned NumElts =
5764 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
5765 Args[1] = Builder.CreateBitCast(
5766 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5767 Args[2] = Builder.CreateBitCast(
5768 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
5769
5770 NewCall = Builder.CreateCall(NewFn, Args);
5771 break;
5772 }
5773
5774 case Intrinsic::thread_pointer: {
5775 NewCall = Builder.CreateCall(NewFn, {});
5776 break;
5777 }
5778
5779 case Intrinsic::memcpy:
5780 case Intrinsic::memmove:
5781 case Intrinsic::memset: {
5782 // We have to make sure that the call signature is what we're expecting.
5783 // We only want to change the old signatures by removing the alignment arg:
5784 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
5785 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
5786 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
5787 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
5788 // Note: i8*'s in the above can be any pointer type
5789 if (CI->arg_size() != 5) {
5790 DefaultCase();
5791 return;
5792 }
5793 // Remove alignment argument (3), and add alignment attributes to the
5794 // dest/src pointers.
5795 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
5796 CI->getArgOperand(2), CI->getArgOperand(4)};
5797 NewCall = Builder.CreateCall(NewFn, Args);
5798 AttributeList OldAttrs = CI->getAttributes();
5799 AttributeList NewAttrs = AttributeList::get(
5800 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
5801 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
5802 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
5803 NewCall->setAttributes(NewAttrs);
5804 auto *MemCI = cast<MemIntrinsic>(NewCall);
5805 // All mem intrinsics support dest alignment.
5806 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
5807 MemCI->setDestAlignment(Align->getMaybeAlignValue());
5808 // Memcpy/Memmove also support source alignment.
5809 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
5810 MTI->setSourceAlignment(Align->getMaybeAlignValue());
5811 break;
5812 }
5813
5814 case Intrinsic::masked_load:
5815 case Intrinsic::masked_gather:
5816 case Intrinsic::masked_store:
5817 case Intrinsic::masked_scatter: {
5818 if (CI->arg_size() != 4) {
5819 DefaultCase();
5820 return;
5821 }
5822
5823 auto GetMaybeAlign = [](Value *Op) {
5824 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
5825 uint64_t Val = CI->getZExtValue();
5826 if (Val == 0)
5827 return MaybeAlign();
5828 if (isPowerOf2_64(Val))
5829 return MaybeAlign(Val);
5830 }
5831 reportFatalUsageError("Invalid alignment argument");
5832 };
5833 auto GetAlign = [&](Value *Op) {
5834 MaybeAlign Align = GetMaybeAlign(Op);
5835 if (Align)
5836 return *Align;
5837 reportFatalUsageError("Invalid zero alignment argument");
5838 };
5839
5840 const DataLayout &DL = CI->getDataLayout();
5841 switch (NewFn->getIntrinsicID()) {
5842 case Intrinsic::masked_load:
5843 NewCall = Builder.CreateMaskedLoad(
5844 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
5845 CI->getArgOperand(2), CI->getArgOperand(3));
5846 break;
5847 case Intrinsic::masked_gather:
5848 NewCall = Builder.CreateMaskedGather(
5849 CI->getType(), CI->getArgOperand(0),
5850 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
5851 CI->getType()->getScalarType()),
5852 CI->getArgOperand(2), CI->getArgOperand(3));
5853 break;
5854 case Intrinsic::masked_store:
5855 NewCall = Builder.CreateMaskedStore(
5856 CI->getArgOperand(0), CI->getArgOperand(1),
5857 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
5858 break;
5859 case Intrinsic::masked_scatter:
5860 NewCall = Builder.CreateMaskedScatter(
5861 CI->getArgOperand(0), CI->getArgOperand(1),
5862 DL.getValueOrABITypeAlignment(
5863 GetMaybeAlign(CI->getArgOperand(2)),
5864 CI->getArgOperand(0)->getType()->getScalarType()),
5865 CI->getArgOperand(3));
5866 break;
5867 default:
5868 llvm_unreachable("Unexpected intrinsic ID");
5869 }
5870 // Previous metadata is still valid.
5871 NewCall->copyMetadata(*CI);
5872 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5873 break;
5874 }
5875
5876 case Intrinsic::lifetime_start:
5877 case Intrinsic::lifetime_end: {
5878 if (CI->arg_size() != 2) {
5879 DefaultCase();
5880 return;
5881 }
5882
5883 Value *Ptr = CI->getArgOperand(1);
5884 // Try to strip pointer casts, such that the lifetime works on an alloca.
5885 Ptr = Ptr->stripPointerCasts();
5886 if (isa<AllocaInst>(Ptr)) {
5887 // Don't use NewFn, as we might have looked through an addrspacecast.
5888 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
5889 NewCall = Builder.CreateLifetimeStart(Ptr);
5890 else
5891 NewCall = Builder.CreateLifetimeEnd(Ptr);
5892 break;
5893 }
5894
5895 // Otherwise remove the lifetime marker.
5896 CI->eraseFromParent();
5897 return;
5898 }
5899
5900 case Intrinsic::x86_avx512_vpdpbusd_128:
5901 case Intrinsic::x86_avx512_vpdpbusd_256:
5902 case Intrinsic::x86_avx512_vpdpbusd_512:
5903 case Intrinsic::x86_avx512_vpdpbusds_128:
5904 case Intrinsic::x86_avx512_vpdpbusds_256:
5905 case Intrinsic::x86_avx512_vpdpbusds_512:
5906 case Intrinsic::x86_avx2_vpdpbssd_128:
5907 case Intrinsic::x86_avx2_vpdpbssd_256:
5908 case Intrinsic::x86_avx10_vpdpbssd_512:
5909 case Intrinsic::x86_avx2_vpdpbssds_128:
5910 case Intrinsic::x86_avx2_vpdpbssds_256:
5911 case Intrinsic::x86_avx10_vpdpbssds_512:
5912 case Intrinsic::x86_avx2_vpdpbsud_128:
5913 case Intrinsic::x86_avx2_vpdpbsud_256:
5914 case Intrinsic::x86_avx10_vpdpbsud_512:
5915 case Intrinsic::x86_avx2_vpdpbsuds_128:
5916 case Intrinsic::x86_avx2_vpdpbsuds_256:
5917 case Intrinsic::x86_avx10_vpdpbsuds_512:
5918 case Intrinsic::x86_avx2_vpdpbuud_128:
5919 case Intrinsic::x86_avx2_vpdpbuud_256:
5920 case Intrinsic::x86_avx10_vpdpbuud_512:
5921 case Intrinsic::x86_avx2_vpdpbuuds_128:
5922 case Intrinsic::x86_avx2_vpdpbuuds_256:
5923 case Intrinsic::x86_avx10_vpdpbuuds_512: {
5924 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
5925 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5926 CI->getArgOperand(2)};
5927 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
5928 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5929 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5930
5931 NewCall = Builder.CreateCall(NewFn, Args);
5932 break;
5933 }
5934 case Intrinsic::x86_avx512_vpdpwssd_128:
5935 case Intrinsic::x86_avx512_vpdpwssd_256:
5936 case Intrinsic::x86_avx512_vpdpwssd_512:
5937 case Intrinsic::x86_avx512_vpdpwssds_128:
5938 case Intrinsic::x86_avx512_vpdpwssds_256:
5939 case Intrinsic::x86_avx512_vpdpwssds_512:
5940 case Intrinsic::x86_avx2_vpdpwsud_128:
5941 case Intrinsic::x86_avx2_vpdpwsud_256:
5942 case Intrinsic::x86_avx10_vpdpwsud_512:
5943 case Intrinsic::x86_avx2_vpdpwsuds_128:
5944 case Intrinsic::x86_avx2_vpdpwsuds_256:
5945 case Intrinsic::x86_avx10_vpdpwsuds_512:
5946 case Intrinsic::x86_avx2_vpdpwusd_128:
5947 case Intrinsic::x86_avx2_vpdpwusd_256:
5948 case Intrinsic::x86_avx10_vpdpwusd_512:
5949 case Intrinsic::x86_avx2_vpdpwusds_128:
5950 case Intrinsic::x86_avx2_vpdpwusds_256:
5951 case Intrinsic::x86_avx10_vpdpwusds_512:
5952 case Intrinsic::x86_avx2_vpdpwuud_128:
5953 case Intrinsic::x86_avx2_vpdpwuud_256:
5954 case Intrinsic::x86_avx10_vpdpwuud_512:
5955 case Intrinsic::x86_avx2_vpdpwuuds_128:
5956 case Intrinsic::x86_avx2_vpdpwuuds_256:
5957 case Intrinsic::x86_avx10_vpdpwuuds_512:
5958 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
5959 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5960 CI->getArgOperand(2)};
5961 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
5962 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5963 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5964
5965 NewCall = Builder.CreateCall(NewFn, Args);
5966 break;
5967 }
5968 assert(NewCall && "Should have either set this variable or returned through "
5969 "the default case");
5970 NewCall->takeName(CI);
5971 CI->replaceAllUsesWith(NewCall);
5972 CI->eraseFromParent();
5973}
5974
5975void llvm::UpgradeCallsToIntrinsic(Function *F) {
5976 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
5977
5978 // Check if this function should be upgraded and get the replacement function
5979 // if there is one.
5980 Function *NewFn;
5981 if (UpgradeIntrinsicFunction(F, NewFn)) {
5982 // Replace all users of the old function with the new function or new
5983 // instructions. This is not a range loop because the call is deleted.
5984 for (User *U : make_early_inc_range(F->users()))
5985 if (CallBase *CB = dyn_cast<CallBase>(U))
5986 UpgradeIntrinsicCall(CB, NewFn);
5987
5988 // Remove old function, no longer used, from the module.
5989 if (F != NewFn)
5990 F->eraseFromParent();
5991 }
5992}
5993
5994MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
5995 const unsigned NumOperands = MD.getNumOperands();
5996 if (NumOperands == 0)
5997 return &MD; // Invalid, punt to a verifier error.
5998
5999 // Check if the tag uses struct-path aware TBAA format.
6000 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6001 return &MD;
6002
6003 auto &Context = MD.getContext();
6004 if (NumOperands == 3) {
6005 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6006 MDNode *ScalarType = MDNode::get(Context, Elts);
6007 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6008 Metadata *Elts2[] = {ScalarType, ScalarType,
6009 ConstantAsMetadata::get(
6010 Constant::getNullValue(Type::getInt64Ty(Context))),
6011 MD.getOperand(2)};
6012 return MDNode::get(Context, Elts2);
6013 }
6014 // Create a MDNode <MD, MD, offset 0>
6015 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6016 Type::getInt64Ty(Context)))};
6017 return MDNode::get(Context, Elts);
6018}
6019
6020Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6021 Instruction *&Temp) {
6022 if (Opc != Instruction::BitCast)
6023 return nullptr;
6024
6025 Temp = nullptr;
6026 Type *SrcTy = V->getType();
6027 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6028 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6029 LLVMContext &Context = V->getContext();
6030
6031 // We have no information about target data layout, so we assume that
6032 // the maximum pointer size is 64bit.
6033 Type *MidTy = Type::getInt64Ty(Context);
6034 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6035
6036 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6037 }
6038
6039 return nullptr;
6040}
6041
6042Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6043 if (Opc != Instruction::BitCast)
6044 return nullptr;
6045
6046 Type *SrcTy = C->getType();
6047 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6048 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6049 LLVMContext &Context = C->getContext();
6050
6051 // We have no information about target data layout, so we assume that
6052 // the maximum pointer size is 64bit.
6053 Type *MidTy = Type::getInt64Ty(Context);
6054
6055 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6056 DestTy);
6057 }
6058
6059 return nullptr;
6060}
6061
6062static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6063 if (Flag.getNumOperands() < 3)
6064 return std::nullopt;
6065 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6066 return Name->getString();
6067 return std::nullopt;
6068}
6069
6070/// Check the debug info version number, if it is out-dated, drop the debug
6071/// info. Return true if module is modified.
6072bool llvm::UpgradeDebugInfo(Module &M) {
6073 if (DisableAutoUpgradeDebugInfo)
6074 return false;
6075
6076 llvm::TimeTraceScope timeScope("Upgrade debug info");
6077 // We need to get metadata before the module is verified (i.e., getModuleFlag
6078 // makes assumptions that we haven't verified yet). Carefully extract the flag
6079 // from the metadata.
6080 unsigned Version = 0;
6081 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6082 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6083 if (auto Name = getModuleFlagNameSafely(*Flag))
6084 return *Name == "Debug Info Version";
6085 return false;
6086 });
6087 if (OpIt != ModFlags->op_end()) {
6088 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6089 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6090 Version = CI->getZExtValue();
6091 }
6092 }
6093
6094 if (Version == DEBUG_METADATA_VERSION) {
6095 bool BrokenDebugInfo = false;
6096 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6097 report_fatal_error("Broken module found, compilation aborted!");
6098 if (!BrokenDebugInfo)
6099 // Everything is ok.
6100 return false;
6101 else {
6102 // Diagnose malformed debug info.
6103 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6104 M.getContext().diagnose(Diag);
6105 }
6106 }
6107 bool Modified = StripDebugInfo(M);
6108 if (Modified && Version != DEBUG_METADATA_VERSION) {
6109 // Diagnose a version mismatch.
6110 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6111 M.getContext().diagnose(DiagVersion);
6112 }
6113 return Modified;
6114}
6115
6116static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6117 GlobalValue *GV, const Metadata *V) {
6118 Function *F = cast<Function>(GV);
6119
6120 constexpr StringLiteral DefaultValue = "1";
6121 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6122 unsigned Length = 0;
6123
6124 if (F->hasFnAttribute(Attr)) {
6125 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6126 // parse these elements placing them into Vect3
6127 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6128 for (; Length < 3 && !S.empty(); Length++) {
6129 auto [Part, Rest] = S.split(',');
6130 Vect3[Length] = Part.trim();
6131 S = Rest;
6132 }
6133 }
6134
6135 const unsigned Dim = DimC - 'x';
6136 assert(Dim < 3 && "Unexpected dim char");
6137
6138 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6139
6140 // local variable required for StringRef in Vect3 to point to.
6141 const std::string VStr = llvm::utostr(VInt);
6142 Vect3[Dim] = VStr;
6143 Length = std::max(Length, Dim + 1);
6144
6145 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6146 F->addFnAttr(Attr, NewAttr);
6147}
6148
6149static inline bool isXYZ(StringRef S) {
6150 return S == "x" || S == "y" || S == "z";
6151}
6152
6153bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6154 const Metadata *V) {
6155 if (K == "kernel") {
6156 if (!mdconst::extract<ConstantInt>(V)->isZero())
6157 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6158 return true;
6159 }
6160 if (K == "align") {
6161 // V is a bitfeild specifying two 16-bit values. The alignment value is
6162 // specfied in low 16-bits, The index is specified in the high bits. For the
6163 // index, 0 indicates the return value while higher values correspond to
6164 // each parameter (idx = param + 1).
6165 const uint64_t AlignIdxValuePair =
6166 mdconst::extract<ConstantInt>(V)->getZExtValue();
6167 const unsigned Idx = (AlignIdxValuePair >> 16);
6168 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6169 cast<Function>(GV)->addAttributeAtIndex(
6170 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6171 return true;
6172 }
6173 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6174 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6175 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6176 return true;
6177 }
6178 if (K == "minctasm") {
6179 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6180 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6181 return true;
6182 }
6183 if (K == "maxnreg") {
6184 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6185 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6186 return true;
6187 }
6188 if (K.consume_front("maxntid") && isXYZ(K)) {
6189 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6190 return true;
6191 }
6192 if (K.consume_front("reqntid") && isXYZ(K)) {
6193 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6194 return true;
6195 }
6196 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6197 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6198 return true;
6199 }
6200 if (K == "grid_constant") {
6201 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6202 for (const auto &Op : cast<MDNode>(V)->operands()) {
6203 // For some reason, the index is 1-based in the metadata. Good thing we're
6204 // able to auto-upgrade it!
6205 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6206 cast<Function>(GV)->addParamAttr(Index, Attr);
6207 }
6208 return true;
6209 }
6210
6211 return false;
6212}
6213
6214void llvm::UpgradeNVVMAnnotations(Module &M) {
6215 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6216 if (!NamedMD)
6217 return;
6218
6219 SmallVector<MDNode *, 8> NewNodes;
6220 SmallPtrSet<const MDNode *, 8> SeenNodes;
6221 for (MDNode *MD : NamedMD->operands()) {
6222 if (!SeenNodes.insert(MD).second)
6223 continue;
6224
6225 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6226 if (!GV)
6227 continue;
6228
6229 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6230
6231 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6232 // Each nvvm.annotations metadata entry will be of the following form:
6233 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6234 // start index = 1, to skip the global variable key
6235 // increment = 2, to skip the value for each property-value pairs
6236 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6237 MDString *K = cast<MDString>(MD->getOperand(j));
6238 const MDOperand &V = MD->getOperand(j + 1);
6239 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6240 if (!Upgraded)
6241 NewOperands.append({K, V});
6242 }
6243
6244 if (NewOperands.size() > 1)
6245 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6246 }
6247
6248 NamedMD->clearOperands();
6249 for (MDNode *N : NewNodes)
6250 NamedMD->addOperand(N);
6251}
6252
6253/// This checks for objc retain release marker which should be upgraded. It
6254/// returns true if module is modified.
6255static bool upgradeRetainReleaseMarker(Module &M) {
6256 bool Changed = false;
6257 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6258 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6259 if (ModRetainReleaseMarker) {
6260 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6261 if (Op) {
6262 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6263 if (ID) {
6264 SmallVector<StringRef, 4> ValueComp;
6265 ID->getString().split(ValueComp, "#");
6266 if (ValueComp.size() == 2) {
6267 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6268 ID = MDString::get(M.getContext(), NewValue);
6269 }
6270 M.addModuleFlag(Module::Error, MarkerKey, ID);
6271 M.eraseNamedMetadata(ModRetainReleaseMarker);
6272 Changed = true;
6273 }
6274 }
6275 }
6276 return Changed;
6277}
6278
6279void llvm::UpgradeARCRuntime(Module &M) {
6280 // This lambda converts normal function calls to ARC runtime functions to
6281 // intrinsic calls.
6282 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6283 llvm::Intrinsic::ID IntrinsicFunc) {
6284 Function *Fn = M.getFunction(OldFunc);
6285
6286 if (!Fn)
6287 return;
6288
6289 Function *NewFn =
6290 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6291
6292 for (User *U : make_early_inc_range(Fn->users())) {
6293 CallInst *CI = dyn_cast<CallInst>(U);
6294 if (!CI || CI->getCalledFunction() != Fn)
6295 continue;
6296
6297 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6298 FunctionType *NewFuncTy = NewFn->getFunctionType();
6299 SmallVector<Value *, 2> Args;
6300
6301 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6302 // value to the return type of the old function.
6303 if (NewFuncTy->getReturnType() != CI->getType() &&
6304 !CastInst::castIsValid(Instruction::BitCast, CI,
6305 NewFuncTy->getReturnType()))
6306 continue;
6307
6308 bool InvalidCast = false;
6309
6310 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6311 Value *Arg = CI->getArgOperand(I);
6312
6313 // Bitcast argument to the parameter type of the new function if it's
6314 // not a variadic argument.
6315 if (I < NewFuncTy->getNumParams()) {
6316 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6317 // to the parameter type of the new function.
6318 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6319 NewFuncTy->getParamType(I))) {
6320 InvalidCast = true;
6321 break;
6322 }
6323 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6324 }
6325 Args.push_back(Arg);
6326 }
6327
6328 if (InvalidCast)
6329 continue;
6330
6331 // Create a call instruction that calls the new function.
6332 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6333 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6334 NewCall->takeName(CI);
6335
6336 // Bitcast the return value back to the type of the old call.
6337 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6338
6339 if (!CI->use_empty())
6340 CI->replaceAllUsesWith(NewRetVal);
6341 CI->eraseFromParent();
6342 }
6343
6344 if (Fn->use_empty())
6345 Fn->eraseFromParent();
6346 };
6347
6348 // Unconditionally convert a call to "clang.arc.use" to a call to
6349 // "llvm.objc.clang.arc.use".
6350 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6351
6352 // Upgrade the retain release marker. If there is no need to upgrade
6353 // the marker, that means either the module is already new enough to contain
6354 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6355 if (!upgradeRetainReleaseMarker(M))
6356 return;
6357
6358 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6359 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6360 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6361 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6362 {"objc_autoreleaseReturnValue",
6363 llvm::Intrinsic::objc_autoreleaseReturnValue},
6364 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6365 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6366 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6367 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6368 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6369 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6370 {"objc_release", llvm::Intrinsic::objc_release},
6371 {"objc_retain", llvm::Intrinsic::objc_retain},
6372 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6373 {"objc_retainAutoreleaseReturnValue",
6374 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6375 {"objc_retainAutoreleasedReturnValue",
6376 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6377 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6378 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6379 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6380 {"objc_unsafeClaimAutoreleasedReturnValue",
6381 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6382 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6383 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6384 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6385 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6386 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6387 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6388 {"objc_arc_annotation_topdown_bbstart",
6389 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6390 {"objc_arc_annotation_topdown_bbend",
6391 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6392 {"objc_arc_annotation_bottomup_bbstart",
6393 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6394 {"objc_arc_annotation_bottomup_bbend",
6395 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6396
6397 for (auto &I : RuntimeFuncs)
6398 UpgradeToIntrinsic(I.first, I.second);
6399}
6400
6401// Upgrade the way signing of pointers to init/fini functions is described.
6402//
6403// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6404// constants, if signing was requested. After the upgrade, these arrays contain
6405// plain function pointers and the desired signing schema is described via a
6406// pair of module flags.
6407//
6408// Note that the upgrade is only performed if all elements of *both* arrays
6409// agree on a common signing schema.
6410static bool upgradePtrauthInitFiniArrays(Module &M) {
6411 // As we cannot always decide whether the particular module should have
6412 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6413 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6414 // returns as soon as it spots any non-signed init/fini pointer: either we
6415 // should request non-signed pointers (safe to omit both flags) or there is
6416 // no common schema (and thus we do not modify anything).
6417 //
6418 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6419 // or whether we should request address diversity in addition to the basic
6420 // constant diversity. There is no value representing "decided not to sign"
6421 // for the reasons explained above.
6422 std::optional<bool> UseAddressDisc;
6423
6424 // Do not attempt upgrading if the new module flags already exist.
6425 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6426 for (const MDNode *Flag : ModFlags->operands()) {
6427 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
6428 if (Name && (*Name == "ptrauth-init-fini" ||
6429 *Name == "ptrauth-init-fini-address-discrimination"))
6430 return false;
6431 }
6432 }
6433
6434 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
6435 constexpr unsigned ExpectedConstDisc = 0xD9D4;
6436 constexpr unsigned ExpectedAddressMarker = 1;
6437
6438 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
6439 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
6440 return nullptr; // Nothing to upgrade or unknown pattern found.
6441
6442 bool HasAddressDisc;
6443 if (!CPA->hasAddressDiscriminator())
6444 HasAddressDisc = false;
6445 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
6446 HasAddressDisc = true;
6447 else
6448 return nullptr; // Unknown pattern.
6449
6450 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6451 return nullptr; // Disagreement with the decided mode.
6452
6453 UseAddressDisc = HasAddressDisc;
6454 return CPA->getPointer();
6455 };
6456
6457 // Do not apply any changes until we know the upgrade is non-ambiguous.
6458 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
6459 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
6460
6461 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6462 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
6463 if (!GV || !GV->hasInitializer())
6464 continue; // Skip, but it is okay to upgrade the other variable.
6465
6466 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
6467 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
6468 return false;
6469
6470 std::vector<Constant *> NewStructors;
6471 NewStructors.reserve(OldStructorsArray->getNumOperands());
6472
6473 for (Use &U : OldStructorsArray->operands()) {
6474 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
6475 if (!Structor || Structor->getNumOperands() != 3)
6476 return false;
6477
6478 Constant *Prio = Structor->getOperand(0);
6479 Constant *Func = Structor->getOperand(1);
6480 Constant *Arg = Structor->getOperand(2);
6481
6482 Func = UpgradeSinglePointer(Func);
6483 if (!Func)
6484 return false;
6485
6486 NewStructors.push_back(
6487 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
6488 }
6489
6490 Constant *NewInit =
6491 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
6492 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
6493 }
6494
6495 if (GlobalArraysToUpgrade.empty())
6496 return false;
6497 assert(UseAddressDisc.has_value());
6498
6499 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6500 GV->setInitializer(NewInit);
6501
6502 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
6503 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
6504 *UseAddressDisc);
6505
6506 return true;
6507}
6508
6509bool llvm::UpgradeModuleFlags(Module &M) {
6510 bool Changed = false;
6511 Changed |= upgradePtrauthInitFiniArrays(M);
6512
6513 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6514 if (!ModFlags)
6515 return Changed;
6516
6517 bool HasObjCFlag = false, HasClassProperties = false;
6518 bool HasSwiftVersionFlag = false;
6519 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6520 uint32_t SwiftABIVersion;
6521 auto Int8Ty = Type::getInt8Ty(M.getContext());
6522 auto Int32Ty = Type::getInt32Ty(M.getContext());
6523
6524 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6525 MDNode *Op = ModFlags->getOperand(I);
6526 if (Op->getNumOperands() != 3)
6527 continue;
6528 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6529 if (!ID)
6530 continue;
6531 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6532 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
6533 Type::getInt32Ty(M.getContext()), B)),
6534 MDString::get(M.getContext(), ID->getString()),
6535 Op->getOperand(2)};
6536 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6537 Changed = true;
6538 };
6539
6540 if (ID->getString() == "Objective-C Image Info Version")
6541 HasObjCFlag = true;
6542 if (ID->getString() == "Objective-C Class Properties")
6543 HasClassProperties = true;
6544 // Upgrade PIC from Error/Max to Min.
6545 if (ID->getString() == "PIC Level") {
6546 if (auto *Behavior =
6547 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6548 uint64_t V = Behavior->getLimitedValue();
6549 if (V == Module::Error || V == Module::Max)
6550 SetBehavior(Module::Min);
6551 }
6552 }
6553 // Upgrade "PIE Level" from Error to Max.
6554 if (ID->getString() == "PIE Level")
6555 if (auto *Behavior =
6556 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
6557 if (Behavior->getLimitedValue() == Module::Error)
6558 SetBehavior(Module::Max);
6559
6560 // Upgrade branch protection and return address signing module flags. The
6561 // module flag behavior for these fields were Error and now they are Min.
6562 if (ID->getString() == "branch-target-enforcement" ||
6563 ID->getString().starts_with("sign-return-address")) {
6564 if (auto *Behavior =
6565 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6566 if (Behavior->getLimitedValue() == Module::Error) {
6567 Type *Int32Ty = Type::getInt32Ty(M.getContext());
6568 Metadata *Ops[3] = {
6569 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
6570 Op->getOperand(1), Op->getOperand(2)};
6571 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6572 Changed = true;
6573 }
6574 }
6575 }
6576
6577 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
6578 // section name so that llvm-lto will not complain about mismatching
6579 // module flags that is functionally the same.
6580 if (ID->getString() == "Objective-C Image Info Section") {
6581 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
6582 SmallVector<StringRef, 4> ValueComp;
6583 Value->getString().split(ValueComp, " ");
6584 if (ValueComp.size() != 1) {
6585 std::string NewValue;
6586 for (auto &S : ValueComp)
6587 NewValue += S.str();
6588 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
6589 MDString::get(M.getContext(), NewValue)};
6590 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6591 Changed = true;
6592 }
6593 }
6594 }
6595
6596 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
6597 // If the higher bits are set, it adds new module flag for swift info.
6598 if (ID->getString() == "Objective-C Garbage Collection") {
6599 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
6600 if (Md) {
6601 assert(Md->getValue() && "Expected non-empty metadata");
6602 auto Type = Md->getValue()->getType();
6603 if (Type == Int8Ty)
6604 continue;
6605 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
6606 if ((Val & 0xff) != Val) {
6607 HasSwiftVersionFlag = true;
6608 SwiftABIVersion = (Val & 0xff00) >> 8;
6609 SwiftMajorVersion = (Val & 0xff000000) >> 24;
6610 SwiftMinorVersion = (Val & 0xff0000) >> 16;
6611 }
6612 Metadata *Ops[3] = {
6613 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
6614 Op->getOperand(1),
6615 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
6616 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6617 Changed = true;
6618 }
6619 }
6620
6621 if (ID->getString() == "amdgpu_code_object_version") {
6622 Metadata *Ops[3] = {
6623 Op->getOperand(0),
6624 MDString::get(M.getContext(), "amdhsa_code_object_version"),
6625 Op->getOperand(2)};
6626 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6627 Changed = true;
6628 }
6629 }
6630
6631 // "Objective-C Class Properties" is recently added for Objective-C. We
6632 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
6633 // flag of value 0, so we can correclty downgrade this flag when trying to
6634 // link an ObjC bitcode without this module flag with an ObjC bitcode with
6635 // this module flag.
6636 if (HasObjCFlag && !HasClassProperties) {
6637 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
6638 (uint32_t)0);
6639 Changed = true;
6640 }
6641
6642 if (HasSwiftVersionFlag) {
6643 M.addModuleFlag(Module::Error, "Swift ABI Version",
6644 SwiftABIVersion);
6645 M.addModuleFlag(Module::Error, "Swift Major Version",
6646 ConstantInt::get(Int8Ty, SwiftMajorVersion));
6647 M.addModuleFlag(Module::Error, "Swift Minor Version",
6648 ConstantInt::get(Int8Ty, SwiftMinorVersion));
6649 Changed = true;
6650 }
6651
6652 return Changed;
6653}
6654
6655bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
6656 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
6657 // If this metadata has operands, we expect all of them to be either from
6658 // before or from after the format change handled here, so we can bail out
6659 // fast if the first (if any) operands is of the new format.
6660 auto MatchesVersion = [](const MDNode *Op) {
6661 return Op->getNumOperands() >= 3 &&
6662 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
6663 cast<ConstantAsMetadata>(Op->getOperand(2))
6664 ->getType()
6665 ->isIntegerTy(64);
6666 };
6667
6668 if (!CFIConsts || !CFIConsts->getNumOperands() ||
6669 MatchesVersion(CFIConsts->getOperand(0)))
6670 return false;
6671
6672 bool Changed = false;
6673 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
6674 MDNode *Op = CFIConsts->getOperand(I);
6675 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
6676 assert(Op->getNumOperands() >= 2 &&
6677 "Expected at least 2 operands - name and linkage type");
6678 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
6679 StringRef Name = NameMD->getString();
6680 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
6681 GlobalValue::dropLLVMManglingEscape(Name));
6682
6683 SmallVector<Metadata *, 4> Elts;
6684 Elts.push_back(Op->getOperand(0));
6685 Elts.push_back(Op->getOperand(1));
6686 Elts.push_back(ConstantAsMetadata::get(
6687 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
6688
6689 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
6690 Elts.push_back(Op->getOperand(J));
6691
6692 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
6693 Changed = true;
6694 }
6695
6696 return Changed;
6697}
6698
6699void llvm::UpgradeSectionAttributes(Module &M) {
6700 auto TrimSpaces = [](StringRef Section) -> std::string {
6701 SmallVector<StringRef, 5> Components;
6702 Section.split(Components, ',');
6703
6704 SmallString<32> Buffer;
6705 raw_svector_ostream OS(Buffer);
6706
6707 for (auto Component : Components)
6708 OS << ',' << Component.trim();
6709
6710 return std::string(OS.str().substr(1));
6711 };
6712
6713 for (auto &GV : M.globals()) {
6714 if (!GV.hasSection())
6715 continue;
6716
6717 StringRef Section = GV.getSection();
6718
6719 if (!Section.starts_with("__DATA, __objc_catlist"))
6720 continue;
6721
6722 // __DATA, __objc_catlist, regular, no_dead_strip
6723 // __DATA,__objc_catlist,regular,no_dead_strip
6724 GV.setSection(TrimSpaces(Section));
6725 }
6726}
6727
6728namespace {
6729// Prior to LLVM 10.0, the strictfp attribute could be used on individual
6730// callsites within a function that did not also have the strictfp attribute.
6731// Since 10.0, if strict FP semantics are needed within a function, the
6732// function must have the strictfp attribute and all calls within the function
6733// must also have the strictfp attribute. This latter restriction is
6734// necessary to prevent unwanted libcall simplification when a function is
6735// being cloned (such as for inlining).
6736//
6737// The "dangling" strictfp attribute usage was only used to prevent constant
6738// folding and other libcall simplification. The nobuiltin attribute on the
6739// callsite has the same effect.
6740struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
6741 StrictFPUpgradeVisitor() = default;
6742
6743 void visitCallBase(CallBase &Call) {
6744 if (!Call.isStrictFP())
6745 return;
6746 if (isa<ConstrainedFPIntrinsic>(&Call))
6747 return;
6748 // If we get here, the caller doesn't have the strictfp attribute
6749 // but this callsite does. Replace the strictfp attribute with nobuiltin.
6750 Call.removeFnAttr(Attribute::StrictFP);
6751 Call.addFnAttr(Attribute::NoBuiltin);
6752 }
6753};
6754
6755/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
6756struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
6757 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
6758 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
6759
6760 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
6761 if (!RMW.isFloatingPointOperation())
6762 return;
6763
6764 MDNode *Empty = MDNode::get(RMW.getContext(), {});
6765 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
6766 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
6767 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
6768 }
6769};
6770} // namespace
6771
6772void llvm::UpgradeFunctionAttributes(Function &F) {
6773 // If a function definition doesn't have the strictfp attribute,
6774 // convert any callsite strictfp attributes to nobuiltin.
6775 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
6776 StrictFPUpgradeVisitor SFPV;
6777 SFPV.visit(F);
6778 }
6779
6780 // Remove all incompatibile attributes from function.
6781 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
6782 F.getReturnType(), F.getAttributes().getRetAttrs()));
6783 for (auto &Arg : F.args())
6784 Arg.removeAttrs(
6785 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
6786
6787 bool AddingAttrs = false, RemovingAttrs = false;
6788 AttrBuilder AttrsToAdd(F.getContext());
6789 AttributeMask AttrsToRemove;
6790
6791 // Older versions of LLVM treated an "implicit-section-name" attribute
6792 // similarly to directly setting the section on a Function.
6793 if (Attribute A = F.getFnAttribute("implicit-section-name");
6794 A.isValid() && A.isStringAttribute()) {
6795 F.setSection(A.getValueAsString());
6796 AttrsToRemove.addAttribute("implicit-section-name");
6797 RemovingAttrs = true;
6798 }
6799
6800 if (Attribute A = F.getFnAttribute("nooutline");
6801 A.isValid() && A.isStringAttribute()) {
6802 AttrsToRemove.addAttribute("nooutline");
6803 AttrsToAdd.addAttribute(Attribute::NoOutline);
6804 AddingAttrs = RemovingAttrs = true;
6805 }
6806
6807 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
6808 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
6809 AttrsToRemove.addAttribute("uniform-work-group-size");
6810 RemovingAttrs = true;
6811 if (A.getValueAsString() == "true") {
6812 AttrsToAdd.addAttribute("uniform-work-group-size");
6813 AddingAttrs = true;
6814 }
6815 }
6816
6817 if (!F.empty()) {
6818 // For some reason this is called twice, and the first time is before any
6819 // instructions are loaded into the body.
6820
6821 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
6822 A.isValid()) {
6823
6824 if (A.getValueAsBool()) {
6825 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
6826 Visitor.visit(F);
6827 }
6828
6829 // We will leave behind dead attribute uses on external declarations, but
6830 // clang never added these to declarations anyway.
6831 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
6832 RemovingAttrs = true;
6833 }
6834 }
6835
6836 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
6837 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
6838
6839 bool HandleDenormalMode = false;
6840
6841 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
6842 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
6843 if (ParsedMode.isValid()) {
6844 DenormalFPMath = ParsedMode;
6845 AttrsToRemove.addAttribute("denormal-fp-math");
6846 AddingAttrs = RemovingAttrs = true;
6847 HandleDenormalMode = true;
6848 }
6849 }
6850
6851 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
6852 Attr.isValid()) {
6853 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
6854 if (ParsedMode.isValid()) {
6855 DenormalFPMathF32 = ParsedMode;
6856 AttrsToRemove.addAttribute("denormal-fp-math-f32");
6857 AddingAttrs = RemovingAttrs = true;
6858 HandleDenormalMode = true;
6859 }
6860 }
6861
6862 if (HandleDenormalMode)
6863 AttrsToAdd.addDenormalFPEnvAttr(
6864 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
6865
6866 if (RemovingAttrs)
6867 F.removeFnAttrs(AttrsToRemove);
6868
6869 if (AddingAttrs)
6870 F.addFnAttrs(AttrsToAdd);
6871}
6872
6873// Check if the function attribute is not present and set it.
6874static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
6875 StringRef Value) {
6876 if (!F.hasFnAttribute(FnAttrName))
6877 F.addFnAttr(FnAttrName, Value);
6878}
6879
6880// Check if the function attribute is not present and set it if needed.
6881// If the attribute is "false" then removes it.
6882// If the attribute is "true" resets it to a valueless attribute.
6883static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
6884 if (!F.hasFnAttribute(FnAttrName)) {
6885 if (Set)
6886 F.addFnAttr(FnAttrName);
6887 } else {
6888 auto A = F.getFnAttribute(FnAttrName);
6889 if ("false" == A.getValueAsString())
6890 F.removeFnAttr(FnAttrName);
6891 else if ("true" == A.getValueAsString()) {
6892 F.removeFnAttr(FnAttrName);
6893 F.addFnAttr(FnAttrName);
6894 }
6895 }
6896}
6897
6898void llvm::copyModuleAttrToFunctions(Module &M) {
6899 Triple T(M.getTargetTriple());
6900 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
6901 return;
6902
6903 uint64_t BTEValue = 0;
6904 uint64_t BPPLRValue = 0;
6905 uint64_t GCSValue = 0;
6906 uint64_t SRAValue = 0;
6907 uint64_t SRAALLValue = 0;
6908 uint64_t SRABKeyValue = 0;
6909
6910 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6911 if (ModFlags) {
6912 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6913 MDNode *Op = ModFlags->getOperand(I);
6914 if (Op->getNumOperands() != 3)
6915 continue;
6916
6917 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6918 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
6919 if (!ID || !CI)
6920 continue;
6921
6922 StringRef IDStr = ID->getString();
6923 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
6924 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
6925 : IDStr == "guarded-control-stack" ? &GCSValue
6926 : IDStr == "sign-return-address" ? &SRAValue
6927 : IDStr == "sign-return-address-all" ? &SRAALLValue
6928 : IDStr == "sign-return-address-with-bkey"
6929 ? &SRABKeyValue
6930 : nullptr;
6931 if (!ValPtr)
6932 continue;
6933
6934 *ValPtr = CI->getZExtValue();
6935 if (*ValPtr == 2)
6936 return;
6937 }
6938 }
6939
6940 bool BTE = BTEValue == 1;
6941 bool BPPLR = BPPLRValue == 1;
6942 bool GCS = GCSValue == 1;
6943 bool SRA = SRAValue == 1;
6944
6945 StringRef SignTypeValue = "non-leaf";
6946 if (SRA && SRAALLValue == 1)
6947 SignTypeValue = "all";
6948
6949 StringRef SignKeyValue = "a_key";
6950 if (SRA && SRABKeyValue == 1)
6951 SignKeyValue = "b_key";
6952
6953 for (Function &F : M.getFunctionList()) {
6954 if (F.isDeclaration())
6955 continue;
6956
6957 if (SRA) {
6958 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
6959 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
6960 } else {
6961 if (auto A = F.getFnAttribute("sign-return-address");
6962 A.isValid() && "none" == A.getValueAsString()) {
6963 F.removeFnAttr("sign-return-address");
6964 F.removeFnAttr("sign-return-address-key");
6965 }
6966 }
6967 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
6968 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
6969 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
6970 }
6971
6972 if (BTE)
6973 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
6974 if (BPPLR)
6975 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
6976 if (GCS)
6977 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
6978 if (SRA) {
6979 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
6980 if (SRAALLValue == 1)
6981 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
6982 if (SRABKeyValue == 1)
6983 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
6984 }
6985}
6986
6987// Old two-operand form: !{!"llvm.loop.distribute.enable", i1 X}. The new
6988// single-operand form uses "llvm.loop.distribute.enable" for X = true and
6989// "llvm.loop.distribute.disable" for X = false.
6990static bool isOldDistributeEnable(const MDTuple *T) {
6991 if (T->getNumOperands() != 2)
6992 return false;
6993 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
6994 if (!Tag || Tag->getString() != "llvm.loop.distribute.enable")
6995 return false;
6996 return mdconst::hasa<ConstantInt>(T->getOperand(1));
6997}
6998
6999static bool isOldLoopArgument(Metadata *MD) {
7000 auto *T = dyn_cast_or_null<MDTuple>(MD);
7001 if (!T)
7002 return false;
7003 if (T->getNumOperands() < 1)
7004 return false;
7005 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7006 if (!S)
7007 return false;
7008 if (S->getString().starts_with("llvm.vectorizer."))
7009 return true;
7010 return isOldDistributeEnable(T);
7011}
7012
7013static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7014 StringRef OldPrefix = "llvm.vectorizer.";
7015 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7016
7017 if (OldTag == "llvm.vectorizer.unroll")
7018 return MDString::get(C, "llvm.loop.interleave.count");
7019
7020 return MDString::get(
7021 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7022 .str());
7023}
7024
7025static Metadata *upgradeLoopArgument(Metadata *MD) {
7026 auto *T = dyn_cast_or_null<MDTuple>(MD);
7027 if (!T)
7028 return MD;
7029 if (T->getNumOperands() < 1)
7030 return MD;
7031 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7032 if (!OldTag)
7033 return MD;
7034
7035 LLVMContext &C = T->getContext();
7036
7037 // Rewrite the old two-operand distribute form to the single-operand pair.
7038 if (isOldDistributeEnable(T)) {
7039 bool Enable = !mdconst::extract<ConstantInt>(T->getOperand(1))->isZero();
7040 return MDTuple::get(
7041 C, {MDString::get(C, Enable ? "llvm.loop.distribute.enable"
7042 : "llvm.loop.distribute.disable")});
7043 }
7044
7045 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7046 return MD;
7047
7048 // This has an old tag. Upgrade it.
7049 SmallVector<Metadata *, 8> Ops;
7050 Ops.reserve(T->getNumOperands());
7051 Ops.push_back(upgradeLoopTag(C, OldTag->getString()));
7052 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7053 Ops.push_back(T->getOperand(I));
7054
7055 return MDTuple::get(C, Ops);
7056}
7057
7058MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7059 auto *T = dyn_cast<MDTuple>(&N);
7060 if (!T)
7061 return &N;
7062
7063 if (none_of(T->operands(), isOldLoopArgument))
7064 return &N;
7065
7066 // Fix the old two-operand llvm.loop.distribute.enable nodes in place: the
7067 // Verifier rejects any MDNode carrying the distribute tag with more than one
7068 // operand, so a leftover reference (from the distinct loop-ID) would still
7069 // trigger a diagnostic. In-place mutation is safe on distinct MDNodes.
7070 if (T->isDistinct()) {
7071 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7072 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7073 if (OpT && isOldDistributeEnable(OpT))
7074 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7075 }
7076 if (none_of(T->operands(), isOldLoopArgument))
7077 return &N;
7078 }
7079
7080 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7081 // attachment; the original distinct loop-ID is kept as the first operand.
7082 SmallVector<Metadata *, 8> Ops;
7083 Ops.reserve(T->getNumOperands());
7084 for (Metadata *MD : T->operands())
7085 Ops.push_back(upgradeLoopArgument(MD));
7086
7087 return MDTuple::get(T->getContext(), Ops);
7088}
7089
7090std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7091 Triple T(TT);
7092 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7093 // the address space of globals to 1. This does not apply to SPIRV Logical.
7094 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7095 !DL.contains("-G") && !DL.starts_with("G")) {
7096 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7097 }
7098
7099 if (T.isLoongArch64() || T.isRISCV64()) {
7100 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7101 auto I = DL.find("-n64-");
7102 if (I != StringRef::npos)
7103 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7104 return DL.str();
7105 }
7106
7107 // AMDGPU data layout upgrades.
7108 std::string Res = DL.str();
7109 if (T.isAMDGPU()) {
7110 // Define address spaces for constants.
7111 if (!DL.contains("-G") && !DL.starts_with("G"))
7112 Res.append(Res.empty() ? "G1" : "-G1");
7113
7114 // AMDGCN data layout upgrades.
7115 if (T.isAMDGCN()) {
7116
7117 // Add missing non-integral declarations.
7118 // This goes before adding new address spaces to prevent incoherent string
7119 // values.
7120 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7121 Res.append("-ni:7:8:9");
7122 // Update ni:7 to ni:7:8:9.
7123 if (DL.ends_with("ni:7"))
7124 Res.append(":8:9");
7125 if (DL.ends_with("ni:7:8"))
7126 Res.append(":9");
7127
7128 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7129 // resources) An empty data layout has already been upgraded to G1 by now.
7130 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7131 Res.append("-p7:160:256:256:32");
7132 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7133 Res.append("-p8:128:128:128:48");
7134 constexpr StringRef OldP8("-p8:128:128-");
7135 if (DL.contains(OldP8))
7136 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7137 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7138 Res.append("-p9:192:256:256:32");
7139 }
7140
7141 // Upgrade the ELF mangling mode.
7142 if (!DL.contains("m:e"))
7143 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7144
7145 return Res;
7146 }
7147
7148 if (T.isSystemZ() && !DL.empty()) {
7149 // Make sure the stack alignment is present.
7150 if (!DL.contains("-S64"))
7151 return "E-S64" + DL.drop_front(1).str();
7152 return DL.str();
7153 }
7154
7155 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7156 // If the datalayout matches the expected format, add pointer size address
7157 // spaces to the datalayout.
7158 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7159 if (!DL.contains(AddrSpaces)) {
7160 SmallVector<StringRef, 4> Groups;
7161 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7162 if (R.match(Res, &Groups))
7163 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7164 }
7165 };
7166
7167 // AArch64 data layout upgrades.
7168 if (T.isAArch64()) {
7169 // Add "-Fn32"
7170 if (!DL.empty() && !DL.contains("-Fn32"))
7171 Res.append("-Fn32");
7172 AddPtr32Ptr64AddrSpaces();
7173 return Res;
7174 }
7175
7176 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7177 T.isWasm()) {
7178 // Mips64 with o32 ABI did not add "-i128:128".
7179 // Add "-i128:128"
7180 std::string I64 = "-i64:64";
7181 std::string I128 = "-i128:128";
7182 if (!StringRef(Res).contains(I128)) {
7183 size_t Pos = Res.find(I64);
7184 if (Pos != size_t(-1))
7185 Res.insert(Pos + I64.size(), I128);
7186 }
7187 }
7188
7189 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7190 size_t Pos = Res.find("-S128");
7191 if (Pos == StringRef::npos)
7192 Pos = Res.size();
7193 Res.insert(Pos, "-f64:32:64");
7194 }
7195
7196 if (!T.isX86())
7197 return Res;
7198
7199 AddPtr32Ptr64AddrSpaces();
7200
7201 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7202 // for i128 operations prior to this being reflected in the data layout, and
7203 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7204 // boundaries, so although this is a breaking change, the upgrade is expected
7205 // to fix more IR than it breaks.
7206 // Intel MCU is an exception and uses 4-byte-alignment.
7207 if (!T.isOSIAMCU()) {
7208 std::string I128 = "-i128:128";
7209 if (StringRef Ref = Res; !Ref.contains(I128)) {
7210 SmallVector<StringRef, 4> Groups;
7211 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7212 if (R.match(Res, &Groups))
7213 Res = (Groups[1] + I128 + Groups[3]).str();
7214 }
7215 }
7216
7217 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7218 // Raising the alignment is safe because Clang did not produce f80 values in
7219 // the MSVC environment before this upgrade was added.
7220 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7221 StringRef Ref = Res;
7222 auto I = Ref.find("-f80:32-");
7223 if (I != StringRef::npos)
7224 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7225 }
7226
7227 return Res;
7228}
7229
7230void llvm::UpgradeAttributes(AttrBuilder &B) {
7231 StringRef FramePointer;
7232 Attribute A = B.getAttribute("no-frame-pointer-elim");
7233 if (A.isValid()) {
7234 // The value can be "true" or "false".
7235 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7236 B.removeAttribute("no-frame-pointer-elim");
7237 }
7238 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7239 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7240 if (FramePointer != "all")
7241 FramePointer = "non-leaf";
7242 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7243 }
7244 if (!FramePointer.empty())
7245 B.addAttribute("frame-pointer", FramePointer);
7246
7247 A = B.getAttribute("null-pointer-is-valid");
7248 if (A.isValid()) {
7249 // The value can be "true" or "false".
7250 bool NullPointerIsValid = A.getValueAsString() == "true";
7251 B.removeAttribute("null-pointer-is-valid");
7252 if (NullPointerIsValid)
7253 B.addAttribute(Attribute::NullPointerIsValid);
7254 }
7255
7256 A = B.getAttribute("uniform-work-group-size");
7257 if (A.isValid()) {
7258 StringRef Val = A.getValueAsString();
7259 if (!Val.empty()) {
7260 bool IsTrue = Val == "true";
7261 B.removeAttribute("uniform-work-group-size");
7262 if (IsTrue)
7263 B.addAttribute("uniform-work-group-size");
7264 }
7265 }
7266}
7267
7268void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7269 // clang.arc.attachedcall bundles are now required to have an operand.
7270 // If they don't, it's okay to drop them entirely: when there is an operand,
7271 // the "attachedcall" is meaningful and required, but without an operand,
7272 // it's just a marker NOP. Dropping it merely prevents an optimization.
7273 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7274 return OBD.getTag() == "clang.arc.attachedcall" &&
7275 OBD.inputs().empty();
7276 });
7277}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:68
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeIntrinsicDeclWithDefaultArgs
static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1311
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2213
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7025
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6149
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1341
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2053
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1194
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6255
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2314
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2540
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1291
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2288
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:124
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:6874
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1219
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6153
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5028
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7013
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6116
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:111
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2168
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2425
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2139
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4687
isOldDistributeEnable
static bool isOldDistributeEnable(const MDTuple *T)
Definition AutoUpgrade.cpp:6990
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2526
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:549
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2465
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6062
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1285
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:178
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2087
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1147
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:6999
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4757
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:95
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5098
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4847
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2404
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5040
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:158
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:787
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5128
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2119
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2151
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2353
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:66
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:79
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:168
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2487
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2273
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:2797
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2519
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2549
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5050
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:6883
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2435
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:70
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2386
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5111
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:142
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:6410
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:2995
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3391
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:81
LHS
Value * LHS
Definition X86PartialReduction.cpp:80
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:760
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:920
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:772
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:776
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:797
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:836
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:790
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:780
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:786
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:840
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:828
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:788
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:808
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:804
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:832
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:967
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:105
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:239
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:507
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:355
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1581
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1701
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3109
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3392
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2470
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2402
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2456
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3472
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2291
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:867
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:168
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:211
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:246
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:166
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:444
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:878
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:216
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:863
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2893
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:551
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:86
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:112
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1751
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:715
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1514
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:94
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:310
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:190
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1565
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:632
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:614
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1482
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1511
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:110
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:117
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:138
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:120
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:152
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:149
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1753
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1535
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1527
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1523
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1542
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1849
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1533
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:889
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2081
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:386
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:532
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:977
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:702
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1236
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:331
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:839
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:308
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5262
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint64_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2356
Changed
Changed
Definition ObjCARCOpts.cpp:2366
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1418
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::lookupIntrinsicID
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
Definition Intrinsics.cpp:758
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1399
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1392
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:578
llvm::Length
@ Length
Definition DWP.cpp:578
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5183
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:6699
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:2788
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2001
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7058
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7230
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:5975
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6214
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:6509
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:6655
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:6898
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7268
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6042
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7090
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2020
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:709
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6020
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6072
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:6772
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:5994
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6279
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7684
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0