LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with("addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with("bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front("sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front("bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front("convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front("convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front("addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front("ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(1)->getType();
1095 Type *Ty = VectorType::get(ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front("tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with("get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 F->getParent(), Intrinsic::vector_extract, Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with("set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 F->getParent(), Intrinsic::vector_insert, Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 F->getParent(), Intrinsic::vector_insert, Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with("rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front("sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front("ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1172// declarations requiring an auto-upgrade. The same set applies to the
1173// cluster (g2s) and CTA (g2s_cta) variants.
1174#define NVVM_TMA_G2S_MODES(M) \
1175 M(tile_1d, "tile.1d") \
1176 M(tile_2d, "tile.2d") \
1177 M(tile_3d, "tile.3d") \
1178 M(tile_4d, "tile.4d") \
1179 M(tile_5d, "tile.5d") \
1180 M(tile_gather4_2d, "tile.gather4.2d") \
1181 M(im2col_3d, "im2col.3d") \
1182 M(im2col_4d, "im2col.4d") \
1183 M(im2col_5d, "im2col.5d") \
1184 M(im2col_w_3d, "im2col.w.3d") \
1185 M(im2col_w_4d, "im2col.w.4d") \
1186 M(im2col_w_5d, "im2col.w.5d") \
1187 M(im2col_w_128_3d, "im2col.w.128.3d") \
1188 M(im2col_w_128_4d, "im2col.w.128.4d") \
1189 M(im2col_w_128_5d, "im2col.w.128.5d")
1190
1191// Two legacy tails are:
1192//
1193// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1194// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1195//
1196// The current tail appends a trailing i32 %flag_valid_pattern, so both
1197// legacy tails are recognized by an i1 at parameter N-2.
1198static Intrinsic::ID
1199shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1200 SmallVectorImpl<Type *> &OvlTys) {
1201 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1202 return Intrinsic::not_intrinsic;
1203
1204#define G2S_ID(ID_SUFFIX, NAME) \
1205 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1206 // clang-format off
1207 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1208 NVVM_TMA_G2S_MODES(G2S_ID)
1209 .Default(Intrinsic::not_intrinsic);
1210#undef G2S_ID
1211 // clang-format on
1212 if (ID == Intrinsic::not_intrinsic)
1213 return ID;
1214
1215 size_t NumParams = F->getFunctionType()->getNumParams();
1216
1217 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1218 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1219 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1220 return Intrinsic::not_intrinsic;
1221
1222 // The multicast mask is the parameter immediately before the i64
1223 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1224 ArrayRef<Type *> Params = F->getFunctionType()->params();
1225 size_t MaskIdx =
1226 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1227 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1228 "expected the i64 cache-hint after the multicast mask");
1229 Type *MaskTy = Params[MaskIdx];
1230 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1231 OvlTys.push_back(MaskTy);
1232
1233 return ID;
1234}
1235
1236// The legacy tail is:
1237//
1238// arg1, arg2, .. i64 %ch, i1 %flag_ch
1239//
1240// The current tail appends a trailing i32 %flag_valid_pattern, so the
1241// legacy tail is recognized by an i1 at parameter N-1.
1242static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1243 StringRef Name) {
1244 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1245 return Intrinsic::not_intrinsic;
1246
1247#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1248 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1249 // clang-format off
1250 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1251 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1252 .Default(Intrinsic::not_intrinsic);
1253#undef G2S_CTA_ID
1254 // clang-format on
1255 if (ID == Intrinsic::not_intrinsic)
1256 return ID;
1257
1258 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1259 // with i32 %flag_valid_pattern, for which N-1 is i32.
1260 if (!F->getFunctionType()
1261 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1262 ->isIntegerTy(1))
1263 return Intrinsic::not_intrinsic;
1264
1265 return ID;
1266}
1267// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1268//
1269// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1270//
1271// The current intrinsic is overloaded on the multicast-mask type and takes a
1272// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1273// parameter N-1.
1274static Intrinsic::ID
1275shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1276 SmallVectorImpl<Type *> &OvlTys) {
1277 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1278 return Intrinsic::not_intrinsic;
1279
1280 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1281 // i32 %flag_valid_pattern, for which N-1 is i32.
1282 size_t NumParams = F->getFunctionType()->getNumParams();
1283 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1284 return Intrinsic::not_intrinsic;
1285
1286 // The multicast mask is parameter 4; legacy IR only uses i16.
1287 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1288 if (!MaskTy->isIntegerTy(16))
1289 return Intrinsic::not_intrinsic;
1290 OvlTys.push_back(MaskTy);
1291
1292 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1293}
1294
1295// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1296//
1297// ..., i64 %ch, i1 %flag_ch
1298//
1299// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1300// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1301// recognized by an i1 at parameter N-1, whereas the current tail ends
1302// with an i32.
1303static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1304 StringRef Name) {
1305 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1306 return Intrinsic::not_intrinsic;
1307
1308 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1309 // i32 %flag_valid_pattern, for which N-1 is i32.
1310 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1311 return Intrinsic::not_intrinsic;
1312
1313 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1314}
1315
1316// The legacy TMA reduction intrinsics encode the reduction operator in their
1317// name, while the current ones take it as an immediate argument. Map the
1318// operator part of a legacy name to the corresponding immediate value.
1319static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1320 return StringSwitch<std::optional<unsigned>>(Name)
1321 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1322 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1323 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1324 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1325 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1326 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1327 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1328 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1329 .Default(std::nullopt);
1330}
1331
1332static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1333 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1334 return Intrinsic::not_intrinsic;
1335
1336 auto [RedOpName, ShapeName] = Name.split('.');
1337 if (!getNVPTXTMAReductionOp(RedOpName))
1338 return Intrinsic::not_intrinsic;
1339
1340 return StringSwitch<Intrinsic::ID>(ShapeName)
1341 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1342 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1343 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1344 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1345 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1346 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1347 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1348 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1349 .Default(Intrinsic::not_intrinsic);
1350}
1351
1352static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1353 StringRef Name) {
1354 if (Name.consume_front("mapa.shared.cluster"))
1355 if (F->getReturnType()->getPointerAddressSpace() ==
1356 NVPTXAS::ADDRESS_SPACE_SHARED)
1357 return Intrinsic::nvvm_mapa_shared_cluster;
1358
1359 if (Name.consume_front("cp.async.bulk.")) {
1360 Intrinsic::ID ID =
1361 StringSwitch<Intrinsic::ID>(Name)
1362 .Case("shared.cta.to.cluster",
1363 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1364 .Default(Intrinsic::not_intrinsic);
1365
1366 if (ID != Intrinsic::not_intrinsic)
1367 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1368 NVPTXAS::ADDRESS_SPACE_SHARED)
1369 return ID;
1370 }
1371
1372 return Intrinsic::not_intrinsic;
1373}
1374
1375static Intrinsic::ID
1376shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1377 if (!Name.consume_front("tcgen05.commit."))
1378 return Intrinsic::not_intrinsic;
1379
1380 if (Name.consume_front("shared."))
1381 return StringSwitch<Intrinsic::ID>(Name)
1382 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1383 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1384 .Default(Intrinsic::not_intrinsic);
1385
1386 if (Name.consume_front("mc.shared.")) {
1387 // Only upgrade older i16 mc variants.
1388 if (!F->getArg(1)->getType()->isIntegerTy(16))
1389 return Intrinsic::not_intrinsic;
1390
1391 return StringSwitch<Intrinsic::ID>(Name)
1392 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1393 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1394 .Default(Intrinsic::not_intrinsic);
1395 }
1396
1397 return Intrinsic::not_intrinsic;
1398}
1399
1400static Intrinsic::ID
1401shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1402 if (F->arg_size() != 2)
1403 return Intrinsic::not_intrinsic;
1404
1405 if (Name.consume_front("tcgen05.alloc.shared.") ||
1406 Name.consume_front("tcgen05.alloc."))
1407 return StringSwitch<Intrinsic::ID>(Name)
1408 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1409 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1410 .Default(Intrinsic::not_intrinsic);
1411
1412 if (Name.consume_front("tcgen05.dealloc."))
1413 return StringSwitch<Intrinsic::ID>(Name)
1414 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1415 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1416 .Default(Intrinsic::not_intrinsic);
1417
1418 return Intrinsic::not_intrinsic;
1419}
1420
1421static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1422 if (Name.consume_front("fma.rn."))
1423 return StringSwitch<Intrinsic::ID>(Name)
1424 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1425 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1426 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1427 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1428 .Default(Intrinsic::not_intrinsic);
1429
1430 if (Name.consume_front("fmax."))
1431 return StringSwitch<Intrinsic::ID>(Name)
1432 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1433 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1434 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1435 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1436 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1437 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1438 .Case("ftz.nan.xorsign.abs.bf16",
1439 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1440 .Case("ftz.nan.xorsign.abs.bf16x2",
1441 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1442 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1443 .Case("ftz.xorsign.abs.bf16x2",
1444 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1445 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1446 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1447 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1448 .Case("nan.xorsign.abs.bf16x2",
1449 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1450 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1451 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1452 .Default(Intrinsic::not_intrinsic);
1453
1454 if (Name.consume_front("fmin."))
1455 return StringSwitch<Intrinsic::ID>(Name)
1456 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1457 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1458 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1459 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1460 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1461 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1462 .Case("ftz.nan.xorsign.abs.bf16",
1463 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1464 .Case("ftz.nan.xorsign.abs.bf16x2",
1465 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1466 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1467 .Case("ftz.xorsign.abs.bf16x2",
1468 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1469 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1470 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1471 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1472 .Case("nan.xorsign.abs.bf16x2",
1473 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1474 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1475 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1476 .Default(Intrinsic::not_intrinsic);
1477
1478 if (Name.consume_front("neg."))
1479 return StringSwitch<Intrinsic::ID>(Name)
1480 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1481 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1482 .Default(Intrinsic::not_intrinsic);
1483
1484 return Intrinsic::not_intrinsic;
1485}
1486
1487static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1488 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1489 FunctionType *OldFnTy = F->getFunctionType();
1490 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1491 return OldTy->getScalarType()->isIntegerTy() &&
1492 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1493 };
1494
1495 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1496 return false;
1497
1498 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1499 return false;
1500
1501 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1502 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1503 return false;
1504
1505 return true;
1506}
1507
1508static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1509getNVVMFAddUpgrade(StringRef Name) {
1510 auto [Modifiers, Type] = Name.rsplit('.');
1511 if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
1512 return std::nullopt;
1513
1514 std::optional<llvm::RoundingMode> RoundingMode =
1515 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1516 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1517 .Case("rz", llvm::RoundingMode::TowardZero)
1518 .Case("rm", llvm::RoundingMode::TowardNegative)
1519 .Case("rp", llvm::RoundingMode::TowardPositive)
1520 .Default(std::nullopt);
1521 if (!RoundingMode)
1522 return std::nullopt;
1523
1524 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
1525 .Case("", Intrinsic::nvvm_fadd)
1526 .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
1527 .Case(".sat", Intrinsic::nvvm_fadd_sat)
1528 .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
1529 .Default(Intrinsic::not_intrinsic);
1530 if (IID == Intrinsic::not_intrinsic)
1531 return std::nullopt;
1532
1533 return std::make_pair(IID, *RoundingMode);
1534}
1535
1536static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1537 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1538 return Intrinsic::not_intrinsic;
1539
1540 return Intrinsic::nvvm_mbarrier_init;
1541}
1542
1543static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1544 return Name.consume_front("local") || Name.consume_front("shared") ||
1545 Name.consume_front("global") || Name.consume_front("constant") ||
1546 Name.consume_front("param");
1547}
1548
1549static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1550 if (!Name.consume_front("vp."))
1551 return 0;
1552 return StringSwitch<unsigned>(Name)
1553 .StartsWith("select", Instruction::Select)
1554 .StartsWith("add", Instruction::Add)
1555 .StartsWith("sub", Instruction::Sub)
1556 .StartsWith("mul", Instruction::Mul)
1557 .StartsWith("ashr", Instruction::AShr)
1558 .StartsWith("lshr", Instruction::LShr)
1559 .StartsWith("shl", Instruction::Shl)
1560 .StartsWith("or", Instruction::Or)
1561 .StartsWith("and", Instruction::And)
1562 .StartsWith("xor", Instruction::Xor)
1563 .StartsWith("fadd", Instruction::FAdd)
1564 .StartsWith("fsub", Instruction::FSub)
1565 .StartsWith("fmuladd", 0)
1566 .StartsWith("fmul", Instruction::FMul)
1567 .StartsWith("fdiv", Instruction::FDiv)
1568 .StartsWith("frem", Instruction::FRem)
1569 .StartsWith("fneg", Instruction::FNeg)
1570 .StartsWith("trunc", Instruction::Trunc)
1571 .StartsWith("zext", Instruction::ZExt)
1572 .StartsWith("sext", Instruction::SExt)
1573 .StartsWith("fptrunc", Instruction::FPTrunc)
1574 .StartsWith("fpext", Instruction::FPExt)
1575 .StartsWith("fptoui", Instruction::FPToUI)
1576 .StartsWith("fptosi", Instruction::FPToSI)
1577 .StartsWith("uitofp", Instruction::UIToFP)
1578 .StartsWith("sitofp", Instruction::SIToFP)
1579 .StartsWith("ptrtoint", Instruction::PtrToInt)
1580 .StartsWith("inttoptr", Instruction::IntToPtr)
1581 .StartsWith("icmp", Instruction::ICmp)
1582 .StartsWith("fcmp", Instruction::FCmp)
1583 .Default(0);
1584}
1585
1586static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1587 if (!Name.consume_front("vp."))
1588 return 0;
1589 return StringSwitch<Intrinsic::ID>(Name)
1590 .StartsWith("abs", Intrinsic::abs)
1591 .StartsWith("smax", Intrinsic::smax)
1592 .StartsWith("smin", Intrinsic::smin)
1593 .StartsWith("umax", Intrinsic::umax)
1594 .StartsWith("umin", Intrinsic::umin)
1595 .StartsWith("copysign", Intrinsic::copysign)
1596 .StartsWith("minnum", Intrinsic::minnum)
1597 .StartsWith("maxnum", Intrinsic::maxnum)
1598 .StartsWith("minimum", Intrinsic::minimum)
1599 .StartsWith("maximum", Intrinsic::maximum)
1600 .StartsWith("fabs", Intrinsic::fabs)
1601 .StartsWith("sqrt", Intrinsic::sqrt)
1602 .StartsWith("fma", Intrinsic::fma)
1603 .StartsWith("fmuladd", Intrinsic::fmuladd)
1604 .StartsWith("ceil", Intrinsic::ceil)
1605 .StartsWith("floor", Intrinsic::floor)
1606 .StartsWith("rint", Intrinsic::rint)
1607 .StartsWith("nearbyint", Intrinsic::nearbyint)
1608 .StartsWith("roundeven", Intrinsic::roundeven)
1609 .StartsWith("roundtozero", Intrinsic::trunc)
1610 .StartsWith("round", Intrinsic::round)
1611 .StartsWith("lrint", Intrinsic::lrint)
1612 .StartsWith("llrint", Intrinsic::llrint)
1613 .StartsWith("bitreverse", Intrinsic::bitreverse)
1614 .StartsWith("bswap", Intrinsic::bswap)
1615 .StartsWith("ctpop", Intrinsic::ctpop)
1616 .StartsWith("ctlz", Intrinsic::ctlz)
1617 .StartsWith("cttz.elts", 0)
1618 .StartsWith("cttz", Intrinsic::cttz)
1619 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1620 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1621 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1622 .StartsWith("usub.sat", Intrinsic::usub_sat)
1623 .StartsWith("fshl", Intrinsic::fshl)
1624 .StartsWith("fshr", Intrinsic::fshr)
1625 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1626 .Default(0);
1627}
1628
1632
1633static bool convertIntrinsicValidType(StringRef Name,
1634 const FunctionType *FuncTy) {
1635 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1636 if (Name.starts_with("to.fp16")) {
1637 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1638 HalfTy) &&
1639 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1640 FuncTy->getReturnType());
1641 }
1642
1643 if (Name.starts_with("from.fp16")) {
1644 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1645 HalfTy) &&
1646 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1647 FuncTy->getReturnType());
1648 }
1649
1650 return false;
1651}
1652
1653static unsigned getFullArgCountForDefaultArgUpgrade(Function *F,
1654 Intrinsic::ID IID) {
1655 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1656 if (Defaults.empty())
1657 return 0;
1658
1659 if (Intrinsic::isOverloaded(IID))
1660 return 0;
1661
1662 unsigned FullArgCount = FirstDefault + Defaults.size();
1663
1664 // Only trailing default arguments can be missing.
1665 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1666 return 0;
1667
1668 return FullArgCount;
1669}
1670
1671static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1672 Intrinsic::ID IID = F->getIntrinsicID();
1673
1674 unsigned FullArgCount = getFullArgCountForDefaultArgUpgrade(F, IID);
1675 if (FullArgCount == 0)
1676 return false;
1677
1678 rename(F);
1679 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1680 assert(NewFn->arg_size() == FullArgCount &&
1681 "total number of default args does not match intrinsic signature");
1682 return true;
1683}
1684
1685static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1686 bool CanUpgradeDebugIntrinsicsToRecords) {
1687 assert(F && "Illegal to upgrade a non-existent Function.");
1688
1689 StringRef Name = F->getName();
1690
1691 // Quickly eliminate it, if it's not a candidate.
1692 if (!Name.consume_front("llvm.") || Name.empty())
1693 return false;
1694
1695 switch (Name[0]) {
1696 default: break;
1697 case 'a': {
1698 bool IsArm = Name.consume_front("arm.");
1699 if (IsArm || Name.consume_front("aarch64.")) {
1700 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1701 return true;
1702 break;
1703 }
1704
1705 if (Name.consume_front("amdgcn.")) {
1706 if (Name == "alignbit") {
1707 // Target specific intrinsic became redundant
1708 NewFn = Intrinsic::getOrInsertDeclaration(
1709 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1710 return true;
1711 }
1712
1713 if (Name.consume_front("atomic.")) {
1714 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1715 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1716 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1717 // and usub_sat so there's no new declaration.
1718 NewFn = nullptr;
1719 return true;
1720 }
1721 break; // No other 'amdgcn.atomic.*'
1722 }
1723
1724 if (Name.starts_with("addrspacecast.nonnull")) {
1725 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1726 // so there's no new declaration.
1727 NewFn = nullptr;
1728 return true;
1729 }
1730
1731 switch (F->getIntrinsicID()) {
1732 default:
1733 break;
1734 // Legacy wmma iu intrinsics without the optional clamp operand.
1735 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1736 if (F->arg_size() == 7) {
1737 NewFn = nullptr;
1738 return true;
1739 }
1740 break;
1741 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1742 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1743 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1744 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1745 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1746 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1747 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1748 if (F->arg_size() == 8) {
1749 NewFn = nullptr;
1750 return true;
1751 }
1752 break;
1753 }
1754
1755 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1756 Name.consume_front("flat.atomic.")) {
1757 if (Name.starts_with("fadd") ||
1758 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1759 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1760 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1761 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1762 // declaration.
1763 NewFn = nullptr;
1764 return true;
1765 }
1766 }
1767
1768 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1769 NewFn = nullptr;
1770 return true;
1771 }
1772
1773 if (Name.starts_with("ldexp.")) {
1774 // Target specific intrinsic became redundant
1775 NewFn = Intrinsic::getOrInsertDeclaration(
1776 F->getParent(), Intrinsic::ldexp,
1777 {F->getReturnType(), F->getArg(1)->getType()});
1778 return true;
1779 }
1780 break; // No other 'amdgcn.*'
1781 }
1782
1783 break;
1784 }
1785 case 'c': {
1786 if (F->arg_size() == 1) {
1787 if (Name.consume_front("convert.")) {
1788 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1789 NewFn = nullptr;
1790 return true;
1791 }
1792 }
1793
1794 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1795 .StartsWith("ctlz.", Intrinsic::ctlz)
1796 .StartsWith("cttz.", Intrinsic::cttz)
1797 .Default(Intrinsic::not_intrinsic);
1798 if (ID != Intrinsic::not_intrinsic) {
1799 rename(F);
1800 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1801 F->arg_begin()->getType());
1802 return true;
1803 }
1804 }
1805
1806 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1807 if (Name == "coro.end" &&
1808 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1809 CoroEndID = Intrinsic::coro_end;
1810 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1811 CoroEndID = Intrinsic::coro_end_async;
1812
1813 if (CoroEndID != Intrinsic::not_intrinsic) {
1814 rename(F);
1815 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1816 return true;
1817 }
1818
1819 break;
1820 }
1821 case 'd':
1822 if (Name.consume_front("dbg.")) {
1823 // Mark debug intrinsics for upgrade to new debug format.
1824 if (CanUpgradeDebugIntrinsicsToRecords) {
1825 if (Name == "addr" || Name == "value" || Name == "assign" ||
1826 Name == "declare" || Name == "label") {
1827 // There's no function to replace these with.
1828 NewFn = nullptr;
1829 // But we do want these to get upgraded.
1830 return true;
1831 }
1832 }
1833 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1834 // converted to DbgVariableRecords later.
1835 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1836 rename(F);
1837 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1838 Intrinsic::dbg_value);
1839 return true;
1840 }
1841 break; // No other 'dbg.*'.
1842 }
1843 break;
1844 case 'e':
1845 if (Name.consume_front("experimental.vector.")) {
1846 Intrinsic::ID ID =
1847 StringSwitch<Intrinsic::ID>(Name)
1848 // Skip over extract.last.active, otherwise it will be 'upgraded'
1849 // to a regular vector extract which is a different operation.
1850 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1851 .StartsWith("extract.", Intrinsic::vector_extract)
1852 .StartsWith("insert.", Intrinsic::vector_insert)
1853 .StartsWith("reverse.", Intrinsic::vector_reverse)
1854 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1855 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1856 .StartsWith("partial.reduce.add",
1857 Intrinsic::vector_partial_reduce_add)
1858 .Default(Intrinsic::not_intrinsic);
1859 if (ID != Intrinsic::not_intrinsic) {
1860 const auto *FT = F->getFunctionType();
1861 SmallVector<Type *, 2> Tys;
1862 if (ID == Intrinsic::vector_extract ||
1863 ID == Intrinsic::vector_interleave2)
1864 // Extracting overloads the return type.
1865 Tys.push_back(FT->getReturnType());
1866 if (ID != Intrinsic::vector_interleave2)
1867 Tys.push_back(FT->getParamType(0));
1868 if (ID == Intrinsic::vector_insert ||
1869 ID == Intrinsic::vector_partial_reduce_add)
1870 // Inserting overloads the inserted type.
1871 Tys.push_back(FT->getParamType(1));
1872 rename(F);
1873 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1874 return true;
1875 }
1876
1877 if (Name.consume_front("reduce.")) {
1878 SmallVector<StringRef, 2> Groups;
1879 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1880 if (R.match(Name, &Groups))
1881 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1882 .Case("add", Intrinsic::vector_reduce_add)
1883 .Case("mul", Intrinsic::vector_reduce_mul)
1884 .Case("and", Intrinsic::vector_reduce_and)
1885 .Case("or", Intrinsic::vector_reduce_or)
1886 .Case("xor", Intrinsic::vector_reduce_xor)
1887 .Case("smax", Intrinsic::vector_reduce_smax)
1888 .Case("smin", Intrinsic::vector_reduce_smin)
1889 .Case("umax", Intrinsic::vector_reduce_umax)
1890 .Case("umin", Intrinsic::vector_reduce_umin)
1891 .Case("fmax", Intrinsic::vector_reduce_fmax)
1892 .Case("fmin", Intrinsic::vector_reduce_fmin)
1893 .Default(Intrinsic::not_intrinsic);
1894
1895 bool V2 = false;
1896 if (ID == Intrinsic::not_intrinsic) {
1897 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1898 Groups.clear();
1899 V2 = true;
1900 if (R2.match(Name, &Groups))
1901 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1902 .Case("fadd", Intrinsic::vector_reduce_fadd)
1903 .Case("fmul", Intrinsic::vector_reduce_fmul)
1904 .Default(Intrinsic::not_intrinsic);
1905 }
1906 if (ID != Intrinsic::not_intrinsic) {
1907 rename(F);
1908 auto Args = F->getFunctionType()->params();
1909 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1910 {Args[V2 ? 1 : 0]});
1911 return true;
1912 }
1913 break; // No other 'expermental.vector.reduce.*'.
1914 }
1915
1916 if (Name.consume_front("splice"))
1917 return true;
1918 break; // No other 'experimental.vector.*'.
1919 }
1920 if (Name.consume_front("experimental.stepvector.")) {
1921 Intrinsic::ID ID = Intrinsic::stepvector;
1922 rename(F);
1923 NewFn = Intrinsic::getOrInsertDeclaration(
1924 F->getParent(), ID, F->getFunctionType()->getReturnType());
1925 return true;
1926 }
1927 break; // No other 'e*'.
1928 case 'f':
1929 if (Name.starts_with("flt.rounds")) {
1930 rename(F);
1931 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1932 Intrinsic::get_rounding);
1933 return true;
1934 }
1935 break;
1936 case 'i':
1937 if (Name.starts_with("invariant.group.barrier")) {
1938 // Rename invariant.group.barrier to launder.invariant.group
1939 auto Args = F->getFunctionType()->params();
1940 Type* ObjectPtr[1] = {Args[0]};
1941 rename(F);
1942 NewFn = Intrinsic::getOrInsertDeclaration(
1943 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1944 return true;
1945 }
1946 break;
1947 case 'l': {
1948 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1949 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1950 if (IsLifetimeStart || IsLifetimeEnd) {
1951 if (F->arg_size() == 2) {
1952 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1953 : Intrinsic::lifetime_end;
1954 rename(F);
1955 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1956 // arguments. Use the Ptr argument to create new declaration.
1957 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1958 F->getArg(1)->getType());
1959 return true;
1960 } else if (F->arg_size() == 1 && Name == ".i64") {
1961 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1962 // Autoupgrade prior to
1963 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1964 // intrinsic with no expected calls. To allow auto-upgrade process to
1965 // delete such invalid intrinsic declaration, set NewFn = nullptr
1966 // and return true here. If there are actual calls to this intrinsic
1967 // (which is not expected), they will be deleted in
1968 // UpgradeIntrinsicCall.
1969 NewFn = nullptr;
1970 return true;
1971 }
1972 }
1973 break;
1974 }
1975 case 'm': {
1976 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1977 // alignment parameter to embedding the alignment as an attribute of
1978 // the pointer args.
1979 if (unsigned ID = StringSwitch<unsigned>(Name)
1980 .StartsWith("memcpy.", Intrinsic::memcpy)
1981 .StartsWith("memmove.", Intrinsic::memmove)
1982 .Default(0)) {
1983 if (F->arg_size() == 5) {
1984 rename(F);
1985 // Get the types of dest, src, and len
1986 ArrayRef<Type *> ParamTypes =
1987 F->getFunctionType()->params().slice(0, 3);
1988 NewFn =
1989 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1990 return true;
1991 }
1992 }
1993 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1994 rename(F);
1995 // Get the types of dest, and len
1996 const auto *FT = F->getFunctionType();
1997 Type *ParamTypes[2] = {
1998 FT->getParamType(0), // Dest
1999 FT->getParamType(2) // len
2000 };
2001 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2002 Intrinsic::memset, ParamTypes);
2003 return true;
2004 }
2005
2006 unsigned MaskedID =
2007 StringSwitch<unsigned>(Name)
2008 .StartsWith("masked.load", Intrinsic::masked_load)
2009 .StartsWith("masked.gather", Intrinsic::masked_gather)
2010 .StartsWith("masked.store", Intrinsic::masked_store)
2011 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2012 .Default(0);
2013 if (MaskedID && F->arg_size() == 4) {
2014 rename(F);
2015 if (MaskedID == Intrinsic::masked_load ||
2016 MaskedID == Intrinsic::masked_gather) {
2017 NewFn = Intrinsic::getOrInsertDeclaration(
2018 F->getParent(), MaskedID,
2019 {F->getReturnType(), F->getArg(0)->getType()});
2020 return true;
2021 }
2022 NewFn = Intrinsic::getOrInsertDeclaration(
2023 F->getParent(), MaskedID,
2024 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2025 return true;
2026 }
2027 break;
2028 }
2029 case 'n': {
2030 if (Name.consume_front("nvvm.")) {
2031 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2032 if (F->arg_size() == 1) {
2033 Intrinsic::ID IID =
2034 StringSwitch<Intrinsic::ID>(Name)
2035 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2036 .Case("clz.i", Intrinsic::ctlz)
2037 .Case("popc.i", Intrinsic::ctpop)
2038 .Default(Intrinsic::not_intrinsic);
2039 if (IID != Intrinsic::not_intrinsic) {
2040 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2041 {F->getReturnType()});
2042 return true;
2043 }
2044 } else if (F->arg_size() == 2) {
2045 Intrinsic::ID IID =
2046 StringSwitch<Intrinsic::ID>(Name)
2047 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2048 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2049 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2050 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2051 .Default(Intrinsic::not_intrinsic);
2052 if (IID != Intrinsic::not_intrinsic) {
2053 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2054 {F->getReturnType()});
2055 return true;
2056 }
2057 }
2058
2059 // Check for nvvm intrinsics that need a return type adjustment.
2060 {
2061 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2062 if (IID != Intrinsic::not_intrinsic &&
2063 isLegacyNVPTXBF16IntSignature(F, IID)) {
2064 NewFn = nullptr;
2065 return true;
2066 }
2067 }
2068
2069 // Upgrade Distributed Shared Memory Intrinsics
2070 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2071 if (IID != Intrinsic::not_intrinsic) {
2072 rename(F);
2073 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2074 return true;
2075 }
2076
2077 // Upgrade TMA reduction intrinsics
2078 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2079 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2080 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2081 if (IID != Intrinsic::not_intrinsic) {
2082 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2083 return true;
2084 }
2085
2086 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2087 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2088 if (IID != Intrinsic::not_intrinsic) {
2089 rename(F);
2090 NewFn = Intrinsic::getOrInsertDeclaration(
2091 F->getParent(), IID, F->getReturnType(),
2092 F->getFunctionType()->params());
2093 return true;
2094 }
2095
2096 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2097 // tcgen05.alloc shared variants to anyptr intrinsics.
2098 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2099 if (IID != Intrinsic::not_intrinsic) {
2100 rename(F);
2101 if (Intrinsic::isOverloaded(IID))
2102 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2103 {F->getArg(0)->getType()});
2104 else
2105 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2106 return true;
2107 }
2108
2109 // Upgrade TMA copy G2S CTA intrinsics.
2110 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2111 if (IID != Intrinsic::not_intrinsic) {
2112 rename(F);
2113 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2114 return true;
2115 }
2116
2117 // Upgrade TMA copy G2S (cluster) intrinsics.
2118 SmallVector<Type *, 1> OvlTys;
2119 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2120 if (IID != Intrinsic::not_intrinsic) {
2121 rename(F);
2122 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2123 return true;
2124 }
2125
2126 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2127 // (multicast-mask overloading + trailing flag_valid_pattern).
2128 SmallVector<Type *, 1> BulkG2SOvlTys;
2129 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2130 if (IID != Intrinsic::not_intrinsic) {
2131 rename(F);
2132 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2133 BulkG2SOvlTys);
2134 return true;
2135 }
2136
2137 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2138 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2139 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2140 if (IID != Intrinsic::not_intrinsic) {
2141 rename(F);
2142 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2143 return true;
2144 }
2145
2146 // Upgrade mbarrier.init intrinsics missing the layout operand.
2147 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2148 if (IID != Intrinsic::not_intrinsic) {
2149 rename(F);
2150 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2151 F->getArg(0)->getType());
2152 return true;
2153 }
2154
2155 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2156 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2157 //
2158 // TODO: We could add lohi.i2d.
2159 bool Expand = false;
2160 if (Name.consume_front("abs."))
2161 // nvvm.abs.{i,ii}
2162 Expand =
2163 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2164 else if (Name.consume_front("fabs."))
2165 // nvvm.fabs.{f,ftz.f,d}
2166 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2167 else if (Name.consume_front("add."))
2168 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2169 Expand = getNVVMFAddUpgrade(Name).has_value();
2170 else if (Name.consume_front("ex2.approx."))
2171 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2172 Expand =
2173 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2174 else if (Name.consume_front("atomic.load."))
2175 // nvvm.atomic.load.add.{f32,f64}.p
2176 // nvvm.atomic.load.{inc,dec}.32.p
2177 Expand = StringSwitch<bool>(Name)
2178 .StartsWith("add.f32.p", true)
2179 .StartsWith("add.f64.p", true)
2180 .StartsWith("inc.32.p", true)
2181 .StartsWith("dec.32.p", true)
2182 .Default(false);
2183 else if (Name.consume_front("atomic."))
2184 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2185 // nvvm.atomic.cas.gen.i.{cta,sys}
2186 Expand = StringSwitch<bool>(Name)
2187 .StartsWith("add.gen.", true)
2188 .StartsWith("exch.gen.", true)
2189 .StartsWith("max.gen.", true)
2190 .StartsWith("min.gen.", true)
2191 .StartsWith("inc.gen.", true)
2192 .StartsWith("dec.gen.", true)
2193 .StartsWith("and.gen.", true)
2194 .StartsWith("or.gen.", true)
2195 .StartsWith("xor.gen.", true)
2196 .StartsWith("cas.gen.", true)
2197 .Default(false);
2198 else if (Name.consume_front("bitcast."))
2199 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2200 Expand =
2201 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2202 else if (Name.consume_front("rotate."))
2203 // nvvm.rotate.{b32,b64,right.b64}
2204 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2205 else if (Name.consume_front("ptr.gen.to."))
2206 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2207 Expand = consumeNVVMPtrAddrSpace(Name);
2208 else if (Name.consume_front("ptr."))
2209 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2210 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2211 else if (Name.consume_front("ldg.global."))
2212 // nvvm.ldg.global.{i,p,f}
2213 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2214 Name.starts_with("p."));
2215 else
2216 Expand = StringSwitch<bool>(Name)
2217 .Case("barrier0", true)
2218 .Case("barrier.n", true)
2219 .Case("barrier.sync.cnt", true)
2220 .Case("barrier.sync", true)
2221 .Case("barrier", true)
2222 .Case("bar.sync", true)
2223 .Case("barrier0.popc", true)
2224 .Case("barrier0.and", true)
2225 .Case("barrier0.or", true)
2226 .Case("clz.ll", true)
2227 .Case("popc.ll", true)
2228 .Case("h2f", true)
2229 .Case("swap.lo.hi.b64", true)
2230 .Case("tanh.approx.f32", true)
2231 .Default(false);
2232
2233 if (Expand) {
2234 NewFn = nullptr;
2235 return true;
2236 }
2237 break; // No other 'nvvm.*'.
2238 }
2239 break;
2240 }
2241 case 'o':
2242 if (Name.starts_with("objectsize.")) {
2243 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2244 if (F->arg_size() == 2 || F->arg_size() == 3) {
2245 rename(F);
2246 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2247 Intrinsic::objectsize, Tys);
2248 return true;
2249 }
2250 }
2251 break;
2252
2253 case 'p':
2254 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2255 rename(F);
2256 NewFn = Intrinsic::getOrInsertDeclaration(
2257 F->getParent(), Intrinsic::ptr_annotation,
2258 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2259 return true;
2260 }
2261 break;
2262
2263 case 'r': {
2264 if (Name.consume_front("riscv.")) {
2265 Intrinsic::ID ID;
2266 ID = StringSwitch<Intrinsic::ID>(Name)
2267 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2268 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2269 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2270 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2271 .Default(Intrinsic::not_intrinsic);
2272 if (ID != Intrinsic::not_intrinsic) {
2273 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2274 rename(F);
2275 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2276 return true;
2277 }
2278 break; // No other applicable upgrades.
2279 }
2280
2281 ID = StringSwitch<Intrinsic::ID>(Name)
2282 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2283 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2284 .Default(Intrinsic::not_intrinsic);
2285 if (ID != Intrinsic::not_intrinsic) {
2286 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2287 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2288 rename(F);
2289 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2290 return true;
2291 }
2292 break; // No other applicable upgrades.
2293 }
2294
2295 ID = StringSwitch<Intrinsic::ID>(Name)
2296 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2297 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2298 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2299 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2300 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2301 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2302 .Default(Intrinsic::not_intrinsic);
2303 if (ID != Intrinsic::not_intrinsic) {
2304 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2305 rename(F);
2306 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2307 return true;
2308 }
2309 break; // No other applicable upgrades.
2310 }
2311
2312 // Replace llvm.riscv.clmul with llvm.clmul.
2313 if (Name == "clmul.i32" || Name == "clmul.i64") {
2314 NewFn = Intrinsic::getOrInsertDeclaration(
2315 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2316 return true;
2317 }
2318
2319 break; // No other 'riscv.*' intrinsics
2320 }
2321 } break;
2322
2323 case 's':
2324 if (Name == "stackprotectorcheck") {
2325 NewFn = nullptr;
2326 return true;
2327 }
2328 break;
2329
2330 case 't':
2331 if (Name == "thread.pointer") {
2332 NewFn = Intrinsic::getOrInsertDeclaration(
2333 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2334 return true;
2335 }
2336 break;
2337
2338 case 'v': {
2339 if (Name == "var.annotation" && F->arg_size() == 4) {
2340 rename(F);
2341 NewFn = Intrinsic::getOrInsertDeclaration(
2342 F->getParent(), Intrinsic::var_annotation,
2343 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2344 return true;
2345 }
2346 if (Name.consume_front("vector.splice")) {
2347 if (Name.starts_with(".left") || Name.starts_with(".right"))
2348 break;
2349 return true;
2350 }
2351 if (shouldUpgradeVPIntrinsic(Name))
2352 return true;
2353 break;
2354 }
2355
2356 case 'w':
2357 if (Name.consume_front("wasm.")) {
2358 Intrinsic::ID ID =
2359 StringSwitch<Intrinsic::ID>(Name)
2360 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2361 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2362 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2363 .Default(Intrinsic::not_intrinsic);
2364 if (ID != Intrinsic::not_intrinsic) {
2365 rename(F);
2366 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2367 F->getReturnType());
2368 return true;
2369 }
2370
2371 if (Name.consume_front("dot.i8x16.i7x16.")) {
2372 ID = StringSwitch<Intrinsic::ID>(Name)
2373 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2374 .Case("add.signed",
2375 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2376 .Default(Intrinsic::not_intrinsic);
2377 if (ID != Intrinsic::not_intrinsic) {
2378 rename(F);
2379 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2380 return true;
2381 }
2382 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2383 }
2384 break; // No other 'wasm.*'.
2385 }
2386 break;
2387
2388 case 'x':
2389 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2390 return true;
2391 }
2392
2393 auto *ST = dyn_cast<StructType>(F->getReturnType());
2394 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2395 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2396 // Replace return type with literal non-packed struct. Only do this for
2397 // intrinsics declared to return a struct, not for intrinsics with
2398 // overloaded return type, in which case the exact struct type will be
2399 // mangled into the name.
2400 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2401 FunctionType *FT = F->getFunctionType();
2402 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2403 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2404 std::string Name = F->getName().str();
2405 rename(F);
2406 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2407 Name, F->getParent());
2408
2409 // The new function may also need remangling.
2410 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2411 NewFn = *Result;
2412 return true;
2413 }
2414 }
2415
2416 // Remangle our intrinsic since we upgrade the mangling
2417 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2418 if (Result != std::nullopt) {
2419 NewFn = *Result;
2420 return true;
2421 }
2422
2423 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2424 return true;
2425
2426 // This may not belong here. This function is effectively being overloaded
2427 // to both detect an intrinsic which needs upgrading, and to provide the
2428 // upgraded form of the intrinsic. We should perhaps have two separate
2429 // functions for this.
2430
2431 return false;
2432}
2433
2434bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2435 bool CanUpgradeDebugIntrinsicsToRecords) {
2436 NewFn = nullptr;
2437 bool Upgraded =
2438 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2439
2440 // Upgrade intrinsic attributes. This does not change the function.
2441 if (NewFn)
2442 F = NewFn;
2443 if (Intrinsic::ID id = F->getIntrinsicID()) {
2444 // Only do this if the intrinsic signature is valid.
2445 SmallVector<Type *> OverloadTys;
2446 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2447 F->setAttributes(
2448 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2449 }
2450 return Upgraded;
2451}
2452
2453GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2454 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2455 GV->getName() == "llvm.global_dtors")) ||
2456 !GV->hasInitializer())
2457 return nullptr;
2458 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2459 if (!ATy)
2460 return nullptr;
2461 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2462 if (!STy || STy->getNumElements() != 2)
2463 return nullptr;
2464
2465 LLVMContext &C = GV->getContext();
2466 IRBuilder<> IRB(C);
2467 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2468 IRB.getPtrTy());
2469 Constant *Init = GV->getInitializer();
2470 unsigned N = Init->getNumOperands();
2471 std::vector<Constant *> NewCtors(N);
2472 for (unsigned i = 0; i != N; ++i) {
2473 auto Ctor = cast<Constant>(Init->getOperand(i));
2474 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2475 Ctor->getAggregateElement(1),
2476 ConstantPointerNull::get(IRB.getPtrTy()));
2477 }
2478 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2479
2480 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2481 NewInit, GV->getName());
2482}
2483
2484// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2485// to byte shuffles.
2486static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2487 unsigned Shift) {
2488 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2489 unsigned NumElts = ResultTy->getNumElements() * 8;
2490
2491 // Bitcast from a 64-bit element type to a byte element type.
2492 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2493 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2494
2495 // We'll be shuffling in zeroes.
2496 Value *Res = Constant::getNullValue(VecTy);
2497
2498 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2499 // we'll just return the zero vector.
2500 if (Shift < 16) {
2501 int Idxs[64];
2502 // 256/512-bit version is split into 2/4 16-byte lanes.
2503 for (unsigned l = 0; l != NumElts; l += 16)
2504 for (unsigned i = 0; i != 16; ++i) {
2505 unsigned Idx = NumElts + i - Shift;
2506 if (Idx < NumElts)
2507 Idx -= NumElts - 16; // end of lane, switch operand.
2508 Idxs[l + i] = Idx + l;
2509 }
2510
2511 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2512 }
2513
2514 // Bitcast back to a 64-bit element type.
2515 return Builder.CreateBitCast(Res, ResultTy, "cast");
2516}
2517
2518// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2519// to byte shuffles.
2520static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2521 unsigned Shift) {
2522 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2523 unsigned NumElts = ResultTy->getNumElements() * 8;
2524
2525 // Bitcast from a 64-bit element type to a byte element type.
2526 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2527 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2528
2529 // We'll be shuffling in zeroes.
2530 Value *Res = Constant::getNullValue(VecTy);
2531
2532 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2533 // we'll just return the zero vector.
2534 if (Shift < 16) {
2535 int Idxs[64];
2536 // 256/512-bit version is split into 2/4 16-byte lanes.
2537 for (unsigned l = 0; l != NumElts; l += 16)
2538 for (unsigned i = 0; i != 16; ++i) {
2539 unsigned Idx = i + Shift;
2540 if (Idx >= 16)
2541 Idx += NumElts - 16; // end of lane, switch operand.
2542 Idxs[l + i] = Idx + l;
2543 }
2544
2545 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2546 }
2547
2548 // Bitcast back to a 64-bit element type.
2549 return Builder.CreateBitCast(Res, ResultTy, "cast");
2550}
2551
2552static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2553 unsigned NumElts) {
2554 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2555 llvm::VectorType *MaskTy = FixedVectorType::get(
2556 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2557 Mask = Builder.CreateBitCast(Mask, MaskTy);
2558
2559 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2560 // i8 and we need to extract down to the right number of elements.
2561 if (NumElts <= 4) {
2562 int Indices[4];
2563 for (unsigned i = 0; i != NumElts; ++i)
2564 Indices[i] = i;
2565 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2566 "extract");
2567 }
2568
2569 return Mask;
2570}
2571
2572static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2573 Value *Op1) {
2574 // If the mask is all ones just emit the first operation.
2575 if (const auto *C = dyn_cast<Constant>(Mask))
2576 if (C->isAllOnesValue())
2577 return Op0;
2578
2579 Mask = getX86MaskVec(Builder, Mask,
2580 cast<FixedVectorType>(Op0->getType())->getNumElements());
2581 return Builder.CreateSelect(Mask, Op0, Op1);
2582}
2583
2584static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2585 Value *Op1) {
2586 // If the mask is all ones just emit the first operation.
2587 if (const auto *C = dyn_cast<Constant>(Mask))
2588 if (C->isAllOnesValue())
2589 return Op0;
2590
2591 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2592 Mask->getType()->getIntegerBitWidth());
2593 Mask = Builder.CreateBitCast(Mask, MaskTy);
2594 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2595 return Builder.CreateSelect(Mask, Op0, Op1);
2596}
2597
2598// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2599// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2600// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2601static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2602 Value *Op1, Value *Shift,
2603 Value *Passthru, Value *Mask,
2604 bool IsVALIGN) {
2605 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2606
2607 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2608 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2609 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2610 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2611
2612 // Mask the immediate for VALIGN.
2613 if (IsVALIGN)
2614 ShiftVal &= (NumElts - 1);
2615
2616 // If palignr is shifting the pair of vectors more than the size of two
2617 // lanes, emit zero.
2618 if (ShiftVal >= 32)
2619 return llvm::Constant::getNullValue(Op0->getType());
2620
2621 // If palignr is shifting the pair of input vectors more than one lane,
2622 // but less than two lanes, convert to shifting in zeroes.
2623 if (ShiftVal > 16) {
2624 ShiftVal -= 16;
2625 Op1 = Op0;
2626 Op0 = llvm::Constant::getNullValue(Op0->getType());
2627 }
2628
2629 int Indices[64];
2630 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2631 for (unsigned l = 0; l < NumElts; l += 16) {
2632 for (unsigned i = 0; i != 16; ++i) {
2633 unsigned Idx = ShiftVal + i;
2634 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2635 Idx += NumElts - 16; // End of lane, switch operand.
2636 Indices[l + i] = Idx + l;
2637 }
2638 }
2639
2640 Value *Align = Builder.CreateShuffleVector(
2641 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2642
2643 return emitX86Select(Builder, Mask, Align, Passthru);
2644}
2645
2646static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2647 bool ZeroMask, bool IndexForm) {
2648 Type *Ty = CI.getType();
2649 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2650 unsigned EltWidth = Ty->getScalarSizeInBits();
2651 bool IsFloat = Ty->isFPOrFPVectorTy();
2652 Intrinsic::ID IID;
2653 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2654 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2655 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2656 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2657 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2658 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2659 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2660 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2661 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2662 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2663 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2664 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2665 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2666 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2667 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2668 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2669 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2670 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2671 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2672 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2673 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2674 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2675 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2676 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2677 else if (VecWidth == 128 && EltWidth == 16)
2678 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2679 else if (VecWidth == 256 && EltWidth == 16)
2680 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2681 else if (VecWidth == 512 && EltWidth == 16)
2682 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2683 else if (VecWidth == 128 && EltWidth == 8)
2684 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2685 else if (VecWidth == 256 && EltWidth == 8)
2686 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2687 else if (VecWidth == 512 && EltWidth == 8)
2688 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2689 else
2690 llvm_unreachable("Unexpected intrinsic");
2691
2692 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2693 CI.getArgOperand(2) };
2694
2695 // If this isn't index form we need to swap operand 0 and 1.
2696 if (!IndexForm)
2697 std::swap(Args[0], Args[1]);
2698
2699 Value *V = Builder.CreateIntrinsic(IID, Args);
2700 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2701 : Builder.CreateBitCast(CI.getArgOperand(1),
2702 Ty);
2703 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2704}
2705
2706static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2707 Intrinsic::ID IID) {
2708 Type *Ty = CI.getType();
2709 Value *Op0 = CI.getOperand(0);
2710 Value *Op1 = CI.getOperand(1);
2711 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2712
2713 if (CI.arg_size() == 4) { // For masked intrinsics.
2714 Value *VecSrc = CI.getOperand(2);
2715 Value *Mask = CI.getOperand(3);
2716 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2717 }
2718 return Res;
2719}
2720
2721static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2722 bool IsRotateRight) {
2723 Type *Ty = CI.getType();
2724 Value *Src = CI.getArgOperand(0);
2725 Value *Amt = CI.getArgOperand(1);
2726
2727 // Amount may be scalar immediate, in which case create a splat vector.
2728 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2729 // we only care about the lowest log2 bits anyway.
2730 if (Amt->getType() != Ty) {
2731 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2732 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2733 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2734 }
2735
2736 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2737 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2738
2739 if (CI.arg_size() == 4) { // For masked intrinsics.
2740 Value *VecSrc = CI.getOperand(2);
2741 Value *Mask = CI.getOperand(3);
2742 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2743 }
2744 return Res;
2745}
2746
2747static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2748 bool IsSigned) {
2749 Type *Ty = CI.getType();
2750 Value *LHS = CI.getArgOperand(0);
2751 Value *RHS = CI.getArgOperand(1);
2752
2753 CmpInst::Predicate Pred;
2754 switch (Imm) {
2755 case 0x0:
2756 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2757 break;
2758 case 0x1:
2759 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2760 break;
2761 case 0x2:
2762 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2763 break;
2764 case 0x3:
2765 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2766 break;
2767 case 0x4:
2768 Pred = ICmpInst::ICMP_EQ;
2769 break;
2770 case 0x5:
2771 Pred = ICmpInst::ICMP_NE;
2772 break;
2773 case 0x6:
2774 return Constant::getNullValue(Ty); // FALSE
2775 case 0x7:
2776 return Constant::getAllOnesValue(Ty); // TRUE
2777 default:
2778 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2779 }
2780
2781 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2782 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2783 return Ext;
2784}
2785
2786static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2787 bool IsShiftRight, bool ZeroMask) {
2788 Type *Ty = CI.getType();
2789 Value *Op0 = CI.getArgOperand(0);
2790 Value *Op1 = CI.getArgOperand(1);
2791 Value *Amt = CI.getArgOperand(2);
2792
2793 if (IsShiftRight)
2794 std::swap(Op0, Op1);
2795
2796 // Amount may be scalar immediate, in which case create a splat vector.
2797 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2798 // we only care about the lowest log2 bits anyway.
2799 if (Amt->getType() != Ty) {
2800 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2801 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2802 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2803 }
2804
2805 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2806 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2807
2808 unsigned NumArgs = CI.arg_size();
2809 if (NumArgs >= 4) { // For masked intrinsics.
2810 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2811 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2812 CI.getArgOperand(0);
2813 Value *Mask = CI.getOperand(NumArgs - 1);
2814 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2815 }
2816 return Res;
2817}
2818
2819static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2820 Value *Mask, bool Aligned) {
2821 const Align Alignment =
2822 Aligned
2823 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2824 : Align(1);
2825
2826 // If the mask is all ones just emit a regular store.
2827 if (const auto *C = dyn_cast<Constant>(Mask))
2828 if (C->isAllOnesValue())
2829 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2830
2831 // Convert the mask from an integer type to a vector of i1.
2832 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2833 Mask = getX86MaskVec(Builder, Mask, NumElts);
2834 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2835}
2836
2837static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2838 Value *Passthru, Value *Mask, bool Aligned) {
2839 Type *ValTy = Passthru->getType();
2840 const Align Alignment =
2841 Aligned
2842 ? Align(
2843 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2844 8)
2845 : Align(1);
2846
2847 // If the mask is all ones just emit a regular store.
2848 if (const auto *C = dyn_cast<Constant>(Mask))
2849 if (C->isAllOnesValue())
2850 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2851
2852 // Convert the mask from an integer type to a vector of i1.
2853 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2854 Mask = getX86MaskVec(Builder, Mask, NumElts);
2855 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2856}
2857
2858static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2859 Type *Ty = CI.getType();
2860 Value *Op0 = CI.getArgOperand(0);
2861 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2862 {Op0, Builder.getInt1(false)});
2863 if (CI.arg_size() == 3)
2864 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2865 return Res;
2866}
2867
2868static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2869 Type *Ty = CI.getType();
2870
2871 // Arguments have a vXi32 type so cast to vXi64.
2872 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2873 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2874
2875 if (IsSigned) {
2876 // Shift left then arithmetic shift right.
2877 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2878 LHS = Builder.CreateShl(LHS, ShiftAmt);
2879 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2880 RHS = Builder.CreateShl(RHS, ShiftAmt);
2881 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2882 } else {
2883 // Clear the upper bits.
2884 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2885 LHS = Builder.CreateAnd(LHS, Mask);
2886 RHS = Builder.CreateAnd(RHS, Mask);
2887 }
2888
2889 Value *Res = Builder.CreateMul(LHS, RHS);
2890
2891 if (CI.arg_size() == 4)
2892 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2893
2894 return Res;
2895}
2896
2897// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2898static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2899 Value *Mask) {
2900 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2901 if (Mask) {
2902 const auto *C = dyn_cast<Constant>(Mask);
2903 if (!C || !C->isAllOnesValue())
2904 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2905 }
2906
2907 if (NumElts < 8) {
2908 int Indices[8];
2909 for (unsigned i = 0; i != NumElts; ++i)
2910 Indices[i] = i;
2911 for (unsigned i = NumElts; i != 8; ++i)
2912 Indices[i] = NumElts + i % NumElts;
2913 Vec = Builder.CreateShuffleVector(Vec,
2914 Constant::getNullValue(Vec->getType()),
2915 Indices);
2916 }
2917 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2918}
2919
2920static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2921 unsigned CC, bool Signed) {
2922 Value *Op0 = CI.getArgOperand(0);
2923 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2924
2925 Value *Cmp;
2926 if (CC == 3) {
2927 Cmp = Constant::getNullValue(
2928 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2929 } else if (CC == 7) {
2930 Cmp = Constant::getAllOnesValue(
2931 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2932 } else {
2933 ICmpInst::Predicate Pred;
2934 switch (CC) {
2935 default: llvm_unreachable("Unknown condition code");
2936 case 0: Pred = ICmpInst::ICMP_EQ; break;
2937 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2938 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2939 case 4: Pred = ICmpInst::ICMP_NE; break;
2940 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2941 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2942 }
2943 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2944 }
2945
2946 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2947
2948 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2949}
2950
2951// Replace a masked intrinsic with an older unmasked intrinsic.
2952static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2953 Intrinsic::ID IID) {
2954 Value *Rep =
2955 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2956 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2957}
2958
2959static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2960 Value* A = CI.getArgOperand(0);
2961 Value* B = CI.getArgOperand(1);
2962 Value* Src = CI.getArgOperand(2);
2963 Value* Mask = CI.getArgOperand(3);
2964
2965 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2966 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2967 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2968 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2969 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2970 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2971}
2972
2973static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2974 Value* Op = CI.getArgOperand(0);
2975 Type* ReturnOp = CI.getType();
2976 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2977 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2978 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2979}
2980
2981// Replace intrinsic with unmasked version and a select.
2982static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2983 CallBase &CI, Value *&Rep) {
2984 Name = Name.substr(12); // Remove avx512.mask.
2985
2986 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2987 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2988 Intrinsic::ID IID;
2989 if (Name.starts_with("max.p")) {
2990 if (VecWidth == 128 && EltWidth == 32)
2991 IID = Intrinsic::x86_sse_max_ps;
2992 else if (VecWidth == 128 && EltWidth == 64)
2993 IID = Intrinsic::x86_sse2_max_pd;
2994 else if (VecWidth == 256 && EltWidth == 32)
2995 IID = Intrinsic::x86_avx_max_ps_256;
2996 else if (VecWidth == 256 && EltWidth == 64)
2997 IID = Intrinsic::x86_avx_max_pd_256;
2998 else
2999 llvm_unreachable("Unexpected intrinsic");
3000 } else if (Name.starts_with("min.p")) {
3001 if (VecWidth == 128 && EltWidth == 32)
3002 IID = Intrinsic::x86_sse_min_ps;
3003 else if (VecWidth == 128 && EltWidth == 64)
3004 IID = Intrinsic::x86_sse2_min_pd;
3005 else if (VecWidth == 256 && EltWidth == 32)
3006 IID = Intrinsic::x86_avx_min_ps_256;
3007 else if (VecWidth == 256 && EltWidth == 64)
3008 IID = Intrinsic::x86_avx_min_pd_256;
3009 else
3010 llvm_unreachable("Unexpected intrinsic");
3011 } else if (Name.starts_with("pshuf.b.")) {
3012 if (VecWidth == 128)
3013 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3014 else if (VecWidth == 256)
3015 IID = Intrinsic::x86_avx2_pshuf_b;
3016 else if (VecWidth == 512)
3017 IID = Intrinsic::x86_avx512_pshuf_b_512;
3018 else
3019 llvm_unreachable("Unexpected intrinsic");
3020 } else if (Name.starts_with("pmul.hr.sw.")) {
3021 if (VecWidth == 128)
3022 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3023 else if (VecWidth == 256)
3024 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3025 else if (VecWidth == 512)
3026 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3027 else
3028 llvm_unreachable("Unexpected intrinsic");
3029 } else if (Name.starts_with("pmulh.w.")) {
3030 if (VecWidth == 128)
3031 IID = Intrinsic::x86_sse2_pmulh_w;
3032 else if (VecWidth == 256)
3033 IID = Intrinsic::x86_avx2_pmulh_w;
3034 else if (VecWidth == 512)
3035 IID = Intrinsic::x86_avx512_pmulh_w_512;
3036 else
3037 llvm_unreachable("Unexpected intrinsic");
3038 } else if (Name.starts_with("pmulhu.w.")) {
3039 if (VecWidth == 128)
3040 IID = Intrinsic::x86_sse2_pmulhu_w;
3041 else if (VecWidth == 256)
3042 IID = Intrinsic::x86_avx2_pmulhu_w;
3043 else if (VecWidth == 512)
3044 IID = Intrinsic::x86_avx512_pmulhu_w_512;
3045 else
3046 llvm_unreachable("Unexpected intrinsic");
3047 } else if (Name.starts_with("pmaddw.d.")) {
3048 if (VecWidth == 128)
3049 IID = Intrinsic::x86_sse2_pmadd_wd;
3050 else if (VecWidth == 256)
3051 IID = Intrinsic::x86_avx2_pmadd_wd;
3052 else if (VecWidth == 512)
3053 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3054 else
3055 llvm_unreachable("Unexpected intrinsic");
3056 } else if (Name.starts_with("pmaddubs.w.")) {
3057 if (VecWidth == 128)
3058 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3059 else if (VecWidth == 256)
3060 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3061 else if (VecWidth == 512)
3062 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3063 else
3064 llvm_unreachable("Unexpected intrinsic");
3065 } else if (Name.starts_with("packsswb.")) {
3066 if (VecWidth == 128)
3067 IID = Intrinsic::x86_sse2_packsswb_128;
3068 else if (VecWidth == 256)
3069 IID = Intrinsic::x86_avx2_packsswb;
3070 else if (VecWidth == 512)
3071 IID = Intrinsic::x86_avx512_packsswb_512;
3072 else
3073 llvm_unreachable("Unexpected intrinsic");
3074 } else if (Name.starts_with("packssdw.")) {
3075 if (VecWidth == 128)
3076 IID = Intrinsic::x86_sse2_packssdw_128;
3077 else if (VecWidth == 256)
3078 IID = Intrinsic::x86_avx2_packssdw;
3079 else if (VecWidth == 512)
3080 IID = Intrinsic::x86_avx512_packssdw_512;
3081 else
3082 llvm_unreachable("Unexpected intrinsic");
3083 } else if (Name.starts_with("packuswb.")) {
3084 if (VecWidth == 128)
3085 IID = Intrinsic::x86_sse2_packuswb_128;
3086 else if (VecWidth == 256)
3087 IID = Intrinsic::x86_avx2_packuswb;
3088 else if (VecWidth == 512)
3089 IID = Intrinsic::x86_avx512_packuswb_512;
3090 else
3091 llvm_unreachable("Unexpected intrinsic");
3092 } else if (Name.starts_with("packusdw.")) {
3093 if (VecWidth == 128)
3094 IID = Intrinsic::x86_sse41_packusdw;
3095 else if (VecWidth == 256)
3096 IID = Intrinsic::x86_avx2_packusdw;
3097 else if (VecWidth == 512)
3098 IID = Intrinsic::x86_avx512_packusdw_512;
3099 else
3100 llvm_unreachable("Unexpected intrinsic");
3101 } else if (Name.starts_with("vpermilvar.")) {
3102 if (VecWidth == 128 && EltWidth == 32)
3103 IID = Intrinsic::x86_avx_vpermilvar_ps;
3104 else if (VecWidth == 128 && EltWidth == 64)
3105 IID = Intrinsic::x86_avx_vpermilvar_pd;
3106 else if (VecWidth == 256 && EltWidth == 32)
3107 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3108 else if (VecWidth == 256 && EltWidth == 64)
3109 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3110 else if (VecWidth == 512 && EltWidth == 32)
3111 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3112 else if (VecWidth == 512 && EltWidth == 64)
3113 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3114 else
3115 llvm_unreachable("Unexpected intrinsic");
3116 } else if (Name == "cvtpd2dq.256") {
3117 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3118 } else if (Name == "cvtpd2ps.256") {
3119 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3120 } else if (Name == "cvttpd2dq.256") {
3121 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3122 } else if (Name == "cvttps2dq.128") {
3123 IID = Intrinsic::x86_sse2_cvttps2dq;
3124 } else if (Name == "cvttps2dq.256") {
3125 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3126 } else if (Name.starts_with("permvar.")) {
3127 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3128 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3129 IID = Intrinsic::x86_avx2_permps;
3130 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3131 IID = Intrinsic::x86_avx2_permd;
3132 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3133 IID = Intrinsic::x86_avx512_permvar_df_256;
3134 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3135 IID = Intrinsic::x86_avx512_permvar_di_256;
3136 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3137 IID = Intrinsic::x86_avx512_permvar_sf_512;
3138 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3139 IID = Intrinsic::x86_avx512_permvar_si_512;
3140 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3141 IID = Intrinsic::x86_avx512_permvar_df_512;
3142 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3143 IID = Intrinsic::x86_avx512_permvar_di_512;
3144 else if (VecWidth == 128 && EltWidth == 16)
3145 IID = Intrinsic::x86_avx512_permvar_hi_128;
3146 else if (VecWidth == 256 && EltWidth == 16)
3147 IID = Intrinsic::x86_avx512_permvar_hi_256;
3148 else if (VecWidth == 512 && EltWidth == 16)
3149 IID = Intrinsic::x86_avx512_permvar_hi_512;
3150 else if (VecWidth == 128 && EltWidth == 8)
3151 IID = Intrinsic::x86_avx512_permvar_qi_128;
3152 else if (VecWidth == 256 && EltWidth == 8)
3153 IID = Intrinsic::x86_avx512_permvar_qi_256;
3154 else if (VecWidth == 512 && EltWidth == 8)
3155 IID = Intrinsic::x86_avx512_permvar_qi_512;
3156 else
3157 llvm_unreachable("Unexpected intrinsic");
3158 } else if (Name.starts_with("dbpsadbw.")) {
3159 if (VecWidth == 128)
3160 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3161 else if (VecWidth == 256)
3162 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3163 else if (VecWidth == 512)
3164 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3165 else
3166 llvm_unreachable("Unexpected intrinsic");
3167 } else if (Name.starts_with("pmultishift.qb.")) {
3168 if (VecWidth == 128)
3169 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3170 else if (VecWidth == 256)
3171 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3172 else if (VecWidth == 512)
3173 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3174 else
3175 llvm_unreachable("Unexpected intrinsic");
3176 } else if (Name.starts_with("conflict.")) {
3177 if (Name[9] == 'd' && VecWidth == 128)
3178 IID = Intrinsic::x86_avx512_conflict_d_128;
3179 else if (Name[9] == 'd' && VecWidth == 256)
3180 IID = Intrinsic::x86_avx512_conflict_d_256;
3181 else if (Name[9] == 'd' && VecWidth == 512)
3182 IID = Intrinsic::x86_avx512_conflict_d_512;
3183 else if (Name[9] == 'q' && VecWidth == 128)
3184 IID = Intrinsic::x86_avx512_conflict_q_128;
3185 else if (Name[9] == 'q' && VecWidth == 256)
3186 IID = Intrinsic::x86_avx512_conflict_q_256;
3187 else if (Name[9] == 'q' && VecWidth == 512)
3188 IID = Intrinsic::x86_avx512_conflict_q_512;
3189 else
3190 llvm_unreachable("Unexpected intrinsic");
3191 } else if (Name.starts_with("pavg.")) {
3192 if (Name[5] == 'b' && VecWidth == 128)
3193 IID = Intrinsic::x86_sse2_pavg_b;
3194 else if (Name[5] == 'b' && VecWidth == 256)
3195 IID = Intrinsic::x86_avx2_pavg_b;
3196 else if (Name[5] == 'b' && VecWidth == 512)
3197 IID = Intrinsic::x86_avx512_pavg_b_512;
3198 else if (Name[5] == 'w' && VecWidth == 128)
3199 IID = Intrinsic::x86_sse2_pavg_w;
3200 else if (Name[5] == 'w' && VecWidth == 256)
3201 IID = Intrinsic::x86_avx2_pavg_w;
3202 else if (Name[5] == 'w' && VecWidth == 512)
3203 IID = Intrinsic::x86_avx512_pavg_w_512;
3204 else
3205 llvm_unreachable("Unexpected intrinsic");
3206 } else
3207 return false;
3208
3209 SmallVector<Value *, 4> Args(CI.args());
3210 Args.pop_back();
3211 Args.pop_back();
3212 Rep = Builder.CreateIntrinsic(IID, Args);
3213 unsigned NumArgs = CI.arg_size();
3214 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3215 CI.getArgOperand(NumArgs - 2));
3216 return true;
3217}
3218
3219/// Upgrade comment in call to inline asm that represents an objc retain release
3220/// marker.
3221void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3222 size_t Pos;
3223 if (AsmStr->find("mov\tfp") == 0 &&
3224 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3225 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3226 AsmStr->replace(Pos, 1, ";");
3227 }
3228}
3229
3230static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3231 Function *F, IRBuilder<> &Builder) {
3232 Value *Rep = nullptr;
3233
3234 if (Name == "abs.i" || Name == "abs.ll") {
3235 Value *Arg = CI->getArgOperand(0);
3236 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3237 {Arg, Builder.getTrue()},
3238 /*FMFSource=*/nullptr, "abs");
3239 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3240 Type *Ty = (Name == "abs.bf16")
3241 ? Builder.getBFloatTy()
3242 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3243 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3244 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3245 Rep = Builder.CreateBitCast(Abs, CI->getType());
3246 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3247 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3248 : Intrinsic::nvvm_fabs;
3249 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3250 } else if (Name.consume_front("add.")) {
3251 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3252 auto FAdd = getNVVMFAddUpgrade(Name);
3253 assert(FAdd && "unsupported nvvm.add.* intrinsic");
3254 auto [IID, RoundingMode] = *FAdd;
3255 Value *A = CI->getArgOperand(0);
3256 Rep = Builder.CreateIntrinsic(
3257 A->getType(), IID,
3258 {A, CI->getArgOperand(1),
3259 Builder.getInt32(static_cast<int>(RoundingMode))});
3260 } else if (Name.consume_front("ex2.approx.")) {
3261 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3262 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3263 : Intrinsic::nvvm_ex2_approx;
3264 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3265 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3266 Name.starts_with("atomic.load.add.f64.p")) {
3267 Value *Ptr = CI->getArgOperand(0);
3268 Value *Val = CI->getArgOperand(1);
3269 Rep = Builder.CreateAtomicRMW(
3270 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3271 CI->getContext().getOrInsertSyncScopeID("device"));
3272 // The default scope for atomic.load.* intrinsics is device
3273 // (= gpu scope in ptx), but the default LLVM atomic scope is
3274 // "system"
3275 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3276 Name.starts_with("atomic.load.dec.32.p")) {
3277 Value *Ptr = CI->getArgOperand(0);
3278 Value *Val = CI->getArgOperand(1);
3279 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3280 : AtomicRMWInst::UDecWrap;
3281 Rep = Builder.CreateAtomicRMW(
3282 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3283 CI->getContext().getOrInsertSyncScopeID("device"));
3284 // See comment above.
3285 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3286 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3287 StringRef Op = Name.substr(StringRef("atomic.").size());
3288 Value *Ptr = CI->getArgOperand(0);
3289 Value *Val = CI->getArgOperand(1);
3290 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3291 Op.contains(".cta.") ? "block" : "");
3292 if (Op.starts_with("cas.")) {
3293 Value *New = CI->getArgOperand(2);
3294 Value *Pair = Builder.CreateAtomicCmpXchg(
3295 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3296 AtomicOrdering::Monotonic, SSID);
3297 Rep = Builder.CreateExtractValue(Pair, 0);
3298 } else {
3299 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3300 // because we were actually missing those intrinsics!
3301 AtomicRMWInst::BinOp BinOp =
3302 StringSwitch<AtomicRMWInst::BinOp>(Op)
3303 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3304 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3305 .StartsWith("exch.", AtomicRMWInst::Xchg)
3306 .StartsWith("max.", AtomicRMWInst::Max)
3307 .StartsWith("min.", AtomicRMWInst::Min)
3308 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3309 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3310 .StartsWith("and.", AtomicRMWInst::And)
3311 .StartsWith("or.", AtomicRMWInst::Or)
3312 .StartsWith("xor.", AtomicRMWInst::Xor)
3313 .Default(AtomicRMWInst::BAD_BINOP);
3314 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3315 "unexpected nvvm scoped atomic intrinsic");
3316 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3317 AtomicOrdering::Monotonic, SSID);
3318 }
3319 } else if (Name == "clz.ll") {
3320 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3321 Value *Arg = CI->getArgOperand(0);
3322 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3323 {Arg, Builder.getFalse()},
3324 /*FMFSource=*/nullptr, "ctlz");
3325 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3326 } else if (Name == "popc.ll") {
3327 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3328 // i64.
3329 Value *Arg = CI->getArgOperand(0);
3330 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3331 Arg, /*FMFSource=*/nullptr, "ctpop");
3332 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3333 } else if (Name == "h2f") {
3334 Value *Cast =
3335 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3336 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3337 } else if (Name.consume_front("bitcast.") &&
3338 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3339 Name == "d2ll")) {
3340 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3341 } else if (Name == "rotate.b32") {
3342 Value *Arg = CI->getOperand(0);
3343 Value *ShiftAmt = CI->getOperand(1);
3344 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3345 {Arg, Arg, ShiftAmt});
3346 } else if (Name == "rotate.b64") {
3347 Type *Int64Ty = Builder.getInt64Ty();
3348 Value *Arg = CI->getOperand(0);
3349 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3350 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3351 {Arg, Arg, ZExtShiftAmt});
3352 } else if (Name == "rotate.right.b64") {
3353 Type *Int64Ty = Builder.getInt64Ty();
3354 Value *Arg = CI->getOperand(0);
3355 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3356 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3357 {Arg, Arg, ZExtShiftAmt});
3358 } else if (Name == "swap.lo.hi.b64") {
3359 Type *Int64Ty = Builder.getInt64Ty();
3360 Value *Arg = CI->getOperand(0);
3361 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3362 {Arg, Arg, Builder.getInt64(32)});
3363 } else if ((Name.consume_front("ptr.gen.to.") &&
3364 consumeNVVMPtrAddrSpace(Name)) ||
3365 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3366 Name.starts_with(".to.gen"))) {
3367 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3368 } else if (Name.consume_front("ldg.global")) {
3369 Value *Ptr = CI->getArgOperand(0);
3370 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3371 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3372 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3373 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3374 MDNode *MD = MDNode::get(Builder.getContext(), {});
3375 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3376 return LD;
3377 } else if (Name == "tanh.approx.f32") {
3378 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3379 FastMathFlags FMF;
3380 FMF.setApproxFunc();
3381 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3382 FMF);
3383 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3384 Value *Arg =
3385 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3386 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3387 {}, {Arg});
3388 } else if (Name == "barrier") {
3389 Rep = Builder.CreateIntrinsic(
3390 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3391 {CI->getArgOperand(0), CI->getArgOperand(1)});
3392 } else if (Name == "barrier.sync") {
3393 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3394 {CI->getArgOperand(0)});
3395 } else if (Name == "barrier.sync.cnt") {
3396 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3397 {CI->getArgOperand(0), CI->getArgOperand(1)});
3398 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3399 Name == "barrier0.or") {
3400 Value *C = CI->getArgOperand(0);
3401 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3402
3403 Intrinsic::ID IID =
3404 StringSwitch<Intrinsic::ID>(Name)
3405 .Case("barrier0.popc",
3406 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3407 .Case("barrier0.and",
3408 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3409 .Case("barrier0.or",
3410 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3411 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3412 Rep = Builder.CreateZExt(Bar, CI->getType());
3413 } else {
3414 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3415 if (IID != Intrinsic::not_intrinsic &&
3416 isLegacyNVPTXBF16IntSignature(F, IID)) {
3417 rename(F);
3418 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3419 SmallVector<Value *, 2> Args;
3420 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3421 Value *Arg = CI->getArgOperand(I);
3422 Type *OldType = Arg->getType();
3423 Type *NewType = NewFn->getArg(I)->getType();
3424 Args.push_back(
3425 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3426 ? Builder.CreateBitCast(Arg, NewType)
3427 : Arg);
3428 }
3429 Rep = Builder.CreateCall(NewFn, Args);
3430 if (F->getReturnType()->isIntegerTy())
3431 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3432 }
3433 }
3434
3435 return Rep;
3436}
3437
3438static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3439 IRBuilder<> &Builder) {
3440 LLVMContext &C = F->getContext();
3441 Value *Rep = nullptr;
3442
3443 if (Name.starts_with("sse4a.movnt.")) {
3444 SmallVector<Metadata *, 1> Elts;
3445 Elts.push_back(
3446 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3447 MDNode *Node = MDNode::get(C, Elts);
3448
3449 Value *Arg0 = CI->getArgOperand(0);
3450 Value *Arg1 = CI->getArgOperand(1);
3451
3452 // Nontemporal (unaligned) store of the 0'th element of the float/double
3453 // vector.
3454 Value *Extract =
3455 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3456
3457 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3458 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3459 } else if (Name.starts_with("avx.movnt.") ||
3460 Name.starts_with("avx512.storent.")) {
3461 SmallVector<Metadata *, 1> Elts;
3462 Elts.push_back(
3463 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3464 MDNode *Node = MDNode::get(C, Elts);
3465
3466 Value *Arg0 = CI->getArgOperand(0);
3467 Value *Arg1 = CI->getArgOperand(1);
3468
3469 StoreInst *SI = Builder.CreateAlignedStore(
3470 Arg1, Arg0,
3471 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3472 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3473 } else if (Name == "sse2.storel.dq") {
3474 Value *Arg0 = CI->getArgOperand(0);
3475 Value *Arg1 = CI->getArgOperand(1);
3476
3477 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3478 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3479 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3480 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3481 } else if (Name.starts_with("sse.storeu.") ||
3482 Name.starts_with("sse2.storeu.") ||
3483 Name.starts_with("avx.storeu.")) {
3484 Value *Arg0 = CI->getArgOperand(0);
3485 Value *Arg1 = CI->getArgOperand(1);
3486 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3487 } else if (Name == "avx512.mask.store.ss") {
3488 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3489 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3490 Mask, false);
3491 } else if (Name.starts_with("avx512.mask.store")) {
3492 // "avx512.mask.storeu." or "avx512.mask.store."
3493 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3494 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3495 CI->getArgOperand(2), Aligned);
3496 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3497 // Upgrade packed integer vector compare intrinsics to compare instructions.
3498 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3499 bool CmpEq = Name[9] == 'e';
3500 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3501 CI->getArgOperand(0), CI->getArgOperand(1));
3502 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3503 } else if (Name.starts_with("avx512.broadcastm")) {
3504 Type *ExtTy = Type::getInt32Ty(C);
3505 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3506 ExtTy = Type::getInt64Ty(C);
3507 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3508 ExtTy->getPrimitiveSizeInBits();
3509 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3510 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3511 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3512 Value *Vec = CI->getArgOperand(0);
3513 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3514 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3515 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3516 } else if (Name.starts_with("avx.sqrt.p") ||
3517 Name.starts_with("sse2.sqrt.p") ||
3518 Name.starts_with("sse.sqrt.p")) {
3519 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3520 {CI->getArgOperand(0)});
3521 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3522 if (CI->arg_size() == 4 &&
3523 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3524 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3525 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3526 : Intrinsic::x86_avx512_sqrt_pd_512;
3527
3528 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3529 Rep = Builder.CreateIntrinsic(IID, Args);
3530 } else {
3531 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3532 {CI->getArgOperand(0)});
3533 }
3534 Rep =
3535 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3536 } else if (Name.starts_with("avx512.ptestm") ||
3537 Name.starts_with("avx512.ptestnm")) {
3538 Value *Op0 = CI->getArgOperand(0);
3539 Value *Op1 = CI->getArgOperand(1);
3540 Value *Mask = CI->getArgOperand(2);
3541 Rep = Builder.CreateAnd(Op0, Op1);
3542 llvm::Type *Ty = Op0->getType();
3543 Value *Zero = llvm::Constant::getNullValue(Ty);
3544 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3545 ? ICmpInst::ICMP_NE
3546 : ICmpInst::ICMP_EQ;
3547 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3548 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3549 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3550 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3551 ->getNumElements();
3552 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3553 Rep =
3554 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3555 } else if (Name.starts_with("avx512.kunpck")) {
3556 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3557 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3558 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3559 int Indices[64];
3560 for (unsigned i = 0; i != NumElts; ++i)
3561 Indices[i] = i;
3562
3563 // First extract half of each vector. This gives better codegen than
3564 // doing it in a single shuffle.
3565 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3566 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3567 // Concat the vectors.
3568 // NOTE: Operands have to be swapped to match intrinsic definition.
3569 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3570 Rep = Builder.CreateBitCast(Rep, CI->getType());
3571 } else if (Name == "avx512.kand.w") {
3572 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3573 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3574 Rep = Builder.CreateAnd(LHS, RHS);
3575 Rep = Builder.CreateBitCast(Rep, CI->getType());
3576 } else if (Name == "avx512.kandn.w") {
3577 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3578 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3579 LHS = Builder.CreateNot(LHS);
3580 Rep = Builder.CreateAnd(LHS, RHS);
3581 Rep = Builder.CreateBitCast(Rep, CI->getType());
3582 } else if (Name == "avx512.kor.w") {
3583 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3584 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3585 Rep = Builder.CreateOr(LHS, RHS);
3586 Rep = Builder.CreateBitCast(Rep, CI->getType());
3587 } else if (Name == "avx512.kxor.w") {
3588 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3589 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3590 Rep = Builder.CreateXor(LHS, RHS);
3591 Rep = Builder.CreateBitCast(Rep, CI->getType());
3592 } else if (Name == "avx512.kxnor.w") {
3593 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3594 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3595 LHS = Builder.CreateNot(LHS);
3596 Rep = Builder.CreateXor(LHS, RHS);
3597 Rep = Builder.CreateBitCast(Rep, CI->getType());
3598 } else if (Name == "avx512.knot.w") {
3599 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3600 Rep = Builder.CreateNot(Rep);
3601 Rep = Builder.CreateBitCast(Rep, CI->getType());
3602 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3603 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3604 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3605 Rep = Builder.CreateOr(LHS, RHS);
3606 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3607 Value *C;
3608 if (Name[14] == 'c')
3609 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3610 else
3611 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3612 Rep = Builder.CreateICmpEQ(Rep, C);
3613 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3614 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3615 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3616 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3617 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3618 Type *I32Ty = Type::getInt32Ty(C);
3619 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3620 ConstantInt::get(I32Ty, 0));
3621 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3622 ConstantInt::get(I32Ty, 0));
3623 Value *EltOp;
3624 if (Name.contains(".add."))
3625 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3626 else if (Name.contains(".sub."))
3627 EltOp = Builder.CreateFSub(Elt0, Elt1);
3628 else if (Name.contains(".mul."))
3629 EltOp = Builder.CreateFMul(Elt0, Elt1);
3630 else
3631 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3632 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3633 ConstantInt::get(I32Ty, 0));
3634 } else if (Name.starts_with("avx512.mask.pcmp")) {
3635 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3636 bool CmpEq = Name[16] == 'e';
3637 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3638 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3639 Type *OpTy = CI->getArgOperand(0)->getType();
3640 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3641 Intrinsic::ID IID;
3642 switch (VecWidth) {
3643 default:
3644 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3645 break;
3646 case 128:
3647 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3648 break;
3649 case 256:
3650 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3651 break;
3652 case 512:
3653 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3654 break;
3655 }
3656
3657 Rep =
3658 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3659 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3660 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3661 Type *OpTy = CI->getArgOperand(0)->getType();
3662 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3663 unsigned EltWidth = OpTy->getScalarSizeInBits();
3664 Intrinsic::ID IID;
3665 if (VecWidth == 128 && EltWidth == 32)
3666 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3667 else if (VecWidth == 256 && EltWidth == 32)
3668 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3669 else if (VecWidth == 512 && EltWidth == 32)
3670 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3671 else if (VecWidth == 128 && EltWidth == 64)
3672 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3673 else if (VecWidth == 256 && EltWidth == 64)
3674 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3675 else if (VecWidth == 512 && EltWidth == 64)
3676 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3677 else
3678 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3679
3680 Rep =
3681 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3682 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3683 } else if (Name.starts_with("avx512.cmp.p")) {
3684 SmallVector<Value *, 4> Args(CI->args());
3685 Type *OpTy = Args[0]->getType();
3686 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3687 unsigned EltWidth = OpTy->getScalarSizeInBits();
3688 Intrinsic::ID IID;
3689 if (VecWidth == 128 && EltWidth == 32)
3690 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3691 else if (VecWidth == 256 && EltWidth == 32)
3692 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3693 else if (VecWidth == 512 && EltWidth == 32)
3694 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3695 else if (VecWidth == 128 && EltWidth == 64)
3696 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3697 else if (VecWidth == 256 && EltWidth == 64)
3698 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3699 else if (VecWidth == 512 && EltWidth == 64)
3700 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3701 else
3702 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3703
3704 Value *Mask = Constant::getAllOnesValue(CI->getType());
3705 if (VecWidth == 512)
3706 std::swap(Mask, Args.back());
3707 Args.push_back(Mask);
3708
3709 Rep = Builder.CreateIntrinsic(IID, Args);
3710 } else if (Name.starts_with("avx512.mask.cmp.")) {
3711 // Integer compare intrinsics.
3712 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3713 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3714 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3715 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3716 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3717 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3718 Name.starts_with("avx512.cvtw2mask.") ||
3719 Name.starts_with("avx512.cvtd2mask.") ||
3720 Name.starts_with("avx512.cvtq2mask.")) {
3721 Value *Op = CI->getArgOperand(0);
3722 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3723 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3724 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3725 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3726 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3727 Name.starts_with("avx512.mask.pabs")) {
3728 Rep = upgradeAbs(Builder, *CI);
3729 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3730 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3731 Name.starts_with("avx512.mask.pmaxs")) {
3732 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3733 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3734 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3735 Name.starts_with("avx512.mask.pmaxu")) {
3736 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3737 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3738 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3739 Name.starts_with("avx512.mask.pmins")) {
3740 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3741 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3742 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3743 Name.starts_with("avx512.mask.pminu")) {
3744 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3745 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3746 Name == "avx512.pmulu.dq.512" ||
3747 Name.starts_with("avx512.mask.pmulu.dq.")) {
3748 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3749 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3750 Name == "avx512.pmul.dq.512" ||
3751 Name.starts_with("avx512.mask.pmul.dq.")) {
3752 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3753 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3754 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3755 Rep =
3756 Builder.CreateSIToFP(CI->getArgOperand(1),
3757 cast<VectorType>(CI->getType())->getElementType());
3758 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3759 } else if (Name == "avx512.cvtusi2sd") {
3760 Rep =
3761 Builder.CreateUIToFP(CI->getArgOperand(1),
3762 cast<VectorType>(CI->getType())->getElementType());
3763 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3764 } else if (Name == "sse2.cvtss2sd") {
3765 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3766 Rep = Builder.CreateFPExt(
3767 Rep, cast<VectorType>(CI->getType())->getElementType());
3768 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3769 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3770 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3771 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3772 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3773 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3774 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3775 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3776 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3777 Name == "avx512.mask.cvtqq2ps.256" ||
3778 Name == "avx512.mask.cvtqq2ps.512" ||
3779 Name == "avx512.mask.cvtuqq2ps.256" ||
3780 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3781 Name == "avx.cvt.ps2.pd.256" ||
3782 Name == "avx512.mask.cvtps2pd.128" ||
3783 Name == "avx512.mask.cvtps2pd.256") {
3784 auto *DstTy = cast<FixedVectorType>(CI->getType());
3785 Rep = CI->getArgOperand(0);
3786 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3787
3788 unsigned NumDstElts = DstTy->getNumElements();
3789 if (NumDstElts < SrcTy->getNumElements()) {
3790 assert(NumDstElts == 2 && "Unexpected vector size");
3791 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3792 }
3793
3794 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3795 bool IsUnsigned = Name.contains("cvtu");
3796 if (IsPS2PD)
3797 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3798 else if (CI->arg_size() == 4 &&
3799 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3800 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3801 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3802 : Intrinsic::x86_avx512_sitofp_round;
3803 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3804 {Rep, CI->getArgOperand(3)});
3805 } else {
3806 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3807 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3808 }
3809
3810 if (CI->arg_size() >= 3)
3811 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3812 CI->getArgOperand(1));
3813 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3814 Name.starts_with("vcvtph2ps.")) {
3815 auto *DstTy = cast<FixedVectorType>(CI->getType());
3816 Rep = CI->getArgOperand(0);
3817 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3818 unsigned NumDstElts = DstTy->getNumElements();
3819 if (NumDstElts != SrcTy->getNumElements()) {
3820 assert(NumDstElts == 4 && "Unexpected vector size");
3821 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3822 }
3823 Rep = Builder.CreateBitCast(
3824 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3825 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3826 if (CI->arg_size() >= 3)
3827 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3828 CI->getArgOperand(1));
3829 } else if (Name.starts_with("avx512.mask.load")) {
3830 // "avx512.mask.loadu." or "avx512.mask.load."
3831 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3832 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3833 CI->getArgOperand(2), Aligned);
3834 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3835 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3836 auto *PtrTy = CI->getOperand(0)->getType();
3837 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3838 ResultTy->getNumElements());
3839 Rep = Builder.CreateIntrinsic(
3840 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3841 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3842 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3843 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3844 auto *PtrTy = CI->getArgOperand(0)->getType();
3845 Value *MaskVec =
3846 getX86MaskVec(Builder, CI->getArgOperand(2),
3847 cast<FixedVectorType>(ResultTy)->getNumElements());
3848 Rep = Builder.CreateIntrinsic(
3849 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3850 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3851 } else if (Name.starts_with("avx512.mask.compress.") ||
3852 Name.starts_with("avx512.mask.expand.")) {
3853 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3854
3855 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3856 ResultTy->getNumElements());
3857
3858 bool IsCompress = Name[12] == 'c';
3859 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3860 : Intrinsic::x86_avx512_mask_expand;
3861 Rep = Builder.CreateIntrinsic(
3862 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3863 } else if (Name.starts_with("xop.vpcom")) {
3864 bool IsSigned;
3865 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3866 Name.ends_with("uq"))
3867 IsSigned = false;
3868 else if (Name.ends_with("b") || Name.ends_with("w") ||
3869 Name.ends_with("d") || Name.ends_with("q"))
3870 IsSigned = true;
3871 else
3872 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3873
3874 unsigned Imm;
3875 if (CI->arg_size() == 3) {
3876 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3877 } else {
3878 Name = Name.substr(9); // strip off "xop.vpcom"
3879 if (Name.starts_with("lt"))
3880 Imm = 0;
3881 else if (Name.starts_with("le"))
3882 Imm = 1;
3883 else if (Name.starts_with("gt"))
3884 Imm = 2;
3885 else if (Name.starts_with("ge"))
3886 Imm = 3;
3887 else if (Name.starts_with("eq"))
3888 Imm = 4;
3889 else if (Name.starts_with("ne"))
3890 Imm = 5;
3891 else if (Name.starts_with("false"))
3892 Imm = 6;
3893 else if (Name.starts_with("true"))
3894 Imm = 7;
3895 else
3896 llvm_unreachable("Unknown condition");
3897 }
3898
3899 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3900 } else if (Name.starts_with("xop.vpcmov")) {
3901 Value *Sel = CI->getArgOperand(2);
3902 Value *NotSel = Builder.CreateNot(Sel);
3903 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3904 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3905 Rep = Builder.CreateOr(Sel0, Sel1);
3906 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3907 Name.starts_with("avx512.mask.prol")) {
3908 Rep = upgradeX86Rotate(Builder, *CI, false);
3909 } else if (Name.starts_with("avx512.pror") ||
3910 Name.starts_with("avx512.mask.pror")) {
3911 Rep = upgradeX86Rotate(Builder, *CI, true);
3912 } else if (Name.starts_with("avx512.vpshld.") ||
3913 Name.starts_with("avx512.mask.vpshld") ||
3914 Name.starts_with("avx512.maskz.vpshld")) {
3915 bool ZeroMask = Name[11] == 'z';
3916 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3917 } else if (Name.starts_with("avx512.vpshrd.") ||
3918 Name.starts_with("avx512.mask.vpshrd") ||
3919 Name.starts_with("avx512.maskz.vpshrd")) {
3920 bool ZeroMask = Name[11] == 'z';
3921 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3922 } else if (Name == "sse42.crc32.64.8") {
3923 Value *Trunc0 =
3924 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3925 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3926 {Trunc0, CI->getArgOperand(1)});
3927 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3928 } else if (Name.starts_with("avx.vbroadcast.s") ||
3929 Name.starts_with("avx512.vbroadcast.s")) {
3930 // Replace broadcasts with a series of insertelements.
3931 auto *VecTy = cast<FixedVectorType>(CI->getType());
3932 Type *EltTy = VecTy->getElementType();
3933 unsigned EltNum = VecTy->getNumElements();
3934 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3935 Type *I32Ty = Type::getInt32Ty(C);
3936 Rep = PoisonValue::get(VecTy);
3937 for (unsigned I = 0; I < EltNum; ++I)
3938 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3939 } else if (Name.starts_with("sse41.pmovsx") ||
3940 Name.starts_with("sse41.pmovzx") ||
3941 Name.starts_with("avx2.pmovsx") ||
3942 Name.starts_with("avx2.pmovzx") ||
3943 Name.starts_with("avx512.mask.pmovsx") ||
3944 Name.starts_with("avx512.mask.pmovzx")) {
3945 auto *DstTy = cast<FixedVectorType>(CI->getType());
3946 unsigned NumDstElts = DstTy->getNumElements();
3947
3948 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3949 SmallVector<int, 8> ShuffleMask(NumDstElts);
3950 for (unsigned i = 0; i != NumDstElts; ++i)
3951 ShuffleMask[i] = i;
3952
3953 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3954
3955 bool DoSext = Name.contains("pmovsx");
3956 Rep =
3957 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3958 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3959 if (CI->arg_size() == 3)
3960 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3961 CI->getArgOperand(1));
3962 } else if (Name == "avx512.mask.pmov.qd.256" ||
3963 Name == "avx512.mask.pmov.qd.512" ||
3964 Name == "avx512.mask.pmov.wb.256" ||
3965 Name == "avx512.mask.pmov.wb.512") {
3966 Type *Ty = CI->getArgOperand(1)->getType();
3967 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3968 Rep =
3969 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3970 } else if (Name.starts_with("avx.vbroadcastf128") ||
3971 Name == "avx2.vbroadcasti128") {
3972 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3973 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3974 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3975 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3976 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3977 if (NumSrcElts == 2)
3978 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3979 else
3980 Rep = Builder.CreateShuffleVector(Load,
3981 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3982 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3983 Name.starts_with("avx512.mask.shuf.f")) {
3984 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3985 Type *VT = CI->getType();
3986 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3987 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3988 unsigned ControlBitsMask = NumLanes - 1;
3989 unsigned NumControlBits = NumLanes / 2;
3990 SmallVector<int, 8> ShuffleMask(0);
3991
3992 for (unsigned l = 0; l != NumLanes; ++l) {
3993 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3994 // We actually need the other source.
3995 if (l >= NumLanes / 2)
3996 LaneMask += NumLanes;
3997 for (unsigned i = 0; i != NumElementsInLane; ++i)
3998 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3999 }
4000 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4001 CI->getArgOperand(1), ShuffleMask);
4002 Rep =
4003 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4004 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4005 Name.starts_with("avx512.mask.broadcasti")) {
4006 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4007 ->getNumElements();
4008 unsigned NumDstElts =
4009 cast<FixedVectorType>(CI->getType())->getNumElements();
4010
4011 SmallVector<int, 8> ShuffleMask(NumDstElts);
4012 for (unsigned i = 0; i != NumDstElts; ++i)
4013 ShuffleMask[i] = i % NumSrcElts;
4014
4015 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4016 CI->getArgOperand(0), ShuffleMask);
4017 Rep =
4018 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4019 } else if (Name.starts_with("avx2.pbroadcast") ||
4020 Name.starts_with("avx2.vbroadcast") ||
4021 Name.starts_with("avx512.pbroadcast") ||
4022 Name.starts_with("avx512.mask.broadcast.s")) {
4023 // Replace vp?broadcasts with a vector shuffle.
4024 Value *Op = CI->getArgOperand(0);
4025 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4026 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4027 SmallVector<int, 8> M;
4028 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
4029 Rep = Builder.CreateShuffleVector(Op, M);
4030
4031 if (CI->arg_size() == 3)
4032 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4033 CI->getArgOperand(1));
4034 } else if (Name.starts_with("sse2.padds.") ||
4035 Name.starts_with("avx2.padds.") ||
4036 Name.starts_with("avx512.padds.") ||
4037 Name.starts_with("avx512.mask.padds.")) {
4038 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4039 } else if (Name.starts_with("sse2.psubs.") ||
4040 Name.starts_with("avx2.psubs.") ||
4041 Name.starts_with("avx512.psubs.") ||
4042 Name.starts_with("avx512.mask.psubs.")) {
4043 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4044 } else if (Name.starts_with("sse2.paddus.") ||
4045 Name.starts_with("avx2.paddus.") ||
4046 Name.starts_with("avx512.mask.paddus.")) {
4047 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4048 } else if (Name.starts_with("sse2.psubus.") ||
4049 Name.starts_with("avx2.psubus.") ||
4050 Name.starts_with("avx512.mask.psubus.")) {
4051 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4052 } else if (Name.starts_with("avx512.mask.palignr.")) {
4053 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4054 CI->getArgOperand(1), CI->getArgOperand(2),
4055 CI->getArgOperand(3), CI->getArgOperand(4),
4056 false);
4057 } else if (Name.starts_with("avx512.mask.valign.")) {
4058 Rep = upgradeX86ALIGNIntrinsics(
4059 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4060 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4061 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4062 // 128/256-bit shift left specified in bits.
4063 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4064 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4065 Shift / 8); // Shift is in bits.
4066 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4067 // 128/256-bit shift right specified in bits.
4068 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4069 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4070 Shift / 8); // Shift is in bits.
4071 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4072 Name == "avx512.psll.dq.512") {
4073 // 128/256/512-bit shift left specified in bytes.
4074 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4075 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4076 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4077 Name == "avx512.psrl.dq.512") {
4078 // 128/256/512-bit shift right specified in bytes.
4079 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4080 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4081 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4082 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4083 Name.starts_with("avx2.pblendd.")) {
4084 Value *Op0 = CI->getArgOperand(0);
4085 Value *Op1 = CI->getArgOperand(1);
4086 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4087 auto *VecTy = cast<FixedVectorType>(CI->getType());
4088 unsigned NumElts = VecTy->getNumElements();
4089
4090 SmallVector<int, 16> Idxs(NumElts);
4091 for (unsigned i = 0; i != NumElts; ++i)
4092 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4093
4094 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4095 } else if (Name.starts_with("avx.vinsertf128.") ||
4096 Name == "avx2.vinserti128" ||
4097 Name.starts_with("avx512.mask.insert")) {
4098 Value *Op0 = CI->getArgOperand(0);
4099 Value *Op1 = CI->getArgOperand(1);
4100 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4101 unsigned DstNumElts =
4102 cast<FixedVectorType>(CI->getType())->getNumElements();
4103 unsigned SrcNumElts =
4104 cast<FixedVectorType>(Op1->getType())->getNumElements();
4105 unsigned Scale = DstNumElts / SrcNumElts;
4106
4107 // Mask off the high bits of the immediate value; hardware ignores those.
4108 Imm = Imm % Scale;
4109
4110 // Extend the second operand into a vector the size of the destination.
4111 SmallVector<int, 8> Idxs(DstNumElts);
4112 for (unsigned i = 0; i != SrcNumElts; ++i)
4113 Idxs[i] = i;
4114 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4115 Idxs[i] = SrcNumElts;
4116 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4117
4118 // Insert the second operand into the first operand.
4119
4120 // Note that there is no guarantee that instruction lowering will actually
4121 // produce a vinsertf128 instruction for the created shuffles. In
4122 // particular, the 0 immediate case involves no lane changes, so it can
4123 // be handled as a blend.
4124
4125 // Example of shuffle mask for 32-bit elements:
4126 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4127 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4128
4129 // First fill with identify mask.
4130 for (unsigned i = 0; i != DstNumElts; ++i)
4131 Idxs[i] = i;
4132 // Then replace the elements where we need to insert.
4133 for (unsigned i = 0; i != SrcNumElts; ++i)
4134 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4135 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4136
4137 // If the intrinsic has a mask operand, handle that.
4138 if (CI->arg_size() == 5)
4139 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4140 CI->getArgOperand(3));
4141 } else if (Name.starts_with("avx.vextractf128.") ||
4142 Name == "avx2.vextracti128" ||
4143 Name.starts_with("avx512.mask.vextract")) {
4144 Value *Op0 = CI->getArgOperand(0);
4145 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4146 unsigned DstNumElts =
4147 cast<FixedVectorType>(CI->getType())->getNumElements();
4148 unsigned SrcNumElts =
4149 cast<FixedVectorType>(Op0->getType())->getNumElements();
4150 unsigned Scale = SrcNumElts / DstNumElts;
4151
4152 // Mask off the high bits of the immediate value; hardware ignores those.
4153 Imm = Imm % Scale;
4154
4155 // Get indexes for the subvector of the input vector.
4156 SmallVector<int, 8> Idxs(DstNumElts);
4157 for (unsigned i = 0; i != DstNumElts; ++i) {
4158 Idxs[i] = i + (Imm * DstNumElts);
4159 }
4160 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4161
4162 // If the intrinsic has a mask operand, handle that.
4163 if (CI->arg_size() == 4)
4164 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4165 CI->getArgOperand(2));
4166 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4167 Name.starts_with("avx512.mask.perm.di.")) {
4168 Value *Op0 = CI->getArgOperand(0);
4169 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4170 auto *VecTy = cast<FixedVectorType>(CI->getType());
4171 unsigned NumElts = VecTy->getNumElements();
4172
4173 SmallVector<int, 8> Idxs(NumElts);
4174 for (unsigned i = 0; i != NumElts; ++i)
4175 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4176
4177 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4178
4179 if (CI->arg_size() == 4)
4180 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4181 CI->getArgOperand(2));
4182 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4183 // The immediate permute control byte looks like this:
4184 // [1:0] - select 128 bits from sources for low half of destination
4185 // [2] - ignore
4186 // [3] - zero low half of destination
4187 // [5:4] - select 128 bits from sources for high half of destination
4188 // [6] - ignore
4189 // [7] - zero high half of destination
4190
4191 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4192
4193 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4194 unsigned HalfSize = NumElts / 2;
4195 SmallVector<int, 8> ShuffleMask(NumElts);
4196
4197 // Determine which operand(s) are actually in use for this instruction.
4198 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4199 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4200
4201 // If needed, replace operands based on zero mask.
4202 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4203 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4204
4205 // Permute low half of result.
4206 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4207 for (unsigned i = 0; i < HalfSize; ++i)
4208 ShuffleMask[i] = StartIndex + i;
4209
4210 // Permute high half of result.
4211 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4212 for (unsigned i = 0; i < HalfSize; ++i)
4213 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4214
4215 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4216
4217 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4218 Name.starts_with("avx512.mask.vpermil.p") ||
4219 Name.starts_with("avx512.mask.pshuf.d.")) {
4220 Value *Op0 = CI->getArgOperand(0);
4221 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4222 auto *VecTy = cast<FixedVectorType>(CI->getType());
4223 unsigned NumElts = VecTy->getNumElements();
4224 // Calculate the size of each index in the immediate.
4225 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4226 unsigned IdxMask = ((1 << IdxSize) - 1);
4227
4228 SmallVector<int, 8> Idxs(NumElts);
4229 // Lookup the bits for this element, wrapping around the immediate every
4230 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4231 // to offset by the first index of each group.
4232 for (unsigned i = 0; i != NumElts; ++i)
4233 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4234
4235 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4236
4237 if (CI->arg_size() == 4)
4238 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4239 CI->getArgOperand(2));
4240 } else if (Name == "sse2.pshufl.w" ||
4241 Name.starts_with("avx512.mask.pshufl.w.")) {
4242 Value *Op0 = CI->getArgOperand(0);
4243 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4244 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4245
4246 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4247 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4248
4249 SmallVector<int, 16> Idxs(NumElts);
4250 for (unsigned l = 0; l != NumElts; l += 8) {
4251 for (unsigned i = 0; i != 4; ++i)
4252 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4253 for (unsigned i = 4; i != 8; ++i)
4254 Idxs[i + l] = i + l;
4255 }
4256
4257 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4258
4259 if (CI->arg_size() == 4)
4260 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4261 CI->getArgOperand(2));
4262 } else if (Name == "sse2.pshufh.w" ||
4263 Name.starts_with("avx512.mask.pshufh.w.")) {
4264 Value *Op0 = CI->getArgOperand(0);
4265 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4266 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4267
4268 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4269 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4270
4271 SmallVector<int, 16> Idxs(NumElts);
4272 for (unsigned l = 0; l != NumElts; l += 8) {
4273 for (unsigned i = 0; i != 4; ++i)
4274 Idxs[i + l] = i + l;
4275 for (unsigned i = 0; i != 4; ++i)
4276 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4277 }
4278
4279 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4280
4281 if (CI->arg_size() == 4)
4282 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4283 CI->getArgOperand(2));
4284 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4285 Value *Op0 = CI->getArgOperand(0);
4286 Value *Op1 = CI->getArgOperand(1);
4287 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4288 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4289
4290 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4291 unsigned HalfLaneElts = NumLaneElts / 2;
4292
4293 SmallVector<int, 16> Idxs(NumElts);
4294 for (unsigned i = 0; i != NumElts; ++i) {
4295 // Base index is the starting element of the lane.
4296 Idxs[i] = i - (i % NumLaneElts);
4297 // If we are half way through the lane switch to the other source.
4298 if ((i % NumLaneElts) >= HalfLaneElts)
4299 Idxs[i] += NumElts;
4300 // Now select the specific element. By adding HalfLaneElts bits from
4301 // the immediate. Wrapping around the immediate every 8-bits.
4302 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4303 }
4304
4305 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4306
4307 Rep =
4308 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4309 } else if (Name.starts_with("avx512.mask.movddup") ||
4310 Name.starts_with("avx512.mask.movshdup") ||
4311 Name.starts_with("avx512.mask.movsldup")) {
4312 Value *Op0 = CI->getArgOperand(0);
4313 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4314 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4315
4316 unsigned Offset = 0;
4317 if (Name.starts_with("avx512.mask.movshdup."))
4318 Offset = 1;
4319
4320 SmallVector<int, 16> Idxs(NumElts);
4321 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4322 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4323 Idxs[i + l + 0] = i + l + Offset;
4324 Idxs[i + l + 1] = i + l + Offset;
4325 }
4326
4327 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4328
4329 Rep =
4330 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4331 } else if (Name.starts_with("avx512.mask.punpckl") ||
4332 Name.starts_with("avx512.mask.unpckl.")) {
4333 Value *Op0 = CI->getArgOperand(0);
4334 Value *Op1 = CI->getArgOperand(1);
4335 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4336 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4337
4338 SmallVector<int, 64> Idxs(NumElts);
4339 for (int l = 0; l != NumElts; l += NumLaneElts)
4340 for (int i = 0; i != NumLaneElts; ++i)
4341 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4342
4343 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4344
4345 Rep =
4346 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4347 } else if (Name.starts_with("avx512.mask.punpckh") ||
4348 Name.starts_with("avx512.mask.unpckh.")) {
4349 Value *Op0 = CI->getArgOperand(0);
4350 Value *Op1 = CI->getArgOperand(1);
4351 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4352 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4353
4354 SmallVector<int, 64> Idxs(NumElts);
4355 for (int l = 0; l != NumElts; l += NumLaneElts)
4356 for (int i = 0; i != NumLaneElts; ++i)
4357 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4358
4359 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4360
4361 Rep =
4362 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4363 } else if (Name.starts_with("avx512.mask.and.") ||
4364 Name.starts_with("avx512.mask.pand.")) {
4365 VectorType *FTy = cast<VectorType>(CI->getType());
4366 VectorType *ITy = VectorType::getInteger(FTy);
4367 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4368 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4369 Rep = Builder.CreateBitCast(Rep, FTy);
4370 Rep =
4371 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4372 } else if (Name.starts_with("avx512.mask.andn.") ||
4373 Name.starts_with("avx512.mask.pandn.")) {
4374 VectorType *FTy = cast<VectorType>(CI->getType());
4375 VectorType *ITy = VectorType::getInteger(FTy);
4376 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4377 Rep = Builder.CreateAnd(Rep,
4378 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4379 Rep = Builder.CreateBitCast(Rep, FTy);
4380 Rep =
4381 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4382 } else if (Name.starts_with("avx512.mask.or.") ||
4383 Name.starts_with("avx512.mask.por.")) {
4384 VectorType *FTy = cast<VectorType>(CI->getType());
4385 VectorType *ITy = VectorType::getInteger(FTy);
4386 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4387 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4388 Rep = Builder.CreateBitCast(Rep, FTy);
4389 Rep =
4390 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4391 } else if (Name.starts_with("avx512.mask.xor.") ||
4392 Name.starts_with("avx512.mask.pxor.")) {
4393 VectorType *FTy = cast<VectorType>(CI->getType());
4394 VectorType *ITy = VectorType::getInteger(FTy);
4395 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4396 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4397 Rep = Builder.CreateBitCast(Rep, FTy);
4398 Rep =
4399 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4400 } else if (Name.starts_with("avx512.mask.padd.")) {
4401 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4402 Rep =
4403 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4404 } else if (Name.starts_with("avx512.mask.psub.")) {
4405 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4406 Rep =
4407 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4408 } else if (Name.starts_with("avx512.mask.pmull.")) {
4409 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4410 Rep =
4411 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4412 } else if (Name.starts_with("avx512.mask.add.p")) {
4413 if (Name.ends_with(".512")) {
4414 Intrinsic::ID IID;
4415 if (Name[17] == 's')
4416 IID = Intrinsic::x86_avx512_add_ps_512;
4417 else
4418 IID = Intrinsic::x86_avx512_add_pd_512;
4419
4420 Rep = Builder.CreateIntrinsic(
4421 IID,
4422 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4423 } else {
4424 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4425 }
4426 Rep =
4427 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4428 } else if (Name.starts_with("avx512.mask.div.p")) {
4429 if (Name.ends_with(".512")) {
4430 Intrinsic::ID IID;
4431 if (Name[17] == 's')
4432 IID = Intrinsic::x86_avx512_div_ps_512;
4433 else
4434 IID = Intrinsic::x86_avx512_div_pd_512;
4435
4436 Rep = Builder.CreateIntrinsic(
4437 IID,
4438 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4439 } else {
4440 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4441 }
4442 Rep =
4443 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4444 } else if (Name.starts_with("avx512.mask.mul.p")) {
4445 if (Name.ends_with(".512")) {
4446 Intrinsic::ID IID;
4447 if (Name[17] == 's')
4448 IID = Intrinsic::x86_avx512_mul_ps_512;
4449 else
4450 IID = Intrinsic::x86_avx512_mul_pd_512;
4451
4452 Rep = Builder.CreateIntrinsic(
4453 IID,
4454 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4455 } else {
4456 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4457 }
4458 Rep =
4459 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4460 } else if (Name.starts_with("avx512.mask.sub.p")) {
4461 if (Name.ends_with(".512")) {
4462 Intrinsic::ID IID;
4463 if (Name[17] == 's')
4464 IID = Intrinsic::x86_avx512_sub_ps_512;
4465 else
4466 IID = Intrinsic::x86_avx512_sub_pd_512;
4467
4468 Rep = Builder.CreateIntrinsic(
4469 IID,
4470 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4471 } else {
4472 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4473 }
4474 Rep =
4475 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4476 } else if ((Name.starts_with("avx512.mask.max.p") ||
4477 Name.starts_with("avx512.mask.min.p")) &&
4478 Name.drop_front(18) == ".512") {
4479 bool IsDouble = Name[17] == 'd';
4480 bool IsMin = Name[13] == 'i';
4481 static const Intrinsic::ID MinMaxTbl[2][2] = {
4482 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4483 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4484 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4485
4486 Rep = Builder.CreateIntrinsic(
4487 IID,
4488 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4489 Rep =
4490 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4491 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4492 Rep =
4493 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4494 {CI->getArgOperand(0), Builder.getInt1(false)});
4495 Rep =
4496 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4497 } else if (Name.starts_with("avx512.mask.psll")) {
4498 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4499 bool IsVariable = Name[16] == 'v';
4500 char Size = Name[16] == '.' ? Name[17]
4501 : Name[17] == '.' ? Name[18]
4502 : Name[18] == '.' ? Name[19]
4503 : Name[20];
4504
4505 Intrinsic::ID IID;
4506 if (IsVariable && Name[17] != '.') {
4507 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4508 IID = Intrinsic::x86_avx2_psllv_q;
4509 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4510 IID = Intrinsic::x86_avx2_psllv_q_256;
4511 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4512 IID = Intrinsic::x86_avx2_psllv_d;
4513 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4514 IID = Intrinsic::x86_avx2_psllv_d_256;
4515 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4516 IID = Intrinsic::x86_avx512_psllv_w_128;
4517 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4518 IID = Intrinsic::x86_avx512_psllv_w_256;
4519 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4520 IID = Intrinsic::x86_avx512_psllv_w_512;
4521 else
4522 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4523 } else if (Name.ends_with(".128")) {
4524 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4525 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4526 : Intrinsic::x86_sse2_psll_d;
4527 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4528 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4529 : Intrinsic::x86_sse2_psll_q;
4530 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4531 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4532 : Intrinsic::x86_sse2_psll_w;
4533 else
4534 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4535 } else if (Name.ends_with(".256")) {
4536 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4537 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4538 : Intrinsic::x86_avx2_psll_d;
4539 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4540 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4541 : Intrinsic::x86_avx2_psll_q;
4542 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4543 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4544 : Intrinsic::x86_avx2_psll_w;
4545 else
4546 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4547 } else {
4548 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4549 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4550 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4551 : Intrinsic::x86_avx512_psll_d_512;
4552 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4553 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4554 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4555 : Intrinsic::x86_avx512_psll_q_512;
4556 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4557 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4558 : Intrinsic::x86_avx512_psll_w_512;
4559 else
4560 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4561 }
4562
4563 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4564 } else if (Name.starts_with("avx512.mask.psrl")) {
4565 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4566 bool IsVariable = Name[16] == 'v';
4567 char Size = Name[16] == '.' ? Name[17]
4568 : Name[17] == '.' ? Name[18]
4569 : Name[18] == '.' ? Name[19]
4570 : Name[20];
4571
4572 Intrinsic::ID IID;
4573 if (IsVariable && Name[17] != '.') {
4574 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4575 IID = Intrinsic::x86_avx2_psrlv_q;
4576 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4577 IID = Intrinsic::x86_avx2_psrlv_q_256;
4578 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4579 IID = Intrinsic::x86_avx2_psrlv_d;
4580 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4581 IID = Intrinsic::x86_avx2_psrlv_d_256;
4582 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4583 IID = Intrinsic::x86_avx512_psrlv_w_128;
4584 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4585 IID = Intrinsic::x86_avx512_psrlv_w_256;
4586 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4587 IID = Intrinsic::x86_avx512_psrlv_w_512;
4588 else
4589 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4590 } else if (Name.ends_with(".128")) {
4591 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4592 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4593 : Intrinsic::x86_sse2_psrl_d;
4594 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4595 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4596 : Intrinsic::x86_sse2_psrl_q;
4597 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4598 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4599 : Intrinsic::x86_sse2_psrl_w;
4600 else
4601 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4602 } else if (Name.ends_with(".256")) {
4603 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4604 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4605 : Intrinsic::x86_avx2_psrl_d;
4606 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4607 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4608 : Intrinsic::x86_avx2_psrl_q;
4609 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4610 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4611 : Intrinsic::x86_avx2_psrl_w;
4612 else
4613 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4614 } else {
4615 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4616 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4617 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4618 : Intrinsic::x86_avx512_psrl_d_512;
4619 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4620 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4621 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4622 : Intrinsic::x86_avx512_psrl_q_512;
4623 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4624 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4625 : Intrinsic::x86_avx512_psrl_w_512;
4626 else
4627 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4628 }
4629
4630 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4631 } else if (Name.starts_with("avx512.mask.psra")) {
4632 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4633 bool IsVariable = Name[16] == 'v';
4634 char Size = Name[16] == '.' ? Name[17]
4635 : Name[17] == '.' ? Name[18]
4636 : Name[18] == '.' ? Name[19]
4637 : Name[20];
4638
4639 Intrinsic::ID IID;
4640 if (IsVariable && Name[17] != '.') {
4641 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4642 IID = Intrinsic::x86_avx2_psrav_d;
4643 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4644 IID = Intrinsic::x86_avx2_psrav_d_256;
4645 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4646 IID = Intrinsic::x86_avx512_psrav_w_128;
4647 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4648 IID = Intrinsic::x86_avx512_psrav_w_256;
4649 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4650 IID = Intrinsic::x86_avx512_psrav_w_512;
4651 else
4652 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4653 } else if (Name.ends_with(".128")) {
4654 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4655 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4656 : Intrinsic::x86_sse2_psra_d;
4657 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4658 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4659 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4660 : Intrinsic::x86_avx512_psra_q_128;
4661 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4662 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4663 : Intrinsic::x86_sse2_psra_w;
4664 else
4665 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4666 } else if (Name.ends_with(".256")) {
4667 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4668 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4669 : Intrinsic::x86_avx2_psra_d;
4670 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4671 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4672 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4673 : Intrinsic::x86_avx512_psra_q_256;
4674 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4675 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4676 : Intrinsic::x86_avx2_psra_w;
4677 else
4678 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4679 } else {
4680 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4681 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4682 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4683 : Intrinsic::x86_avx512_psra_d_512;
4684 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4685 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4686 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4687 : Intrinsic::x86_avx512_psra_q_512;
4688 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4689 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4690 : Intrinsic::x86_avx512_psra_w_512;
4691 else
4692 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4693 }
4694
4695 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4696 } else if (Name.starts_with("avx512.mask.move.s")) {
4697 Rep = upgradeMaskedMove(Builder, *CI);
4698 } else if (Name.starts_with("avx512.cvtmask2")) {
4699 Rep = upgradeMaskToInt(Builder, *CI);
4700 } else if (Name.ends_with(".movntdqa")) {
4701 MDNode *Node = MDNode::get(
4702 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4703
4704 LoadInst *LI = Builder.CreateAlignedLoad(
4705 CI->getType(), CI->getArgOperand(0),
4706 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4707 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4708 Rep = LI;
4709 } else if (Name.starts_with("fma.vfmadd.") ||
4710 Name.starts_with("fma.vfmsub.") ||
4711 Name.starts_with("fma.vfnmadd.") ||
4712 Name.starts_with("fma.vfnmsub.")) {
4713 bool NegMul = Name[6] == 'n';
4714 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4715 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4716
4717 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4718 CI->getArgOperand(2)};
4719
4720 if (IsScalar) {
4721 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4722 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4723 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4724 }
4725
4726 if (NegMul && !IsScalar)
4727 Ops[0] = Builder.CreateFNeg(Ops[0]);
4728 if (NegMul && IsScalar)
4729 Ops[1] = Builder.CreateFNeg(Ops[1]);
4730 if (NegAcc)
4731 Ops[2] = Builder.CreateFNeg(Ops[2]);
4732
4733 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4734
4735 if (IsScalar)
4736 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4737 } else if (Name.starts_with("fma4.vfmadd.s")) {
4738 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4739 CI->getArgOperand(2)};
4740
4741 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4742 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4743 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4744
4745 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4746
4747 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4748 Rep, (uint64_t)0);
4749 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4750 Name.starts_with("avx512.maskz.vfmadd.s") ||
4751 Name.starts_with("avx512.mask3.vfmadd.s") ||
4752 Name.starts_with("avx512.mask3.vfmsub.s") ||
4753 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4754 bool IsMask3 = Name[11] == '3';
4755 bool IsMaskZ = Name[11] == 'z';
4756 // Drop the "avx512.mask." to make it easier.
4757 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4758 bool NegMul = Name[2] == 'n';
4759 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4760
4761 Value *A = CI->getArgOperand(0);
4762 Value *B = CI->getArgOperand(1);
4763 Value *C = CI->getArgOperand(2);
4764
4765 if (NegMul && (IsMask3 || IsMaskZ))
4766 A = Builder.CreateFNeg(A);
4767 if (NegMul && !(IsMask3 || IsMaskZ))
4768 B = Builder.CreateFNeg(B);
4769 if (NegAcc)
4770 C = Builder.CreateFNeg(C);
4771
4772 A = Builder.CreateExtractElement(A, (uint64_t)0);
4773 B = Builder.CreateExtractElement(B, (uint64_t)0);
4774 C = Builder.CreateExtractElement(C, (uint64_t)0);
4775
4776 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4777 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4778 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4779
4780 Intrinsic::ID IID;
4781 if (Name.back() == 'd')
4782 IID = Intrinsic::x86_avx512_vfmadd_f64;
4783 else
4784 IID = Intrinsic::x86_avx512_vfmadd_f32;
4785 Rep = Builder.CreateIntrinsic(IID, Ops);
4786 } else {
4787 Rep = Builder.CreateFMA(A, B, C);
4788 }
4789
4790 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4791 : IsMask3 ? C
4792 : A;
4793
4794 // For Mask3 with NegAcc, we need to create a new extractelement that
4795 // avoids the negation above.
4796 if (NegAcc && IsMask3)
4797 PassThru =
4798 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4799
4800 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4801 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4802 (uint64_t)0);
4803 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4804 Name.starts_with("avx512.mask.vfnmadd.p") ||
4805 Name.starts_with("avx512.mask.vfnmsub.p") ||
4806 Name.starts_with("avx512.mask3.vfmadd.p") ||
4807 Name.starts_with("avx512.mask3.vfmsub.p") ||
4808 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4809 Name.starts_with("avx512.maskz.vfmadd.p")) {
4810 bool IsMask3 = Name[11] == '3';
4811 bool IsMaskZ = Name[11] == 'z';
4812 // Drop the "avx512.mask." to make it easier.
4813 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4814 bool NegMul = Name[2] == 'n';
4815 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4816
4817 Value *A = CI->getArgOperand(0);
4818 Value *B = CI->getArgOperand(1);
4819 Value *C = CI->getArgOperand(2);
4820
4821 if (NegMul && (IsMask3 || IsMaskZ))
4822 A = Builder.CreateFNeg(A);
4823 if (NegMul && !(IsMask3 || IsMaskZ))
4824 B = Builder.CreateFNeg(B);
4825 if (NegAcc)
4826 C = Builder.CreateFNeg(C);
4827
4828 if (CI->arg_size() == 5 &&
4829 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4830 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4831 Intrinsic::ID IID;
4832 // Check the character before ".512" in string.
4833 if (Name[Name.size() - 5] == 's')
4834 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4835 else
4836 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4837
4838 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4839 } else {
4840 Rep = Builder.CreateFMA(A, B, C);
4841 }
4842
4843 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4844 : IsMask3 ? CI->getArgOperand(2)
4845 : CI->getArgOperand(0);
4846
4847 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4848 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4849 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4850 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4851 Intrinsic::ID IID;
4852 if (VecWidth == 128 && EltWidth == 32)
4853 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4854 else if (VecWidth == 256 && EltWidth == 32)
4855 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4856 else if (VecWidth == 128 && EltWidth == 64)
4857 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4858 else if (VecWidth == 256 && EltWidth == 64)
4859 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4860 else
4861 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4862
4863 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4864 CI->getArgOperand(2)};
4865 Ops[2] = Builder.CreateFNeg(Ops[2]);
4866 Rep = Builder.CreateIntrinsic(IID, Ops);
4867 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4868 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4869 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4870 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4871 bool IsMask3 = Name[11] == '3';
4872 bool IsMaskZ = Name[11] == 'z';
4873 // Drop the "avx512.mask." to make it easier.
4874 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4875 bool IsSubAdd = Name[3] == 's';
4876 if (CI->arg_size() == 5) {
4877 Intrinsic::ID IID;
4878 // Check the character before ".512" in string.
4879 if (Name[Name.size() - 5] == 's')
4880 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4881 else
4882 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4883
4884 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4885 CI->getArgOperand(2), CI->getArgOperand(4)};
4886 if (IsSubAdd)
4887 Ops[2] = Builder.CreateFNeg(Ops[2]);
4888
4889 Rep = Builder.CreateIntrinsic(IID, Ops);
4890 } else {
4891 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4892
4893 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4894 CI->getArgOperand(2)};
4895
4896 Function *FMA = Intrinsic::getOrInsertDeclaration(
4897 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4898 Value *Odd = Builder.CreateCall(FMA, Ops);
4899 Ops[2] = Builder.CreateFNeg(Ops[2]);
4900 Value *Even = Builder.CreateCall(FMA, Ops);
4901
4902 if (IsSubAdd)
4903 std::swap(Even, Odd);
4904
4905 SmallVector<int, 32> Idxs(NumElts);
4906 for (int i = 0; i != NumElts; ++i)
4907 Idxs[i] = i + (i % 2) * NumElts;
4908
4909 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4910 }
4911
4912 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4913 : IsMask3 ? CI->getArgOperand(2)
4914 : CI->getArgOperand(0);
4915
4916 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4917 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4918 Name.starts_with("avx512.maskz.pternlog.")) {
4919 bool ZeroMask = Name[11] == 'z';
4920 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4921 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4922 Intrinsic::ID IID;
4923 if (VecWidth == 128 && EltWidth == 32)
4924 IID = Intrinsic::x86_avx512_pternlog_d_128;
4925 else if (VecWidth == 256 && EltWidth == 32)
4926 IID = Intrinsic::x86_avx512_pternlog_d_256;
4927 else if (VecWidth == 512 && EltWidth == 32)
4928 IID = Intrinsic::x86_avx512_pternlog_d_512;
4929 else if (VecWidth == 128 && EltWidth == 64)
4930 IID = Intrinsic::x86_avx512_pternlog_q_128;
4931 else if (VecWidth == 256 && EltWidth == 64)
4932 IID = Intrinsic::x86_avx512_pternlog_q_256;
4933 else if (VecWidth == 512 && EltWidth == 64)
4934 IID = Intrinsic::x86_avx512_pternlog_q_512;
4935 else
4936 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4937
4938 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4939 CI->getArgOperand(2), CI->getArgOperand(3)};
4940 Rep = Builder.CreateIntrinsic(IID, Args);
4941 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4942 : CI->getArgOperand(0);
4943 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4944 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4945 Name.starts_with("avx512.maskz.vpmadd52")) {
4946 bool ZeroMask = Name[11] == 'z';
4947 bool High = Name[20] == 'h' || Name[21] == 'h';
4948 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4949 Intrinsic::ID IID;
4950 if (VecWidth == 128 && !High)
4951 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4952 else if (VecWidth == 256 && !High)
4953 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4954 else if (VecWidth == 512 && !High)
4955 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4956 else if (VecWidth == 128 && High)
4957 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4958 else if (VecWidth == 256 && High)
4959 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4960 else if (VecWidth == 512 && High)
4961 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4962 else
4963 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4964
4965 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4966 CI->getArgOperand(2)};
4967 Rep = Builder.CreateIntrinsic(IID, Args);
4968 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4969 : CI->getArgOperand(0);
4970 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4971 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4972 Name.starts_with("avx512.mask.vpermt2var.") ||
4973 Name.starts_with("avx512.maskz.vpermt2var.")) {
4974 bool ZeroMask = Name[11] == 'z';
4975 bool IndexForm = Name[17] == 'i';
4976 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4977 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4978 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4979 Name.starts_with("avx512.mask.vpdpbusds.") ||
4980 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4981 bool ZeroMask = Name[11] == 'z';
4982 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4983 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4984 Intrinsic::ID IID;
4985 if (VecWidth == 128 && !IsSaturating)
4986 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4987 else if (VecWidth == 256 && !IsSaturating)
4988 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4989 else if (VecWidth == 512 && !IsSaturating)
4990 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4991 else if (VecWidth == 128 && IsSaturating)
4992 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4993 else if (VecWidth == 256 && IsSaturating)
4994 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4995 else if (VecWidth == 512 && IsSaturating)
4996 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4997 else
4998 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4999
5000 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5001 CI->getArgOperand(2)};
5002
5003 // Input arguments types were incorrectly set to vectors of i32 before but
5004 // they should be vectors of i8. Insert bit cast when encountering the old
5005 // types
5006 if (Args[1]->getType()->isVectorTy() &&
5007 cast<VectorType>(Args[1]->getType())
5008 ->getElementType()
5009 ->isIntegerTy(32) &&
5010 Args[2]->getType()->isVectorTy() &&
5011 cast<VectorType>(Args[2]->getType())
5012 ->getElementType()
5013 ->isIntegerTy(32)) {
5014 Type *NewArgType = nullptr;
5015 if (VecWidth == 128)
5016 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5017 else if (VecWidth == 256)
5018 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5019 else if (VecWidth == 512)
5020 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5021 else
5022 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5023 CI);
5024
5025 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5026 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5027 }
5028
5029 Rep = Builder.CreateIntrinsic(IID, Args);
5030 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5031 : CI->getArgOperand(0);
5032 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5033 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5034 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5035 Name.starts_with("avx512.mask.vpdpwssds.") ||
5036 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5037 bool ZeroMask = Name[11] == 'z';
5038 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5039 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5040 Intrinsic::ID IID;
5041 if (VecWidth == 128 && !IsSaturating)
5042 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5043 else if (VecWidth == 256 && !IsSaturating)
5044 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5045 else if (VecWidth == 512 && !IsSaturating)
5046 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5047 else if (VecWidth == 128 && IsSaturating)
5048 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5049 else if (VecWidth == 256 && IsSaturating)
5050 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5051 else if (VecWidth == 512 && IsSaturating)
5052 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5053 else
5054 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5055
5056 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5057 CI->getArgOperand(2)};
5058
5059 // Input arguments types were incorrectly set to vectors of i32 before but
5060 // they should be vectors of i16. Insert bit cast when encountering the old
5061 // types
5062 if (Args[1]->getType()->isVectorTy() &&
5063 cast<VectorType>(Args[1]->getType())
5064 ->getElementType()
5065 ->isIntegerTy(32) &&
5066 Args[2]->getType()->isVectorTy() &&
5067 cast<VectorType>(Args[2]->getType())
5068 ->getElementType()
5069 ->isIntegerTy(32)) {
5070 Type *NewArgType = nullptr;
5071 if (VecWidth == 128)
5072 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5073 else if (VecWidth == 256)
5074 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5075 else if (VecWidth == 512)
5076 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5077 else
5078 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5079 CI);
5080
5081 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5082 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5083 }
5084
5085 Rep = Builder.CreateIntrinsic(IID, Args);
5086 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5087 : CI->getArgOperand(0);
5088 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5089 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5090 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5091 Name == "subborrow.u32" || Name == "subborrow.u64") {
5092 Intrinsic::ID IID;
5093 if (Name[0] == 'a' && Name.back() == '2')
5094 IID = Intrinsic::x86_addcarry_32;
5095 else if (Name[0] == 'a' && Name.back() == '4')
5096 IID = Intrinsic::x86_addcarry_64;
5097 else if (Name[0] == 's' && Name.back() == '2')
5098 IID = Intrinsic::x86_subborrow_32;
5099 else if (Name[0] == 's' && Name.back() == '4')
5100 IID = Intrinsic::x86_subborrow_64;
5101 else
5102 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5103
5104 // Make a call with 3 operands.
5105 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5106 CI->getArgOperand(2)};
5107 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5108
5109 // Extract the second result and store it.
5110 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5111 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5112 // Replace the original call result with the first result of the new call.
5113 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5114
5115 CI->replaceAllUsesWith(CF);
5116 Rep = nullptr;
5117 } else if (Name.starts_with("avx512.mask.") &&
5118 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5119 // Rep will be updated by the call in the condition.
5120 } else if (Name.starts_with("bmi.pdep.")) {
5121 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5122 } else if (Name.starts_with("bmi.pext.")) {
5123 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5124 } else
5125 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5126
5127 return Rep;
5128}
5129
5130static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5131 Function *F, IRBuilder<> &Builder) {
5132 if (Name.starts_with("neon.bfcvt")) {
5133 if (Name.starts_with("neon.bfcvtn2")) {
5134 SmallVector<int, 32> LoMask(4);
5135 std::iota(LoMask.begin(), LoMask.end(), 0);
5136 SmallVector<int, 32> ConcatMask(8);
5137 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5138 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5139 Value *Trunc =
5140 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5141 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5142 } else if (Name.starts_with("neon.bfcvtn")) {
5143 SmallVector<int, 32> ConcatMask(8);
5144 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5145 Type *V4BF16 =
5146 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5147 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5148 dbgs() << "Trunc: " << *Trunc << "\n";
5149 return Builder.CreateShuffleVector(
5150 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5151 } else {
5152 return Builder.CreateFPTrunc(CI->getOperand(0),
5153 Type::getBFloatTy(F->getContext()));
5154 }
5155 } else if (Name.starts_with("sve.fcvt")) {
5156 Intrinsic::ID NewID =
5157 StringSwitch<Intrinsic::ID>(Name)
5158 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5159 .Case("sve.fcvtnt.bf16f32",
5160 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5161 .Default(Intrinsic::not_intrinsic);
5162 if (NewID == Intrinsic::not_intrinsic)
5163 llvm_unreachable("Unhandled Intrinsic!");
5164
5165 SmallVector<Value *, 3> Args(CI->args());
5166
5167 // The original intrinsics incorrectly used a predicate based on the
5168 // smallest element type rather than the largest.
5169 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5170 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5171
5172 if (Args[1]->getType() != BadPredTy)
5173 llvm_unreachable("Unexpected predicate type!");
5174
5175 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5176 BadPredTy, Args[1]);
5177 Args[1] = Builder.CreateIntrinsic(
5178 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5179
5180 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5181 CI->getName());
5182 }
5183
5184 if (Name == "neon.vcvtfp2hf")
5185 return Builder.CreateBitCast(
5186 Builder.CreateFPTrunc(
5187 CI->getOperand(0),
5188 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5189 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5190 if (Name == "neon.vcvthf2fp")
5191 return Builder.CreateFPExt(
5192 Builder.CreateBitCast(
5193 CI->getOperand(0),
5194 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5195 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5196
5197 llvm_unreachable("Unhandled Intrinsic!");
5198}
5199
5200static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5201 IRBuilder<> &Builder) {
5202 if (Name == "mve.vctp64.old") {
5203 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5204 // correct type.
5205 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5206 CI->getArgOperand(0),
5207 /*FMFSource=*/nullptr, CI->getName());
5208 Value *C1 = Builder.CreateIntrinsic(
5209 Intrinsic::arm_mve_pred_v2i,
5210 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5211 return Builder.CreateIntrinsic(
5212 Intrinsic::arm_mve_pred_i2v,
5213 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5214 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5215 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5216 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5217 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5218 Name ==
5219 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5220 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5221 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5222 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5223 Name ==
5224 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5225 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5226 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5227 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5228 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5229 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5230 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5231 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5232 std::vector<Type *> Tys;
5233 unsigned ID = CI->getIntrinsicID();
5234 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5235 switch (ID) {
5236 case Intrinsic::arm_mve_mull_int_predicated:
5237 case Intrinsic::arm_mve_vqdmull_predicated:
5238 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5239 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5240 break;
5241 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5242 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5243 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5244 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5245 V2I1Ty};
5246 break;
5247 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5248 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5249 CI->getOperand(1)->getType(), V2I1Ty};
5250 break;
5251 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5252 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5253 CI->getOperand(2)->getType(), V2I1Ty};
5254 break;
5255 case Intrinsic::arm_cde_vcx1q_predicated:
5256 case Intrinsic::arm_cde_vcx1qa_predicated:
5257 case Intrinsic::arm_cde_vcx2q_predicated:
5258 case Intrinsic::arm_cde_vcx2qa_predicated:
5259 case Intrinsic::arm_cde_vcx3q_predicated:
5260 case Intrinsic::arm_cde_vcx3qa_predicated:
5261 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5262 break;
5263 default:
5264 llvm_unreachable("Unhandled Intrinsic!");
5265 }
5266
5267 std::vector<Value *> Ops;
5268 for (Value *Op : CI->args()) {
5269 Type *Ty = Op->getType();
5270 if (Ty->getScalarSizeInBits() == 1) {
5271 Value *C1 = Builder.CreateIntrinsic(
5272 Intrinsic::arm_mve_pred_v2i,
5273 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5274 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5275 }
5276 Ops.push_back(Op);
5277 }
5278
5279 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5280 CI->getName());
5281 }
5282 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5283}
5284
5285// These are expected to have the arguments:
5286// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5287//
5288// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5289//
5290static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5291 Function *F, IRBuilder<> &Builder) {
5292 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5293 // for compatibility.
5294 auto UpgradeLegacyWMMAIUIntrinsicCall =
5295 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5296 ArrayRef<Type *> OverloadTys) -> Value * {
5297 // Prepare arguments, append clamp=0 for compatibility
5298 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5299 Args.push_back(Builder.getFalse());
5300
5301 // Insert the declaration for the right overload types
5302 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5303 F->getParent(), F->getIntrinsicID(), OverloadTys);
5304
5305 // Copy operand bundles if any
5306 SmallVector<OperandBundleDef, 1> Bundles;
5307 CI->getOperandBundlesAsDefs(Bundles);
5308
5309 // Create the new call and copy calling properties
5310 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5311 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5312 NewCall->setCallingConv(CI->getCallingConv());
5313 NewCall->setAttributes(CI->getAttributes());
5314 NewCall->copyMetadata(*CI);
5315 return NewCall;
5316 };
5317
5318 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5319 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5320 "intrinsic should have 7 arguments");
5321 Type *T1 = CI->getArgOperand(4)->getType();
5322 Type *T2 = CI->getArgOperand(1)->getType();
5323 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5324 }
5325 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5326 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5327 "intrinsic should have 8 arguments");
5328 Type *T1 = CI->getArgOperand(4)->getType();
5329 Type *T2 = CI->getArgOperand(1)->getType();
5330 Type *T3 = CI->getArgOperand(3)->getType();
5331 Type *T4 = CI->getArgOperand(5)->getType();
5332 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5333 }
5334
5335 switch (F->getIntrinsicID()) {
5336 default:
5337 break;
5338 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5339 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5340 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5341 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5342 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5343 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5344 // Drop src0 and src1 modifiers.
5345 const Value *Op0 = CI->getArgOperand(0);
5346 const Value *Op2 = CI->getArgOperand(2);
5347 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5348 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5349 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5350 if (!ModA->isZero() || !ModB->isZero())
5351 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5352
5353 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5354 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5355 Args.push_back(CI->getArgOperand(I));
5356
5357 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5358 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5359 Overloads.push_back(Args[3]->getType());
5360 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5361 F->getParent(), F->getIntrinsicID(), Overloads);
5362
5363 SmallVector<OperandBundleDef, 1> Bundles;
5364 CI->getOperandBundlesAsDefs(Bundles);
5365
5366 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5367 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5368 NewCall->setCallingConv(CI->getCallingConv());
5369 NewCall->setAttributes(CI->getAttributes());
5370 NewCall->copyMetadata(*CI);
5371 NewCall->takeName(CI);
5372 return NewCall;
5373 }
5374 }
5375
5376 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5377 Value *LHS = CI->getArgOperand(0);
5378 Value *RHS = CI->getArgOperand(1);
5379 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5380 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5381 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5382 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5383 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5384 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5385 NewCall->setCallingConv(CI->getCallingConv());
5386 NewCall->copyMetadata(*CI);
5387 NewCall->takeName(CI);
5388 return NewCall;
5389 }
5390
5391 if (Name.starts_with("addrspacecast.nonnull")) {
5392 if (CI->getNumOperands() < 2) // Malformed bitcode.
5393 return nullptr;
5394 Value *ASC = Builder.CreateAddrSpaceCast(
5395 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5396 ASC->takeName(CI);
5397 return ASC;
5398 }
5399
5400 AtomicRMWInst::BinOp RMWOp =
5401 StringSwitch<AtomicRMWInst::BinOp>(Name)
5402 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5403 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5404 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5405 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5406 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5407 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5408 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5409 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5410 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5411 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5412 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5413 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5414 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5415
5416 unsigned NumOperands = CI->getNumOperands();
5417 if (NumOperands < 3) // Malformed bitcode.
5418 return nullptr;
5419
5420 Value *Ptr = CI->getArgOperand(0);
5421 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5422 if (!PtrTy) // Malformed.
5423 return nullptr;
5424
5425 Value *Val = CI->getArgOperand(1);
5426 if (Val->getType() != CI->getType()) // Malformed.
5427 return nullptr;
5428
5429 ConstantInt *OrderArg = nullptr;
5430 bool IsVolatile = false;
5431
5432 // These should have 5 arguments (plus the callee). A separate version of the
5433 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5434 if (NumOperands > 3)
5435 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5436
5437 // Ignore scope argument at 3
5438
5439 if (NumOperands > 5) {
5440 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5441 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5442 }
5443
5444 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5445 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5446 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5447 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5448 Order = AtomicOrdering::SequentiallyConsistent;
5449
5450 LLVMContext &Ctx = F->getContext();
5451
5452 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5453 Type *RetTy = CI->getType();
5454 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5455 if (VT->getElementType()->isIntegerTy(16)) {
5456 VectorType *AsBF16 =
5457 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5458 Val = Builder.CreateBitCast(Val, AsBF16);
5459 }
5460 }
5461
5462 // The scope argument never really worked correctly. Use agent as the most
5463 // conservative option which should still always produce the instruction.
5464 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5465 AtomicRMWInst *RMW =
5466 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5467
5468 unsigned AddrSpace = PtrTy->getAddressSpace();
5469 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5470 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5471 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5472 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5473 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5474 }
5475
5476 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5477 MDBuilder MDB(F->getContext());
5478 MDNode *RangeNotPrivate =
5479 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5480 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5481 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5482 }
5483
5484 if (IsVolatile)
5485 RMW->setVolatile(true);
5486
5487 return Builder.CreateBitCast(RMW, RetTy);
5488}
5489
5490/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5491/// plain MDNode, as it's the verifier's job to check these are the correct
5492/// types later.
5493static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5494 if (Op < CI->arg_size()) {
5495 if (MetadataAsValue *MAV =
5496 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5497 Metadata *MD = MAV->getMetadata();
5498 return dyn_cast_if_present<MDNode>(MD);
5499 }
5500 }
5501 return nullptr;
5502}
5503
5504/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5505static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5506 if (Op < CI->arg_size())
5507 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5508 return MAV->getMetadata();
5509 return nullptr;
5510}
5511
5512/// Convert debug intrinsic calls to non-instruction debug records.
5513/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5514/// \p CI - The debug intrinsic call.
5515static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5516 DbgRecord *DR = nullptr;
5517 if (Name == "label") {
5518 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5519 } else if (Name == "assign") {
5520 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5521 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5522 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5523 unwrapMAVMetadataOp(CI, 4),
5524 /*The address is a Value ref, it will be stored as a Metadata */
5525 unwrapMAVOp(CI, 5));
5526 } else if (Name == "declare") {
5527 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5528 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5529 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5530 } else if (Name == "addr") {
5531 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5532 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5533 // Don't try to add something to the expression if it's not an expression.
5534 // Instead, allow the verifier to fail later.
5535 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5536 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5537 }
5538 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5539 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5540 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5541 } else if (Name == "value") {
5542 // An old version of dbg.value had an extra offset argument.
5543 unsigned VarOp = 1;
5544 unsigned ExprOp = 2;
5545 if (CI->arg_size() == 4) {
5546 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5547 // Nonzero offset dbg.values get dropped without a replacement.
5548 if (!Offset || !Offset->isNullValue())
5549 return;
5550 VarOp = 2;
5551 ExprOp = 3;
5552 }
5553 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5554 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5555 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5556 nullptr);
5557 }
5558 DR->setDebugLoc(CI->getDebugLoc());
5559 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5560 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5561}
5562
5563static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5564 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5565 if (!Offset)
5566 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5567 int64_t OffsetVal = Offset->getSExtValue();
5568 return Builder.CreateIntrinsic(OffsetVal >= 0
5569 ? Intrinsic::vector_splice_left
5570 : Intrinsic::vector_splice_right,
5571 CI->getType(),
5572 {CI->getArgOperand(0), CI->getArgOperand(1),
5573 Builder.getInt32(std::abs(OffsetVal))});
5574}
5575
5576static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5577 Function *F, IRBuilder<> &Builder) {
5578 if (Name.starts_with("to.fp16")) {
5579 Value *Cast =
5580 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5581 return Builder.CreateBitCast(Cast, CI->getType());
5582 }
5583
5584 if (Name.starts_with("from.fp16")) {
5585 Value *Cast =
5586 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5587 return Builder.CreateFPExt(Cast, CI->getType());
5588 }
5589
5590 return nullptr;
5591}
5592
5593static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5594 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5595 if (!MD || !isa<MDString>(MD))
5596 return ICmpInst::BAD_ICMP_PREDICATE;
5597 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5598 .Case("eq", ICmpInst::ICMP_EQ)
5599 .Case("ne", ICmpInst::ICMP_NE)
5600 .Case("ugt", ICmpInst::ICMP_UGT)
5601 .Case("uge", ICmpInst::ICMP_UGE)
5602 .Case("ult", ICmpInst::ICMP_ULT)
5603 .Case("ule", ICmpInst::ICMP_ULE)
5604 .Case("sgt", ICmpInst::ICMP_SGT)
5605 .Case("sge", ICmpInst::ICMP_SGE)
5606 .Case("slt", ICmpInst::ICMP_SLT)
5607 .Case("sle", ICmpInst::ICMP_SLE)
5608 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5609}
5610
5611static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5612 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5613 if (!MD || !isa<MDString>(MD))
5614 return FCmpInst::BAD_FCMP_PREDICATE;
5615 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5616 .Case("oeq", FCmpInst::FCMP_OEQ)
5617 .Case("ogt", FCmpInst::FCMP_OGT)
5618 .Case("oge", FCmpInst::FCMP_OGE)
5619 .Case("olt", FCmpInst::FCMP_OLT)
5620 .Case("ole", FCmpInst::FCMP_OLE)
5621 .Case("one", FCmpInst::FCMP_ONE)
5622 .Case("ord", FCmpInst::FCMP_ORD)
5623 .Case("uno", FCmpInst::FCMP_UNO)
5624 .Case("ueq", FCmpInst::FCMP_UEQ)
5625 .Case("ugt", FCmpInst::FCMP_UGT)
5626 .Case("uge", FCmpInst::FCMP_UGE)
5627 .Case("ult", FCmpInst::FCMP_ULT)
5628 .Case("ule", FCmpInst::FCMP_ULE)
5629 .Case("une", FCmpInst::FCMP_UNE)
5630 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5631}
5632
5633static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5634 IRBuilder<> &Builder) {
5635 Value *Rep;
5636 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5637 if (Opcode && Instruction::isUnaryOp(Opcode))
5638 Rep =
5639 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5640 else if (Opcode && Instruction::isBinaryOp(Opcode))
5641 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5642 CI->getArgOperand(0), CI->getArgOperand(1));
5643 else if (Opcode && Instruction::isCast(Opcode))
5644 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5645 CI->getType());
5646 else if (Opcode == Instruction::ICmp)
5647 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5648 CI->getArgOperand(0), CI->getArgOperand(1));
5649 else if (Opcode == Instruction::FCmp)
5650 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5651 CI->getArgOperand(0), CI->getArgOperand(1));
5652 else if (Opcode == Instruction::Select)
5653 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5654 CI->getArgOperand(2));
5655 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5656 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5657 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5658 } else
5659 llvm_unreachable("Unexpected vp intrinsic");
5660 Rep->takeName(CI);
5661 return Rep;
5662}
5663
5664static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5665 IRBuilder<> &Builder) {
5666 Intrinsic::ID IID = NewFn->getIntrinsicID();
5667
5668 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5669 if (Defaults.empty())
5670 return false;
5671
5672 unsigned OldArgCount = CI->arg_size();
5673 unsigned NewArgCount = NewFn->arg_size();
5674
5675 if (OldArgCount < FirstDefault)
5676 return false;
5677
5678 // More arguments than the new intrinsic accepts, cannot upgrade.
5679 if (OldArgCount > NewArgCount)
5680 return false;
5681
5682 // The call already passes the defaulted arguments explicitly; only the
5683 // callee is still the old, shorter declaration, so retarget it.
5684 if (OldArgCount == NewArgCount) {
5685 if (CI->getFunctionType() != NewFn->getFunctionType())
5686 return false;
5687 CI->setCalledFunction(NewFn);
5688 return true;
5689 }
5690
5691 // OldArgCount < NewArgCount: Fill in each missing trailing default
5692 // argument from the table.
5693 SmallVector<Value *, 8> NewArgs(CI->args());
5694
5695 FunctionType *NewFT = NewFn->getFunctionType();
5696 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5697 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5698 "missing argument outside the default range");
5699 Type *ParamTy = NewFT->getParamType(Idx);
5700
5701 // Only integer types are supported (i1, i8, i16, i32, i64).
5702 if (!ParamTy->isIntegerTy())
5703 return false;
5704 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5705 }
5706
5707 // Preserve operand bundles by creating the call with them.
5708 SmallVector<OperandBundleDef, 1> OpBundles;
5709 CI->getOperandBundlesAsDefs(OpBundles);
5710 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5711
5712 NewCall->takeName(CI);
5713 NewCall->setCallingConv(CI->getCallingConv());
5714 NewCall->copyMetadata(*CI);
5715 if (auto *OldCI = dyn_cast<CallInst>(CI))
5716 NewCall->setTailCallKind(OldCI->getTailCallKind());
5717
5718 CI->replaceAllUsesWith(NewCall);
5719 CI->eraseFromParent();
5720 return true;
5721}
5722
5723/// Upgrade a call to an old intrinsic. All argument and return casting must be
5724/// provided to seamlessly integrate with existing context.
5725void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5726 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5727 // checks the callee's function type matches. It's likely we need to handle
5728 // type changes here.
5729 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5730 if (!F)
5731 return;
5732
5733 LLVMContext &C = CI->getContext();
5734 IRBuilder<> Builder(C);
5735 if (isa<FPMathOperator>(CI))
5736 Builder.setFastMathFlags(CI->getFastMathFlags());
5737 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5738
5739 if (!NewFn) {
5740 // Get the Function's name.
5741 StringRef Name = F->getName();
5742 if (!Name.consume_front("llvm."))
5743 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5744
5745 bool IsX86 = Name.consume_front("x86.");
5746 bool IsNVVM = Name.consume_front("nvvm.");
5747 bool IsAArch64 = Name.consume_front("aarch64.");
5748 bool IsARM = Name.consume_front("arm.");
5749 bool IsAMDGCN = Name.consume_front("amdgcn.");
5750 bool IsDbg = Name.consume_front("dbg.");
5751 bool IsOldSplice =
5752 (Name.consume_front("experimental.vector.splice") ||
5753 Name.consume_front("vector.splice")) &&
5754 !(Name.starts_with(".left") || Name.starts_with(".right"));
5755 Value *Rep = nullptr;
5756
5757 if (!IsX86 && Name == "stackprotectorcheck") {
5758 Rep = nullptr;
5759 } else if (IsNVVM) {
5760 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5761 } else if (IsX86) {
5762 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5763 } else if (IsAArch64) {
5764 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5765 } else if (IsARM) {
5766 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5767 } else if (IsAMDGCN) {
5768 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5769 } else if (IsDbg) {
5770 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5771 } else if (IsOldSplice) {
5772 Rep = upgradeVectorSplice(CI, Builder);
5773 } else if (Name.consume_front("convert.")) {
5774 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5775 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5776 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5777 Rep = nullptr;
5778 } else if (shouldUpgradeVPIntrinsic(Name)) {
5779 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5780 } else {
5781 llvm_unreachable("Unknown function for CallBase upgrade.");
5782 }
5783
5784 if (Rep)
5785 CI->replaceAllUsesWith(Rep);
5786 CI->eraseFromParent();
5787 return;
5788 }
5789
5790 const auto &DefaultCase = [&]() -> void {
5791 if (F == NewFn)
5792 return;
5793
5794 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5795 // Handle generic mangling change.
5796 assert(
5797 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5798 "Unknown function for CallBase upgrade and isn't just a name change");
5799 CI->setCalledFunction(NewFn);
5800 return;
5801 }
5802
5803 // This must be an upgrade from a named to a literal struct.
5804 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5805 assert(OldST != NewFn->getReturnType() &&
5806 "Return type must have changed");
5807 assert(OldST->getNumElements() ==
5808 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5809 "Must have same number of elements");
5810
5811 SmallVector<Value *> Args(CI->args());
5812 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5813 NewCI->setAttributes(CI->getAttributes());
5814 Value *Res = PoisonValue::get(OldST);
5815 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5816 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5817 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5818 }
5819 CI->replaceAllUsesWith(Res);
5820 CI->eraseFromParent();
5821 return;
5822 }
5823
5824 // We're probably about to produce something invalid. Let the verifier catch
5825 // it instead of dying here.
5826 CI->setCalledOperand(
5827 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5828 return;
5829 };
5830 CallInst *NewCall = nullptr;
5831 switch (NewFn->getIntrinsicID()) {
5832 default: {
5833 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5834 return;
5835 DefaultCase();
5836 return;
5837 }
5838 case Intrinsic::arm_neon_vst1:
5839 case Intrinsic::arm_neon_vst2:
5840 case Intrinsic::arm_neon_vst3:
5841 case Intrinsic::arm_neon_vst4:
5842 case Intrinsic::arm_neon_vst2lane:
5843 case Intrinsic::arm_neon_vst3lane:
5844 case Intrinsic::arm_neon_vst4lane: {
5845 SmallVector<Value *, 4> Args(CI->args());
5846 NewCall = Builder.CreateCall(NewFn, Args);
5847 break;
5848 }
5849 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5850 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5851 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5852 LLVMContext &Ctx = F->getParent()->getContext();
5853 SmallVector<Value *, 4> Args(CI->args());
5854 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5855 cast<ConstantInt>(Args[3])->getZExtValue());
5856 NewCall = Builder.CreateCall(NewFn, Args);
5857 break;
5858 }
5859 case Intrinsic::aarch64_sve_ld3_sret:
5860 case Intrinsic::aarch64_sve_ld4_sret:
5861 case Intrinsic::aarch64_sve_ld2_sret: {
5862 // Is this a trivial remangle of the name to support ptr address spaces?
5863 if (isa<StructType>(F->getReturnType())) {
5864 DefaultCase();
5865 return;
5866 }
5867
5868 StringRef Name = F->getName();
5869 Name = Name.substr(5);
5870 unsigned N = StringSwitch<unsigned>(Name)
5871 .StartsWith("aarch64.sve.ld2", 2)
5872 .StartsWith("aarch64.sve.ld3", 3)
5873 .StartsWith("aarch64.sve.ld4", 4)
5874 .Default(0);
5875 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5876 unsigned MinElts = RetTy->getMinNumElements() / N;
5877 SmallVector<Value *, 2> Args(CI->args());
5878 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5879 Value *Ret = llvm::PoisonValue::get(RetTy);
5880 for (unsigned I = 0; I < N; I++) {
5881 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5882 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5883 }
5884 NewCall = dyn_cast<CallInst>(Ret);
5885 break;
5886 }
5887
5888 case Intrinsic::coro_end_async:
5889 case Intrinsic::coro_end: {
5890 SmallVector<Value *, 3> Args(CI->args());
5891 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5892 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5893 NewCall = Builder.CreateCall(NewFn, Args);
5894
5895 if (!CI->getType()->isVoidTy()) {
5896 if (!CI->use_empty()) {
5897 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5898 CI->getModule(), Intrinsic::coro_is_in_ramp);
5899 Value *InRamp = Builder.CreateCall(IsInRamp);
5900 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5901 }
5902 CI->eraseFromParent();
5903 return;
5904 }
5905
5906 break;
5907 }
5908
5909 case Intrinsic::vector_extract: {
5910 StringRef Name = F->getName();
5911 Name = Name.substr(5); // Strip llvm
5912 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5913 DefaultCase();
5914 return;
5915 }
5916 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5917 unsigned MinElts = RetTy->getMinNumElements();
5918 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5919 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5920 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5921 break;
5922 }
5923
5924 case Intrinsic::vector_insert: {
5925 StringRef Name = F->getName();
5926 Name = Name.substr(5);
5927 if (!Name.starts_with("aarch64.sve.tuple")) {
5928 DefaultCase();
5929 return;
5930 }
5931 if (Name.starts_with("aarch64.sve.tuple.set")) {
5932 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5933 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5934 Value *NewIdx =
5935 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5936 NewCall = Builder.CreateCall(
5937 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5938 break;
5939 }
5940 if (Name.starts_with("aarch64.sve.tuple.create")) {
5941 unsigned N = StringSwitch<unsigned>(Name)
5942 .StartsWith("aarch64.sve.tuple.create2", 2)
5943 .StartsWith("aarch64.sve.tuple.create3", 3)
5944 .StartsWith("aarch64.sve.tuple.create4", 4)
5945 .Default(0);
5946 assert(N > 1 && "Create is expected to be between 2-4");
5947 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5948 Value *Ret = llvm::PoisonValue::get(RetTy);
5949 unsigned MinElts = RetTy->getMinNumElements() / N;
5950 for (unsigned I = 0; I < N; I++) {
5951 Value *V = CI->getArgOperand(I);
5952 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5953 }
5954 NewCall = dyn_cast<CallInst>(Ret);
5955 }
5956 break;
5957 }
5958
5959 case Intrinsic::arm_neon_bfdot:
5960 case Intrinsic::arm_neon_bfmmla:
5961 case Intrinsic::arm_neon_bfmlalb:
5962 case Intrinsic::arm_neon_bfmlalt:
5963 case Intrinsic::aarch64_neon_bfdot:
5964 case Intrinsic::aarch64_neon_bfmmla:
5965 case Intrinsic::aarch64_neon_bfmlalb:
5966 case Intrinsic::aarch64_neon_bfmlalt: {
5967 SmallVector<Value *, 3> Args;
5968 assert(CI->arg_size() == 3 &&
5969 "Mismatch between function args and call args");
5970 size_t OperandWidth =
5971 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5972 assert((OperandWidth == 64 || OperandWidth == 128) &&
5973 "Unexpected operand width");
5974 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5975 auto Iter = CI->args().begin();
5976 Args.push_back(*Iter++);
5977 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5978 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5979 NewCall = Builder.CreateCall(NewFn, Args);
5980 break;
5981 }
5982
5983 case Intrinsic::bitreverse:
5984 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5985 break;
5986
5987 case Intrinsic::ctlz:
5988 case Intrinsic::cttz: {
5989 if (CI->arg_size() != 1) {
5990 DefaultCase();
5991 return;
5992 }
5993
5994 NewCall =
5995 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5996 break;
5997 }
5998
5999 case Intrinsic::objectsize: {
6000 Value *NullIsUnknownSize =
6001 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6002 Value *Dynamic =
6003 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6004 NewCall = Builder.CreateCall(
6005 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6006 break;
6007 }
6008
6009 case Intrinsic::ctpop:
6010 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6011 break;
6012 case Intrinsic::dbg_value: {
6013 StringRef Name = F->getName();
6014 Name = Name.substr(5); // Strip llvm.
6015 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6016 if (Name.starts_with("dbg.addr")) {
6017 DIExpression *Expr = cast<DIExpression>(
6018 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6019 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6020 NewCall =
6021 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6022 MetadataAsValue::get(C, Expr)});
6023 break;
6024 }
6025
6026 // Upgrade from the old version that had an extra offset argument.
6027 assert(CI->arg_size() == 4);
6028 // Drop nonzero offsets instead of attempting to upgrade them.
6029 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
6030 if (Offset->isNullValue()) {
6031 NewCall = Builder.CreateCall(
6032 NewFn,
6033 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6034 break;
6035 }
6036 CI->eraseFromParent();
6037 return;
6038 }
6039
6040 case Intrinsic::ptr_annotation:
6041 // Upgrade from versions that lacked the annotation attribute argument.
6042 if (CI->arg_size() != 4) {
6043 DefaultCase();
6044 return;
6045 }
6046
6047 // Create a new call with an added null annotation attribute argument.
6048 NewCall = Builder.CreateCall(
6049 NewFn,
6050 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6051 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6052 NewCall->takeName(CI);
6053 CI->replaceAllUsesWith(NewCall);
6054 CI->eraseFromParent();
6055 return;
6056
6057 case Intrinsic::var_annotation:
6058 // Upgrade from versions that lacked the annotation attribute argument.
6059 if (CI->arg_size() != 4) {
6060 DefaultCase();
6061 return;
6062 }
6063 // Create a new call with an added null annotation attribute argument.
6064 NewCall = Builder.CreateCall(
6065 NewFn,
6066 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6067 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6068 NewCall->takeName(CI);
6069 CI->replaceAllUsesWith(NewCall);
6070 CI->eraseFromParent();
6071 return;
6072
6073 case Intrinsic::riscv_aes32dsi:
6074 case Intrinsic::riscv_aes32dsmi:
6075 case Intrinsic::riscv_aes32esi:
6076 case Intrinsic::riscv_aes32esmi:
6077 case Intrinsic::riscv_sm4ks:
6078 case Intrinsic::riscv_sm4ed: {
6079 // The last argument to these intrinsics used to be i8 and changed to i32.
6080 // The type overload for sm4ks and sm4ed was removed.
6081 Value *Arg2 = CI->getArgOperand(2);
6082 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6083 return;
6084
6085 Value *Arg0 = CI->getArgOperand(0);
6086 Value *Arg1 = CI->getArgOperand(1);
6087 if (CI->getType()->isIntegerTy(64)) {
6088 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6089 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6090 }
6091
6092 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6093 cast<ConstantInt>(Arg2)->getZExtValue());
6094
6095 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6096 Value *Res = NewCall;
6097 if (Res->getType() != CI->getType())
6098 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6099 NewCall->takeName(CI);
6100 CI->replaceAllUsesWith(Res);
6101 CI->eraseFromParent();
6102 return;
6103 }
6104 case Intrinsic::nvvm_mapa_shared_cluster: {
6105 // Create a new call with the correct address space.
6106 NewCall =
6107 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6108 Value *Res = NewCall;
6109 Res = Builder.CreateAddrSpaceCast(
6110 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6111 NewCall->takeName(CI);
6112 CI->replaceAllUsesWith(Res);
6113 CI->eraseFromParent();
6114 return;
6115 }
6116 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6117 SmallVector<Value *, 4> Args(CI->args());
6118 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6119 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6120 Args[0] = Builder.CreateAddrSpaceCast(
6121 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6122
6123 // Append the missing trailing flag_valid_pattern (0 = disabled).
6124 Args.push_back(Builder.getInt32(0));
6125
6126 NewCall = Builder.CreateCall(NewFn, Args);
6127 NewCall->takeName(CI);
6128 CI->replaceAllUsesWith(NewCall);
6129 CI->eraseFromParent();
6130 return;
6131 }
6132 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6133 // (dst, mbar, src, size, ch, flag_ch)
6134 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6135 // i32 0 /* flag_valid_pattern=disabled */)
6136 SmallVector<Value *, 10> Args;
6137 for (unsigned I = 0; I < 4; ++I)
6138 Args.push_back(CI->getArgOperand(I));
6139 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6140 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6141 Args.push_back(CI->getArgOperand(4)); // cache_hint
6142 Args.push_back(CI->getArgOperand(5)); // flag_ch
6143 Args.push_back(Builder.getInt1(false)); // flag_oob
6144 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6145
6146 NewCall = Builder.CreateCall(NewFn, Args);
6147 NewCall->takeName(CI);
6148 CI->replaceAllUsesWith(NewCall);
6149 CI->eraseFromParent();
6150 return;
6151 }
6152 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6153 // Create a new call with the correct address space.
6154 SmallVector<Value *, 4> Args(CI->args());
6155 Args[0] = Builder.CreateAddrSpaceCast(
6156 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6157
6158 NewCall = Builder.CreateCall(NewFn, Args);
6159 NewCall->takeName(CI);
6160 CI->replaceAllUsesWith(NewCall);
6161 CI->eraseFromParent();
6162 return;
6163 }
6164 // clang-format off
6165#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6166 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6167 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6168#undef G2S_CLUSTER_CASE
6169 {
6170 SmallVector<Value *, 16> Args(CI->args());
6171 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6172 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6173 Args[0] = Builder.CreateAddrSpaceCast(
6174 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6175
6176 // Append the missing trailing arguments with default values (cta_group,
6177 // flag_valid_pattern).
6178 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6179 Args.push_back(Builder.getInt32(0));
6180
6181 NewCall = Builder.CreateCall(NewFn, Args);
6182 NewCall->takeName(CI);
6183 CI->replaceAllUsesWith(NewCall);
6184 CI->eraseFromParent();
6185 return;
6186 }
6187
6188#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6189 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6190 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6191#undef G2S_CTA_CASE
6192 {
6193 SmallVector<Value *, 16> Args(CI->args());
6194 // Append the missing trailing flag_valid_pattern argument with default
6195 // value 0.
6196 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6197 "expected only the trailing flag_valid_pattern to be missing");
6198 Args.push_back(Builder.getInt32(0));
6199
6200 NewCall = Builder.CreateCall(NewFn, Args);
6201 NewCall->takeName(CI);
6202 CI->replaceAllUsesWith(NewCall);
6203 CI->eraseFromParent();
6204 return;
6205 }
6206#undef NVVM_TMA_G2S_MODES
6207 // clang-format on
6208
6209 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6210 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6211 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6212 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6213 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6214 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6215 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6216 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6217 StringRef Name = F->getName();
6218 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6219 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6220
6221 SmallVector<Value *, 16> Args(CI->args());
6222 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6223 NewCall = Builder.CreateCall(NewFn, Args);
6224 break;
6225 }
6226 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6227 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6228 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6229 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6230 NewCall =
6231 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6232 Builder.getFalse()});
6233 break;
6234 case Intrinsic::nvvm_mbarrier_init: {
6235 SmallVector<Value *, 3> Args(CI->args());
6236 // The .shared variant folded into the overloaded form without gaining an
6237 // operand, so only the pre-layout two-argument form needs one appended.
6238 if (Args.size() == 2)
6239 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6240 NewCall = Builder.CreateCall(NewFn, Args);
6241 break;
6242 }
6243 case Intrinsic::riscv_sha256sig0:
6244 case Intrinsic::riscv_sha256sig1:
6245 case Intrinsic::riscv_sha256sum0:
6246 case Intrinsic::riscv_sha256sum1:
6247 case Intrinsic::riscv_sm3p0:
6248 case Intrinsic::riscv_sm3p1: {
6249 // The last argument to these intrinsics used to be i8 and changed to i32.
6250 // The type overload for sm4ks and sm4ed was removed.
6251 if (!CI->getType()->isIntegerTy(64))
6252 return;
6253
6254 Value *Arg =
6255 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6256
6257 NewCall = Builder.CreateCall(NewFn, Arg);
6258 Value *Res =
6259 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6260 NewCall->takeName(CI);
6261 CI->replaceAllUsesWith(Res);
6262 CI->eraseFromParent();
6263 return;
6264 }
6265
6266 case Intrinsic::x86_xop_vfrcz_ss:
6267 case Intrinsic::x86_xop_vfrcz_sd:
6268 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6269 break;
6270
6271 case Intrinsic::x86_xop_vpermil2pd:
6272 case Intrinsic::x86_xop_vpermil2ps:
6273 case Intrinsic::x86_xop_vpermil2pd_256:
6274 case Intrinsic::x86_xop_vpermil2ps_256: {
6275 SmallVector<Value *, 4> Args(CI->args());
6276 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6277 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6278 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6279 NewCall = Builder.CreateCall(NewFn, Args);
6280 break;
6281 }
6282
6283 case Intrinsic::x86_sse41_ptestc:
6284 case Intrinsic::x86_sse41_ptestz:
6285 case Intrinsic::x86_sse41_ptestnzc: {
6286 // The arguments for these intrinsics used to be v4f32, and changed
6287 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6288 // So, the only thing required is a bitcast for both arguments.
6289 // First, check the arguments have the old type.
6290 Value *Arg0 = CI->getArgOperand(0);
6291 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6292 return;
6293
6294 // Old intrinsic, add bitcasts
6295 Value *Arg1 = CI->getArgOperand(1);
6296
6297 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6298
6299 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6300 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6301
6302 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6303 break;
6304 }
6305
6306 case Intrinsic::x86_rdtscp: {
6307 // This used to take 1 arguments. If we have no arguments, it is already
6308 // upgraded.
6309 if (CI->getNumOperands() == 0)
6310 return;
6311
6312 NewCall = Builder.CreateCall(NewFn);
6313 // Extract the second result and store it.
6314 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6315 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6316 // Replace the original call result with the first result of the new call.
6317 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6318
6319 NewCall->takeName(CI);
6320 CI->replaceAllUsesWith(TSC);
6321 CI->eraseFromParent();
6322 return;
6323 }
6324
6325 case Intrinsic::x86_sse41_insertps:
6326 case Intrinsic::x86_sse41_dppd:
6327 case Intrinsic::x86_sse41_dpps:
6328 case Intrinsic::x86_sse41_mpsadbw:
6329 case Intrinsic::x86_avx_dp_ps_256:
6330 case Intrinsic::x86_avx2_mpsadbw: {
6331 // Need to truncate the last argument from i32 to i8 -- this argument models
6332 // an inherently 8-bit immediate operand to these x86 instructions.
6333 SmallVector<Value *, 4> Args(CI->args());
6334
6335 // Replace the last argument with a trunc.
6336 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6337 NewCall = Builder.CreateCall(NewFn, Args);
6338 break;
6339 }
6340
6341 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6342 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6343 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6344 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6345 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6346 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6347 SmallVector<Value *, 4> Args(CI->args());
6348 unsigned NumElts =
6349 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6350 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6351
6352 NewCall = Builder.CreateCall(NewFn, Args);
6353 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6354
6355 NewCall->takeName(CI);
6356 CI->replaceAllUsesWith(Res);
6357 CI->eraseFromParent();
6358 return;
6359 }
6360
6361 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6362 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6363 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6364 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6365 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6366 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6367 SmallVector<Value *, 4> Args(CI->args());
6368 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6369 if (NewFn->getIntrinsicID() ==
6370 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6371 Args[1] = Builder.CreateBitCast(
6372 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6373
6374 NewCall = Builder.CreateCall(NewFn, Args);
6375 Value *Res = Builder.CreateBitCast(
6376 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6377
6378 NewCall->takeName(CI);
6379 CI->replaceAllUsesWith(Res);
6380 CI->eraseFromParent();
6381 return;
6382 }
6383 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6384 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6385 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6386 SmallVector<Value *, 4> Args(CI->args());
6387 unsigned NumElts =
6388 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6389 Args[1] = Builder.CreateBitCast(
6390 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6391 Args[2] = Builder.CreateBitCast(
6392 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6393
6394 NewCall = Builder.CreateCall(NewFn, Args);
6395 break;
6396 }
6397
6398 case Intrinsic::thread_pointer: {
6399 NewCall = Builder.CreateCall(NewFn, {});
6400 break;
6401 }
6402
6403 case Intrinsic::memcpy:
6404 case Intrinsic::memmove:
6405 case Intrinsic::memset: {
6406 // We have to make sure that the call signature is what we're expecting.
6407 // We only want to change the old signatures by removing the alignment arg:
6408 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6409 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6410 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6411 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6412 // Note: i8*'s in the above can be any pointer type
6413 if (CI->arg_size() != 5) {
6414 DefaultCase();
6415 return;
6416 }
6417 // Remove alignment argument (3), and add alignment attributes to the
6418 // dest/src pointers.
6419 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6420 CI->getArgOperand(2), CI->getArgOperand(4)};
6421 NewCall = Builder.CreateCall(NewFn, Args);
6422 AttributeList OldAttrs = CI->getAttributes();
6423 AttributeList NewAttrs = AttributeList::get(
6424 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6425 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6426 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6427 NewCall->setAttributes(NewAttrs);
6428 auto *MemCI = cast<MemIntrinsic>(NewCall);
6429 // All mem intrinsics support dest alignment.
6430 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6431 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6432 // Memcpy/Memmove also support source alignment.
6433 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6434 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6435 break;
6436 }
6437
6438 case Intrinsic::masked_load:
6439 case Intrinsic::masked_gather:
6440 case Intrinsic::masked_store:
6441 case Intrinsic::masked_scatter: {
6442 if (CI->arg_size() != 4) {
6443 DefaultCase();
6444 return;
6445 }
6446
6447 auto GetMaybeAlign = [](Value *Op) {
6448 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6449 uint64_t Val = CI->getZExtValue();
6450 if (Val == 0)
6451 return MaybeAlign();
6452 if (isPowerOf2_64(Val))
6453 return MaybeAlign(Val);
6454 }
6455 reportFatalUsageError("Invalid alignment argument");
6456 };
6457 auto GetAlign = [&](Value *Op) {
6458 MaybeAlign Align = GetMaybeAlign(Op);
6459 if (Align)
6460 return *Align;
6461 reportFatalUsageError("Invalid zero alignment argument");
6462 };
6463
6464 const DataLayout &DL = CI->getDataLayout();
6465 switch (NewFn->getIntrinsicID()) {
6466 case Intrinsic::masked_load:
6467 NewCall = Builder.CreateMaskedLoad(
6468 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6469 CI->getArgOperand(2), CI->getArgOperand(3));
6470 break;
6471 case Intrinsic::masked_gather:
6472 NewCall = Builder.CreateMaskedGather(
6473 CI->getType(), CI->getArgOperand(0),
6474 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6475 CI->getType()->getScalarType()),
6476 CI->getArgOperand(2), CI->getArgOperand(3));
6477 break;
6478 case Intrinsic::masked_store:
6479 NewCall = Builder.CreateMaskedStore(
6480 CI->getArgOperand(0), CI->getArgOperand(1),
6481 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6482 break;
6483 case Intrinsic::masked_scatter:
6484 NewCall = Builder.CreateMaskedScatter(
6485 CI->getArgOperand(0), CI->getArgOperand(1),
6486 DL.getValueOrABITypeAlignment(
6487 GetMaybeAlign(CI->getArgOperand(2)),
6488 CI->getArgOperand(0)->getType()->getScalarType()),
6489 CI->getArgOperand(3));
6490 break;
6491 default:
6492 llvm_unreachable("Unexpected intrinsic ID");
6493 }
6494 // Previous metadata is still valid.
6495 NewCall->copyMetadata(*CI);
6496 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6497 break;
6498 }
6499
6500 case Intrinsic::lifetime_start:
6501 case Intrinsic::lifetime_end: {
6502 if (CI->arg_size() != 2) {
6503 DefaultCase();
6504 return;
6505 }
6506
6507 Value *Ptr = CI->getArgOperand(1);
6508 // Try to strip pointer casts, such that the lifetime works on an alloca.
6509 Ptr = Ptr->stripPointerCasts();
6510 if (isa<AllocaInst>(Ptr)) {
6511 // Don't use NewFn, as we might have looked through an addrspacecast.
6512 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6513 NewCall = Builder.CreateLifetimeStart(Ptr);
6514 else
6515 NewCall = Builder.CreateLifetimeEnd(Ptr);
6516 break;
6517 }
6518
6519 // Otherwise remove the lifetime marker.
6520 CI->eraseFromParent();
6521 return;
6522 }
6523
6524 case Intrinsic::x86_avx512_vpdpbusd_128:
6525 case Intrinsic::x86_avx512_vpdpbusd_256:
6526 case Intrinsic::x86_avx512_vpdpbusd_512:
6527 case Intrinsic::x86_avx512_vpdpbusds_128:
6528 case Intrinsic::x86_avx512_vpdpbusds_256:
6529 case Intrinsic::x86_avx512_vpdpbusds_512:
6530 case Intrinsic::x86_avx2_vpdpbssd_128:
6531 case Intrinsic::x86_avx2_vpdpbssd_256:
6532 case Intrinsic::x86_avx10_vpdpbssd_512:
6533 case Intrinsic::x86_avx2_vpdpbssds_128:
6534 case Intrinsic::x86_avx2_vpdpbssds_256:
6535 case Intrinsic::x86_avx10_vpdpbssds_512:
6536 case Intrinsic::x86_avx2_vpdpbsud_128:
6537 case Intrinsic::x86_avx2_vpdpbsud_256:
6538 case Intrinsic::x86_avx10_vpdpbsud_512:
6539 case Intrinsic::x86_avx2_vpdpbsuds_128:
6540 case Intrinsic::x86_avx2_vpdpbsuds_256:
6541 case Intrinsic::x86_avx10_vpdpbsuds_512:
6542 case Intrinsic::x86_avx2_vpdpbuud_128:
6543 case Intrinsic::x86_avx2_vpdpbuud_256:
6544 case Intrinsic::x86_avx10_vpdpbuud_512:
6545 case Intrinsic::x86_avx2_vpdpbuuds_128:
6546 case Intrinsic::x86_avx2_vpdpbuuds_256:
6547 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6548 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6549 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6550 CI->getArgOperand(2)};
6551 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6552 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6553 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6554
6555 NewCall = Builder.CreateCall(NewFn, Args);
6556 break;
6557 }
6558 case Intrinsic::x86_avx512_vpdpwssd_128:
6559 case Intrinsic::x86_avx512_vpdpwssd_256:
6560 case Intrinsic::x86_avx512_vpdpwssd_512:
6561 case Intrinsic::x86_avx512_vpdpwssds_128:
6562 case Intrinsic::x86_avx512_vpdpwssds_256:
6563 case Intrinsic::x86_avx512_vpdpwssds_512:
6564 case Intrinsic::x86_avx2_vpdpwsud_128:
6565 case Intrinsic::x86_avx2_vpdpwsud_256:
6566 case Intrinsic::x86_avx10_vpdpwsud_512:
6567 case Intrinsic::x86_avx2_vpdpwsuds_128:
6568 case Intrinsic::x86_avx2_vpdpwsuds_256:
6569 case Intrinsic::x86_avx10_vpdpwsuds_512:
6570 case Intrinsic::x86_avx2_vpdpwusd_128:
6571 case Intrinsic::x86_avx2_vpdpwusd_256:
6572 case Intrinsic::x86_avx10_vpdpwusd_512:
6573 case Intrinsic::x86_avx2_vpdpwusds_128:
6574 case Intrinsic::x86_avx2_vpdpwusds_256:
6575 case Intrinsic::x86_avx10_vpdpwusds_512:
6576 case Intrinsic::x86_avx2_vpdpwuud_128:
6577 case Intrinsic::x86_avx2_vpdpwuud_256:
6578 case Intrinsic::x86_avx10_vpdpwuud_512:
6579 case Intrinsic::x86_avx2_vpdpwuuds_128:
6580 case Intrinsic::x86_avx2_vpdpwuuds_256:
6581 case Intrinsic::x86_avx10_vpdpwuuds_512:
6582 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6583 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6584 CI->getArgOperand(2)};
6585 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6586 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6587 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6588
6589 NewCall = Builder.CreateCall(NewFn, Args);
6590 break;
6591 }
6592 assert(NewCall && "Should have either set this variable or returned through "
6593 "the default case");
6594 NewCall->takeName(CI);
6595 CI->replaceAllUsesWith(NewCall);
6596 CI->eraseFromParent();
6597}
6598
6599void llvm::UpgradeCallsToIntrinsic(Function *F) {
6600 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6601
6602 // Check if this function should be upgraded and get the replacement function
6603 // if there is one.
6604 Function *NewFn;
6605 if (UpgradeIntrinsicFunction(F, NewFn)) {
6606 // Replace all users of the old function with the new function or new
6607 // instructions. This is not a range loop because the call is deleted.
6608 for (User *U : make_early_inc_range(F->users()))
6609 if (CallBase *CB = dyn_cast<CallBase>(U))
6610 UpgradeIntrinsicCall(CB, NewFn);
6611
6612 // Remove old function, no longer used, from the module.
6613 if (F != NewFn)
6614 F->eraseFromParent();
6615 }
6616}
6617
6618MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6619 const unsigned NumOperands = MD.getNumOperands();
6620 if (NumOperands == 0)
6621 return &MD; // Invalid, punt to a verifier error.
6622
6623 // Check if the tag uses struct-path aware TBAA format.
6624 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6625 return &MD;
6626
6627 auto &Context = MD.getContext();
6628 if (NumOperands == 3) {
6629 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6630 MDNode *ScalarType = MDNode::get(Context, Elts);
6631 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6632 Metadata *Elts2[] = {ScalarType, ScalarType,
6633 ConstantAsMetadata::get(
6634 Constant::getNullValue(Type::getInt64Ty(Context))),
6635 MD.getOperand(2)};
6636 return MDNode::get(Context, Elts2);
6637 }
6638 // Create a MDNode <MD, MD, offset 0>
6639 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6640 Type::getInt64Ty(Context)))};
6641 return MDNode::get(Context, Elts);
6642}
6643
6644Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6645 Instruction *&Temp) {
6646 if (Opc != Instruction::BitCast)
6647 return nullptr;
6648
6649 Temp = nullptr;
6650 Type *SrcTy = V->getType();
6651 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6652 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6653 LLVMContext &Context = V->getContext();
6654
6655 // We have no information about target data layout, so we assume that
6656 // the maximum pointer size is 64bit.
6657 Type *MidTy = Type::getInt64Ty(Context);
6658 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6659
6660 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6661 }
6662
6663 return nullptr;
6664}
6665
6666Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6667 if (Opc != Instruction::BitCast)
6668 return nullptr;
6669
6670 Type *SrcTy = C->getType();
6671 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6672 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6673 LLVMContext &Context = C->getContext();
6674
6675 // We have no information about target data layout, so we assume that
6676 // the maximum pointer size is 64bit.
6677 Type *MidTy = Type::getInt64Ty(Context);
6678
6679 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6680 DestTy);
6681 }
6682
6683 return nullptr;
6684}
6685
6686static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6687 if (Flag.getNumOperands() < 3)
6688 return std::nullopt;
6689 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6690 return Name->getString();
6691 return std::nullopt;
6692}
6693
6694/// Check the debug info version number, if it is out-dated, drop the debug
6695/// info. Return true if module is modified.
6696bool llvm::UpgradeDebugInfo(Module &M) {
6697 if (DisableAutoUpgradeDebugInfo)
6698 return false;
6699
6700 llvm::TimeTraceScope timeScope("Upgrade debug info");
6701 // We need to get metadata before the module is verified (i.e., getModuleFlag
6702 // makes assumptions that we haven't verified yet). Carefully extract the flag
6703 // from the metadata.
6704 unsigned Version = 0;
6705 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6706 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6707 if (auto Name = getModuleFlagNameSafely(*Flag))
6708 return *Name == "Debug Info Version";
6709 return false;
6710 });
6711 if (OpIt != ModFlags->op_end()) {
6712 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6713 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6714 Version = CI->getZExtValue();
6715 }
6716 }
6717
6718 if (Version == DEBUG_METADATA_VERSION) {
6719 bool BrokenDebugInfo = false;
6720 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6721 report_fatal_error("Broken module found, compilation aborted!");
6722 if (!BrokenDebugInfo)
6723 // Everything is ok.
6724 return false;
6725 else {
6726 // Diagnose malformed debug info.
6727 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6728 M.getContext().diagnose(Diag);
6729 }
6730 }
6731 bool Modified = StripDebugInfo(M);
6732 if (Modified && Version != DEBUG_METADATA_VERSION) {
6733 // Diagnose a version mismatch.
6734 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6735 M.getContext().diagnose(DiagVersion);
6736 }
6737 return Modified;
6738}
6739
6740static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6741 GlobalValue *GV, const Metadata *V) {
6742 Function *F = cast<Function>(GV);
6743
6744 constexpr StringLiteral DefaultValue = "1";
6745 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6746 unsigned Length = 0;
6747
6748 if (F->hasFnAttribute(Attr)) {
6749 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6750 // parse these elements placing them into Vect3
6751 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6752 for (; Length < 3 && !S.empty(); Length++) {
6753 auto [Part, Rest] = S.split(',');
6754 Vect3[Length] = Part.trim();
6755 S = Rest;
6756 }
6757 }
6758
6759 const unsigned Dim = DimC - 'x';
6760 assert(Dim < 3 && "Unexpected dim char");
6761
6762 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6763
6764 // local variable required for StringRef in Vect3 to point to.
6765 const std::string VStr = llvm::utostr(VInt);
6766 Vect3[Dim] = VStr;
6767 Length = std::max(Length, Dim + 1);
6768
6769 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6770 F->addFnAttr(Attr, NewAttr);
6771}
6772
6773static inline bool isXYZ(StringRef S) {
6774 return S == "x" || S == "y" || S == "z";
6775}
6776
6777bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6778 const Metadata *V) {
6779 if (K == "kernel") {
6780 if (!mdconst::extract<ConstantInt>(V)->isZero())
6781 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6782 return true;
6783 }
6784 if (K == "align") {
6785 // V is a bitfeild specifying two 16-bit values. The alignment value is
6786 // specfied in low 16-bits, The index is specified in the high bits. For the
6787 // index, 0 indicates the return value while higher values correspond to
6788 // each parameter (idx = param + 1).
6789 const uint64_t AlignIdxValuePair =
6790 mdconst::extract<ConstantInt>(V)->getZExtValue();
6791 const unsigned Idx = (AlignIdxValuePair >> 16);
6792 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6793 cast<Function>(GV)->addAttributeAtIndex(
6794 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6795 return true;
6796 }
6797 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6798 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6799 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6800 return true;
6801 }
6802 if (K == "minctasm") {
6803 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6804 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6805 return true;
6806 }
6807 if (K == "maxnreg") {
6808 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6809 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6810 return true;
6811 }
6812 if (K.consume_front("maxntid") && isXYZ(K)) {
6813 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6814 return true;
6815 }
6816 if (K.consume_front("reqntid") && isXYZ(K)) {
6817 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6818 return true;
6819 }
6820 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6821 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6822 return true;
6823 }
6824 if (K == "grid_constant") {
6825 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6826 for (const auto &Op : cast<MDNode>(V)->operands()) {
6827 // For some reason, the index is 1-based in the metadata. Good thing we're
6828 // able to auto-upgrade it!
6829 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6830 cast<Function>(GV)->addParamAttr(Index, Attr);
6831 }
6832 return true;
6833 }
6834
6835 return false;
6836}
6837
6838void llvm::UpgradeNVVMAnnotations(Module &M) {
6839 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6840 if (!NamedMD)
6841 return;
6842
6843 SmallVector<MDNode *, 8> NewNodes;
6844 SmallPtrSet<const MDNode *, 8> SeenNodes;
6845 for (MDNode *MD : NamedMD->operands()) {
6846 if (!SeenNodes.insert(MD).second)
6847 continue;
6848
6849 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6850 if (!GV)
6851 continue;
6852
6853 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6854
6855 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6856 // Each nvvm.annotations metadata entry will be of the following form:
6857 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6858 // start index = 1, to skip the global variable key
6859 // increment = 2, to skip the value for each property-value pairs
6860 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6861 MDString *K = cast<MDString>(MD->getOperand(j));
6862 const MDOperand &V = MD->getOperand(j + 1);
6863 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6864 if (!Upgraded)
6865 NewOperands.append({K, V});
6866 }
6867
6868 if (NewOperands.size() > 1)
6869 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6870 }
6871
6872 NamedMD->clearOperands();
6873 for (MDNode *N : NewNodes)
6874 NamedMD->addOperand(N);
6875}
6876
6877/// This checks for objc retain release marker which should be upgraded. It
6878/// returns true if module is modified.
6879static bool upgradeRetainReleaseMarker(Module &M) {
6880 bool Changed = false;
6881 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6882 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6883 if (ModRetainReleaseMarker) {
6884 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6885 if (Op) {
6886 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6887 if (ID) {
6888 SmallVector<StringRef, 4> ValueComp;
6889 ID->getString().split(ValueComp, "#");
6890 if (ValueComp.size() == 2) {
6891 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6892 ID = MDString::get(M.getContext(), NewValue);
6893 }
6894 M.addModuleFlag(Module::Error, MarkerKey, ID);
6895 M.eraseNamedMetadata(ModRetainReleaseMarker);
6896 Changed = true;
6897 }
6898 }
6899 }
6900 return Changed;
6901}
6902
6903void llvm::UpgradeARCRuntime(Module &M) {
6904 // This lambda converts normal function calls to ARC runtime functions to
6905 // intrinsic calls.
6906 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6907 llvm::Intrinsic::ID IntrinsicFunc) {
6908 Function *Fn = M.getFunction(OldFunc);
6909
6910 if (!Fn)
6911 return;
6912
6913 Function *NewFn =
6914 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6915
6916 for (User *U : make_early_inc_range(Fn->users())) {
6917 CallInst *CI = dyn_cast<CallInst>(U);
6918 if (!CI || CI->getCalledFunction() != Fn)
6919 continue;
6920
6921 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6922 FunctionType *NewFuncTy = NewFn->getFunctionType();
6923 SmallVector<Value *, 2> Args;
6924
6925 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6926 // value to the return type of the old function.
6927 if (NewFuncTy->getReturnType() != CI->getType() &&
6928 !CastInst::castIsValid(Instruction::BitCast, CI,
6929 NewFuncTy->getReturnType()))
6930 continue;
6931
6932 bool InvalidCast = false;
6933
6934 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6935 Value *Arg = CI->getArgOperand(I);
6936
6937 // Bitcast argument to the parameter type of the new function if it's
6938 // not a variadic argument.
6939 if (I < NewFuncTy->getNumParams()) {
6940 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6941 // to the parameter type of the new function.
6942 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6943 NewFuncTy->getParamType(I))) {
6944 InvalidCast = true;
6945 break;
6946 }
6947 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6948 }
6949 Args.push_back(Arg);
6950 }
6951
6952 if (InvalidCast)
6953 continue;
6954
6955 // Create a call instruction that calls the new function.
6956 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6957 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6958 NewCall->takeName(CI);
6959
6960 // Bitcast the return value back to the type of the old call.
6961 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6962
6963 if (!CI->use_empty())
6964 CI->replaceAllUsesWith(NewRetVal);
6965 CI->eraseFromParent();
6966 }
6967
6968 if (Fn->use_empty())
6969 Fn->eraseFromParent();
6970 };
6971
6972 // Unconditionally convert a call to "clang.arc.use" to a call to
6973 // "llvm.objc.clang.arc.use".
6974 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6975
6976 // Upgrade the retain release marker. If there is no need to upgrade
6977 // the marker, that means either the module is already new enough to contain
6978 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6979 if (!upgradeRetainReleaseMarker(M))
6980 return;
6981
6982 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6983 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6984 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6985 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6986 {"objc_autoreleaseReturnValue",
6987 llvm::Intrinsic::objc_autoreleaseReturnValue},
6988 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6989 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6990 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6991 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6992 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6993 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6994 {"objc_release", llvm::Intrinsic::objc_release},
6995 {"objc_retain", llvm::Intrinsic::objc_retain},
6996 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6997 {"objc_retainAutoreleaseReturnValue",
6998 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6999 {"objc_retainAutoreleasedReturnValue",
7000 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7001 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7002 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7003 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7004 {"objc_unsafeClaimAutoreleasedReturnValue",
7005 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7006 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7007 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7008 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7009 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7010 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7011 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7012 {"objc_arc_annotation_topdown_bbstart",
7013 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7014 {"objc_arc_annotation_topdown_bbend",
7015 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7016 {"objc_arc_annotation_bottomup_bbstart",
7017 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7018 {"objc_arc_annotation_bottomup_bbend",
7019 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7020
7021 for (auto &I : RuntimeFuncs)
7022 UpgradeToIntrinsic(I.first, I.second);
7023}
7024
7025// Upgrade the way signing of pointers to init/fini functions is described.
7026//
7027// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7028// constants, if signing was requested. After the upgrade, these arrays contain
7029// plain function pointers and the desired signing schema is described via a
7030// pair of module flags.
7031//
7032// Note that the upgrade is only performed if all elements of *both* arrays
7033// agree on a common signing schema.
7034static bool upgradePtrauthInitFiniArrays(Module &M) {
7035 // As we cannot always decide whether the particular module should have
7036 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7037 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7038 // returns as soon as it spots any non-signed init/fini pointer: either we
7039 // should request non-signed pointers (safe to omit both flags) or there is
7040 // no common schema (and thus we do not modify anything).
7041 //
7042 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7043 // or whether we should request address diversity in addition to the basic
7044 // constant diversity. There is no value representing "decided not to sign"
7045 // for the reasons explained above.
7046 std::optional<bool> UseAddressDisc;
7047
7048 // Do not attempt upgrading if the new module flags already exist.
7049 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7050 for (const MDNode *Flag : ModFlags->operands()) {
7051 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7052 if (Name && (*Name == "ptrauth-init-fini" ||
7053 *Name == "ptrauth-init-fini-address-discrimination"))
7054 return false;
7055 }
7056 }
7057
7058 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7059 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7060 constexpr unsigned ExpectedAddressMarker = 1;
7061
7062 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7063 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7064 return nullptr; // Nothing to upgrade or unknown pattern found.
7065
7066 bool HasAddressDisc;
7067 if (!CPA->hasAddressDiscriminator())
7068 HasAddressDisc = false;
7069 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7070 HasAddressDisc = true;
7071 else
7072 return nullptr; // Unknown pattern.
7073
7074 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7075 return nullptr; // Disagreement with the decided mode.
7076
7077 UseAddressDisc = HasAddressDisc;
7078 return CPA->getPointer();
7079 };
7080
7081 // Do not apply any changes until we know the upgrade is non-ambiguous.
7082 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7083 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7084
7085 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7086 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7087 if (!GV || !GV->hasInitializer())
7088 continue; // Skip, but it is okay to upgrade the other variable.
7089
7090 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7091 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7092 return false;
7093
7094 std::vector<Constant *> NewStructors;
7095 NewStructors.reserve(OldStructorsArray->getNumOperands());
7096
7097 for (Use &U : OldStructorsArray->operands()) {
7098 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7099 if (!Structor || Structor->getNumOperands() != 3)
7100 return false;
7101
7102 Constant *Prio = Structor->getOperand(0);
7103 Constant *Func = Structor->getOperand(1);
7104 Constant *Arg = Structor->getOperand(2);
7105
7106 Func = UpgradeSinglePointer(Func);
7107 if (!Func)
7108 return false;
7109
7110 NewStructors.push_back(
7111 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7112 }
7113
7114 Constant *NewInit =
7115 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7116 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7117 }
7118
7119 if (GlobalArraysToUpgrade.empty())
7120 return false;
7121 assert(UseAddressDisc.has_value());
7122
7123 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7124 GV->setInitializer(NewInit);
7125
7126 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7127 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7128 *UseAddressDisc);
7129
7130 return true;
7131}
7132
7133bool llvm::UpgradeModuleFlags(Module &M) {
7134 bool Changed = false;
7135 Changed |= upgradePtrauthInitFiniArrays(M);
7136
7137 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7138 if (!ModFlags)
7139 return Changed;
7140
7141 bool HasObjCFlag = false, HasClassProperties = false;
7142 bool HasSwiftVersionFlag = false;
7143 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7144 uint32_t SwiftABIVersion;
7145 auto Int8Ty = Type::getInt8Ty(M.getContext());
7146 auto Int32Ty = Type::getInt32Ty(M.getContext());
7147
7148 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7149 MDNode *Op = ModFlags->getOperand(I);
7150 if (Op->getNumOperands() != 3)
7151 continue;
7152 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7153 if (!ID)
7154 continue;
7155 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7156 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7157 Type::getInt32Ty(M.getContext()), B)),
7158 MDString::get(M.getContext(), ID->getString()),
7159 Op->getOperand(2)};
7160 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7161 Changed = true;
7162 };
7163
7164 if (ID->getString() == "Objective-C Image Info Version")
7165 HasObjCFlag = true;
7166 if (ID->getString() == "Objective-C Class Properties")
7167 HasClassProperties = true;
7168 // Upgrade PIC from Error/Max to Min.
7169 if (ID->getString() == "PIC Level") {
7170 if (auto *Behavior =
7171 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7172 uint64_t V = Behavior->getLimitedValue();
7173 if (V == Module::Error || V == Module::Max)
7174 SetBehavior(Module::Min);
7175 }
7176 }
7177 // Upgrade "PIE Level" from Error to Max.
7178 if (ID->getString() == "PIE Level")
7179 if (auto *Behavior =
7180 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7181 if (Behavior->getLimitedValue() == Module::Error)
7182 SetBehavior(Module::Max);
7183
7184 // Upgrade branch protection and return address signing module flags. The
7185 // module flag behavior for these fields were Error and now they are Min.
7186 if (ID->getString() == "branch-target-enforcement" ||
7187 ID->getString().starts_with("sign-return-address")) {
7188 if (auto *Behavior =
7189 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7190 if (Behavior->getLimitedValue() == Module::Error) {
7191 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7192 Metadata *Ops[3] = {
7193 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7194 Op->getOperand(1), Op->getOperand(2)};
7195 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7196 Changed = true;
7197 }
7198 }
7199 }
7200
7201 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7202 // section name so that llvm-lto will not complain about mismatching
7203 // module flags that is functionally the same.
7204 if (ID->getString() == "Objective-C Image Info Section") {
7205 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7206 SmallVector<StringRef, 4> ValueComp;
7207 Value->getString().split(ValueComp, " ");
7208 if (ValueComp.size() != 1) {
7209 std::string NewValue;
7210 for (auto &S : ValueComp)
7211 NewValue += S.str();
7212 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7213 MDString::get(M.getContext(), NewValue)};
7214 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7215 Changed = true;
7216 }
7217 }
7218 }
7219
7220 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7221 // If the higher bits are set, it adds new module flag for swift info.
7222 if (ID->getString() == "Objective-C Garbage Collection") {
7223 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7224 if (Md) {
7225 assert(Md->getValue() && "Expected non-empty metadata");
7226 auto Type = Md->getValue()->getType();
7227 if (Type == Int8Ty)
7228 continue;
7229 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7230 if ((Val & 0xff) != Val) {
7231 HasSwiftVersionFlag = true;
7232 SwiftABIVersion = (Val & 0xff00) >> 8;
7233 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7234 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7235 }
7236 Metadata *Ops[3] = {
7237 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7238 Op->getOperand(1),
7239 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7240 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7241 Changed = true;
7242 }
7243 }
7244
7245 if (ID->getString() == "amdgpu_code_object_version") {
7246 Metadata *Ops[3] = {
7247 Op->getOperand(0),
7248 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7249 Op->getOperand(2)};
7250 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7251 Changed = true;
7252 }
7253
7254 // clang/PowerPC used to use "float-abi" to describe the long double format;
7255 // it has been renamed to "long-double-type", with its values changed to the
7256 // corresponding IR floating-point type names.
7257 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7258 StringRef Format;
7259 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7260 Format = S->getString();
7261
7262 // The "float-abi" key is now reserved for the target-independent
7263 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7264 // (including unrecognized ones, which were never valid) to the default.
7265 if (!FloatABI::parseABIType(Format)) {
7266 LongDoubleFormat NewFormat =
7267 StringSwitch<LongDoubleFormat>(Format)
7268 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7269 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7270 .Default(LongDoubleFormat::PPCDoubleDouble);
7271 Metadata *Ops[3] = {
7272 Op->getOperand(0),
7273 MDString::get(M.getContext(), "long-double-type"),
7274 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7275 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7276 Changed = true;
7277 }
7278 }
7279 }
7280
7281 // "Objective-C Class Properties" is recently added for Objective-C. We
7282 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7283 // flag of value 0, so we can correclty downgrade this flag when trying to
7284 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7285 // this module flag.
7286 if (HasObjCFlag && !HasClassProperties) {
7287 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7288 (uint32_t)0);
7289 Changed = true;
7290 }
7291
7292 if (HasSwiftVersionFlag) {
7293 M.addModuleFlag(Module::Error, "Swift ABI Version",
7294 SwiftABIVersion);
7295 M.addModuleFlag(Module::Error, "Swift Major Version",
7296 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7297 M.addModuleFlag(Module::Error, "Swift Minor Version",
7298 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7299 Changed = true;
7300 }
7301
7302 return Changed;
7303}
7304
7305bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7306 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7307 // If this metadata has operands, we expect all of them to be either from
7308 // before or from after the format change handled here, so we can bail out
7309 // fast if the first (if any) operands is of the new format.
7310 auto MatchesVersion = [](const MDNode *Op) {
7311 return Op->getNumOperands() >= 3 &&
7312 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7313 cast<ConstantAsMetadata>(Op->getOperand(2))
7314 ->getType()
7315 ->isIntegerTy(64);
7316 };
7317
7318 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7319 MatchesVersion(CFIConsts->getOperand(0)))
7320 return false;
7321
7322 bool Changed = false;
7323 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7324 MDNode *Op = CFIConsts->getOperand(I);
7325 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7326 assert(Op->getNumOperands() >= 2 &&
7327 "Expected at least 2 operands - name and linkage type");
7328 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7329 StringRef Name = NameMD->getString();
7330 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7331 GlobalValue::dropLLVMManglingEscape(Name));
7332
7333 SmallVector<Metadata *, 4> Elts;
7334 Elts.push_back(Op->getOperand(0));
7335 Elts.push_back(Op->getOperand(1));
7336 Elts.push_back(ConstantAsMetadata::get(
7337 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7338
7339 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7340 Elts.push_back(Op->getOperand(J));
7341
7342 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7343 Changed = true;
7344 }
7345
7346 return Changed;
7347}
7348
7349void llvm::UpgradeSectionAttributes(Module &M) {
7350 auto TrimSpaces = [](StringRef Section) -> std::string {
7351 SmallVector<StringRef, 5> Components;
7352 Section.split(Components, ',');
7353
7354 SmallString<32> Buffer;
7355 raw_svector_ostream OS(Buffer);
7356
7357 for (auto Component : Components)
7358 OS << ',' << Component.trim();
7359
7360 return std::string(OS.str().substr(1));
7361 };
7362
7363 for (auto &GV : M.globals()) {
7364 if (!GV.hasSection())
7365 continue;
7366
7367 StringRef Section = GV.getSection();
7368
7369 if (!Section.starts_with("__DATA, __objc_catlist"))
7370 continue;
7371
7372 // __DATA, __objc_catlist, regular, no_dead_strip
7373 // __DATA,__objc_catlist,regular,no_dead_strip
7374 GV.setSection(TrimSpaces(Section));
7375 }
7376}
7377
7378namespace {
7379// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7380// callsites within a function that did not also have the strictfp attribute.
7381// Since 10.0, if strict FP semantics are needed within a function, the
7382// function must have the strictfp attribute and all calls within the function
7383// must also have the strictfp attribute. This latter restriction is
7384// necessary to prevent unwanted libcall simplification when a function is
7385// being cloned (such as for inlining).
7386//
7387// The "dangling" strictfp attribute usage was only used to prevent constant
7388// folding and other libcall simplification. The nobuiltin attribute on the
7389// callsite has the same effect.
7390struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7391 StrictFPUpgradeVisitor() = default;
7392
7393 void visitCallBase(CallBase &Call) {
7394 if (!Call.isStrictFP())
7395 return;
7396 if (isa<ConstrainedFPIntrinsic>(&Call))
7397 return;
7398 // If we get here, the caller doesn't have the strictfp attribute
7399 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7400 Call.removeFnAttr(Attribute::StrictFP);
7401 Call.addFnAttr(Attribute::NoBuiltin);
7402 }
7403};
7404
7405/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7406struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7407 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7408 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7409
7410 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7411 if (!RMW.isFloatingPointOperation())
7412 return;
7413
7414 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7415 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7416 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7417 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7418 }
7419};
7420} // namespace
7421
7422void llvm::UpgradeFunctionAttributes(Function &F) {
7423 // If a function definition doesn't have the strictfp attribute,
7424 // convert any callsite strictfp attributes to nobuiltin.
7425 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7426 StrictFPUpgradeVisitor SFPV;
7427 SFPV.visit(F);
7428 }
7429
7430 // Remove all incompatibile attributes from function.
7431 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7432 F.getReturnType(), F.getAttributes().getRetAttrs()));
7433 for (auto &Arg : F.args())
7434 Arg.removeAttrs(
7435 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7436
7437 bool AddingAttrs = false, RemovingAttrs = false;
7438 AttrBuilder AttrsToAdd(F.getContext());
7439 AttributeMask AttrsToRemove;
7440
7441 // Older versions of LLVM treated an "implicit-section-name" attribute
7442 // similarly to directly setting the section on a Function.
7443 if (Attribute A = F.getFnAttribute("implicit-section-name");
7444 A.isValid() && A.isStringAttribute()) {
7445 F.setSection(A.getValueAsString());
7446 AttrsToRemove.addAttribute("implicit-section-name");
7447 RemovingAttrs = true;
7448 }
7449
7450 if (Attribute A = F.getFnAttribute("nooutline");
7451 A.isValid() && A.isStringAttribute()) {
7452 AttrsToRemove.addAttribute("nooutline");
7453 AttrsToAdd.addAttribute(Attribute::NoOutline);
7454 AddingAttrs = RemovingAttrs = true;
7455 }
7456
7457 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7458 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7459 AttrsToRemove.addAttribute("uniform-work-group-size");
7460 RemovingAttrs = true;
7461 if (A.getValueAsString() == "true") {
7462 AttrsToAdd.addAttribute("uniform-work-group-size");
7463 AddingAttrs = true;
7464 }
7465 }
7466
7467 if (!F.empty()) {
7468 // For some reason this is called twice, and the first time is before any
7469 // instructions are loaded into the body.
7470
7471 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7472 A.isValid()) {
7473
7474 if (A.getValueAsBool()) {
7475 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7476 Visitor.visit(F);
7477 }
7478
7479 // We will leave behind dead attribute uses on external declarations, but
7480 // clang never added these to declarations anyway.
7481 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7482 RemovingAttrs = true;
7483 }
7484 }
7485
7486 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7487 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7488
7489 bool HandleDenormalMode = false;
7490
7491 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7492 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7493 if (ParsedMode.isValid()) {
7494 DenormalFPMath = ParsedMode;
7495 AttrsToRemove.addAttribute("denormal-fp-math");
7496 AddingAttrs = RemovingAttrs = true;
7497 HandleDenormalMode = true;
7498 }
7499 }
7500
7501 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7502 Attr.isValid()) {
7503 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7504 if (ParsedMode.isValid()) {
7505 DenormalFPMathF32 = ParsedMode;
7506 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7507 AddingAttrs = RemovingAttrs = true;
7508 HandleDenormalMode = true;
7509 }
7510 }
7511
7512 if (HandleDenormalMode)
7513 AttrsToAdd.addDenormalFPEnvAttr(
7514 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7515
7516 if (RemovingAttrs)
7517 F.removeFnAttrs(AttrsToRemove);
7518
7519 if (AddingAttrs)
7520 F.addFnAttrs(AttrsToAdd);
7521}
7522
7523// Check if the function attribute is not present and set it.
7524static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7525 StringRef Value) {
7526 if (!F.hasFnAttribute(FnAttrName))
7527 F.addFnAttr(FnAttrName, Value);
7528}
7529
7530// Check if the function attribute is not present and set it if needed.
7531// If the attribute is "false" then removes it.
7532// If the attribute is "true" resets it to a valueless attribute.
7533static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7534 if (!F.hasFnAttribute(FnAttrName)) {
7535 if (Set)
7536 F.addFnAttr(FnAttrName);
7537 } else {
7538 auto A = F.getFnAttribute(FnAttrName);
7539 if ("false" == A.getValueAsString())
7540 F.removeFnAttr(FnAttrName);
7541 else if ("true" == A.getValueAsString()) {
7542 F.removeFnAttr(FnAttrName);
7543 F.addFnAttr(FnAttrName);
7544 }
7545 }
7546}
7547
7548void llvm::copyModuleAttrToFunctions(Module &M) {
7549 Triple T(M.getTargetTriple());
7550 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7551 return;
7552
7553 uint64_t BTEValue = 0;
7554 uint64_t BPPLRValue = 0;
7555 uint64_t GCSValue = 0;
7556 uint64_t SRAValue = 0;
7557 uint64_t SRAALLValue = 0;
7558 uint64_t SRABKeyValue = 0;
7559
7560 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7561 if (ModFlags) {
7562 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7563 MDNode *Op = ModFlags->getOperand(I);
7564 if (Op->getNumOperands() != 3)
7565 continue;
7566
7567 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7568 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7569 if (!ID || !CI)
7570 continue;
7571
7572 StringRef IDStr = ID->getString();
7573 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7574 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7575 : IDStr == "guarded-control-stack" ? &GCSValue
7576 : IDStr == "sign-return-address" ? &SRAValue
7577 : IDStr == "sign-return-address-all" ? &SRAALLValue
7578 : IDStr == "sign-return-address-with-bkey"
7579 ? &SRABKeyValue
7580 : nullptr;
7581 if (!ValPtr)
7582 continue;
7583
7584 *ValPtr = CI->getZExtValue();
7585 if (*ValPtr == 2)
7586 return;
7587 }
7588 }
7589
7590 bool BTE = BTEValue == 1;
7591 bool BPPLR = BPPLRValue == 1;
7592 bool GCS = GCSValue == 1;
7593 bool SRA = SRAValue == 1;
7594
7595 StringRef SignTypeValue = "non-leaf";
7596 if (SRA && SRAALLValue == 1)
7597 SignTypeValue = "all";
7598
7599 StringRef SignKeyValue = "a_key";
7600 if (SRA && SRABKeyValue == 1)
7601 SignKeyValue = "b_key";
7602
7603 for (Function &F : M.getFunctionList()) {
7604 if (F.isDeclaration())
7605 continue;
7606
7607 if (SRA) {
7608 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7609 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7610 } else {
7611 if (auto A = F.getFnAttribute("sign-return-address");
7612 A.isValid() && "none" == A.getValueAsString()) {
7613 F.removeFnAttr("sign-return-address");
7614 F.removeFnAttr("sign-return-address-key");
7615 }
7616 }
7617 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7618 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7619 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7620 }
7621
7622 if (BTE)
7623 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7624 if (BPPLR)
7625 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7626 if (GCS)
7627 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7628 if (SRA) {
7629 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7630 if (SRAALLValue == 1)
7631 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7632 if (SRABKeyValue == 1)
7633 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7634 }
7635}
7636
7637/// Return the replacement tags if \p T still uses a removed two-operand form.
7638static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7639 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7640 return nullptr;
7641 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7642 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7643}
7644
7645/// Build the single-operand node that replaces a boolean operand: nonzero
7646/// selects the enable tag, zero the disable tag.
7647static Metadata *makeBooleanLoopNode(LLVMContext &C,
7648 const BooleanLoopTags &Tags,
7649 const MDOperand &Op) {
7650 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7651 return MDTuple::get(C,
7652 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7653}
7654
7655static bool isOldLoopArgument(Metadata *MD) {
7656 auto *T = dyn_cast_or_null<MDTuple>(MD);
7657 if (!T)
7658 return false;
7659 if (T->getNumOperands() < 1)
7660 return false;
7661 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7662 if (!S)
7663 return false;
7664 if (S->getString().starts_with("llvm.vectorizer."))
7665 return true;
7666 return getOldBooleanLoopTags(T) != nullptr;
7667}
7668
7669static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7670 StringRef OldPrefix = "llvm.vectorizer.";
7671 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7672
7673 if (OldTag == "llvm.vectorizer.unroll")
7674 return MDString::get(C, "llvm.loop.interleave.count");
7675
7676 return MDString::get(
7677 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7678 .str());
7679}
7680
7681static Metadata *upgradeLoopArgument(Metadata *MD) {
7682 auto *T = dyn_cast_or_null<MDTuple>(MD);
7683 if (!T)
7684 return MD;
7685 if (T->getNumOperands() < 1)
7686 return MD;
7687 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7688 if (!OldTag)
7689 return MD;
7690
7691 LLVMContext &C = T->getContext();
7692
7693 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7694 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7695 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7696
7697 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7698 return MD;
7699
7700 // This has an old tag. Upgrade it.
7701 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7702
7703 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7704 // vectorize.enable/disable pair, not a two-operand enable node.
7705 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7706 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7707 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7708
7709 SmallVector<Metadata *, 8> Ops;
7710 Ops.reserve(T->getNumOperands());
7711 Ops.push_back(NewTag);
7712 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7713 Ops.push_back(T->getOperand(I));
7714
7715 return MDTuple::get(C, Ops);
7716}
7717
7718MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7719 auto *T = dyn_cast<MDTuple>(&N);
7720 if (!T)
7721 return &N;
7722
7723 if (none_of(T->operands(), isOldLoopArgument))
7724 return &N;
7725
7726 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7727 // any MDNode carrying those tags with more than one operand, so a leftover
7728 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7729 // In-place mutation is safe on distinct MDNodes.
7730 if (T->isDistinct()) {
7731 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7732 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7733 if (OpT && getOldBooleanLoopTags(OpT))
7734 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7735 }
7736 if (none_of(T->operands(), isOldLoopArgument))
7737 return &N;
7738 }
7739
7740 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7741 // attachment; the original distinct loop-ID is kept as the first operand.
7742 SmallVector<Metadata *, 8> Ops;
7743 Ops.reserve(T->getNumOperands());
7744 for (Metadata *MD : T->operands())
7745 Ops.push_back(upgradeLoopArgument(MD));
7746
7747 return MDTuple::get(T->getContext(), Ops);
7748}
7749
7750std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7751 Triple T(TT);
7752 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7753 // the address space of globals to 1. This does not apply to SPIRV Logical.
7754 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7755 !DL.contains("-G") && !DL.starts_with("G")) {
7756 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7757 }
7758
7759 if (T.isLoongArch64() || T.isRISCV64()) {
7760 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7761 auto I = DL.find("-n64-");
7762 if (I != StringRef::npos)
7763 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7764 return DL.str();
7765 }
7766
7767 // AMDGPU data layout upgrades.
7768 std::string Res = DL.str();
7769 if (T.isAMDGPU()) {
7770 // Define address spaces for constants.
7771 if (!DL.contains("-G") && !DL.starts_with("G"))
7772 Res.append(Res.empty() ? "G1" : "-G1");
7773
7774 // AMDGCN data layout upgrades.
7775 if (T.isAMDGCN()) {
7776
7777 // Add missing non-integral declarations.
7778 // This goes before adding new address spaces to prevent incoherent string
7779 // values.
7780 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7781 Res.append("-ni:7:8:9");
7782 // Update ni:7 to ni:7:8:9.
7783 if (DL.ends_with("ni:7"))
7784 Res.append(":8:9");
7785 if (DL.ends_with("ni:7:8"))
7786 Res.append(":9");
7787
7788 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7789 // resources) An empty data layout has already been upgraded to G1 by now.
7790 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7791 Res.append("-p7:160:256:256:32");
7792 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7793 Res.append("-p8:128:128:128:48");
7794 constexpr StringRef OldP8("-p8:128:128-");
7795 if (DL.contains(OldP8))
7796 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7797 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7798 Res.append("-p9:192:256:256:32");
7799
7800 // Add sizing for address space 10 through 15.
7801 // AS 10-14 are reserved and defaulted to 32:32
7802 // AS 15 is in use and is 32:32.
7803 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7804 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
7805 Res.append(("-" + AS + ":32:32").str());
7806 }
7807 }
7808
7809 // Upgrade the ELF mangling mode.
7810 if (!DL.contains("m:e"))
7811 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7812
7813 return Res;
7814 }
7815
7816 if (T.isSystemZ() && !DL.empty()) {
7817 // Make sure the stack alignment is present.
7818 if (!DL.contains("-S64"))
7819 return "E-S64" + DL.drop_front(1).str();
7820 return DL.str();
7821 }
7822
7823 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7824 // If the datalayout matches the expected format, add pointer size address
7825 // spaces to the datalayout.
7826 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7827 if (!DL.contains(AddrSpaces)) {
7828 SmallVector<StringRef, 4> Groups;
7829 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7830 if (R.match(Res, &Groups))
7831 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7832 }
7833 };
7834
7835 // AArch64 data layout upgrades.
7836 if (T.isAArch64()) {
7837 // Add "-Fn32"
7838 if (!DL.empty() && !DL.contains("-Fn32"))
7839 Res.append("-Fn32");
7840 AddPtr32Ptr64AddrSpaces();
7841 return Res;
7842 }
7843
7844 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7845 T.isWasm()) {
7846 // Mips64 with o32 ABI did not add "-i128:128".
7847 // Add "-i128:128"
7848 std::string I64 = "-i64:64";
7849 std::string I128 = "-i128:128";
7850 if (!StringRef(Res).contains(I128)) {
7851 size_t Pos = Res.find(I64);
7852 if (Pos != size_t(-1))
7853 Res.insert(Pos + I64.size(), I128);
7854 }
7855 }
7856
7857 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7858 size_t Pos = Res.find("-S128");
7859 if (Pos == StringRef::npos)
7860 Pos = Res.size();
7861 Res.insert(Pos, "-f64:32:64");
7862 }
7863
7864 if (!T.isX86())
7865 return Res;
7866
7867 AddPtr32Ptr64AddrSpaces();
7868
7869 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7870 // for i128 operations prior to this being reflected in the data layout, and
7871 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7872 // boundaries, so although this is a breaking change, the upgrade is expected
7873 // to fix more IR than it breaks.
7874 // Intel MCU is an exception and uses 4-byte-alignment.
7875 if (!T.isOSIAMCU()) {
7876 std::string I128 = "-i128:128";
7877 if (StringRef Ref = Res; !Ref.contains(I128)) {
7878 SmallVector<StringRef, 4> Groups;
7879 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7880 if (R.match(Res, &Groups))
7881 Res = (Groups[1] + I128 + Groups[3]).str();
7882 }
7883 }
7884
7885 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7886 // Raising the alignment is safe because Clang did not produce f80 values in
7887 // the MSVC environment before this upgrade was added.
7888 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7889 StringRef Ref = Res;
7890 auto I = Ref.find("-f80:32-");
7891 if (I != StringRef::npos)
7892 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7893 }
7894
7895 return Res;
7896}
7897
7898void llvm::UpgradeAttributes(AttrBuilder &B) {
7899 StringRef FramePointer;
7900 Attribute A = B.getAttribute("no-frame-pointer-elim");
7901 if (A.isValid()) {
7902 // The value can be "true" or "false".
7903 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7904 B.removeAttribute("no-frame-pointer-elim");
7905 }
7906 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7907 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7908 if (FramePointer != "all")
7909 FramePointer = "non-leaf";
7910 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7911 }
7912 if (!FramePointer.empty())
7913 B.addAttribute("frame-pointer", FramePointer);
7914
7915 A = B.getAttribute("null-pointer-is-valid");
7916 if (A.isValid()) {
7917 // The value can be "true" or "false".
7918 bool NullPointerIsValid = A.getValueAsString() == "true";
7919 B.removeAttribute("null-pointer-is-valid");
7920 if (NullPointerIsValid)
7921 B.addAttribute(Attribute::NullPointerIsValid);
7922 }
7923
7924 A = B.getAttribute("uniform-work-group-size");
7925 if (A.isValid()) {
7926 StringRef Val = A.getValueAsString();
7927 if (!Val.empty()) {
7928 bool IsTrue = Val == "true";
7929 B.removeAttribute("uniform-work-group-size");
7930 if (IsTrue)
7931 B.addAttribute("uniform-work-group-size");
7932 }
7933 }
7934}
7935
7936void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7937 // clang.arc.attachedcall bundles are now required to have an operand.
7938 // If they don't, it's okay to drop them entirely: when there is an operand,
7939 // the "attachedcall" is meaningful and required, but without an operand,
7940 // it's just a marker NOP. Dropping it merely prevents an optimization.
7941 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7942 return OBD.getTag() == "clang.arc.attachedcall" &&
7943 OBD.inputs().empty();
7944 });
7945}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:534
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:535
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2646
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1487
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7681
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1536
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6773
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1685
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2486
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1352
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5633
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1319
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1332
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6879
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2747
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2973
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1633
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2721
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1586
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7524
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1421
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6777
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5493
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7669
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5593
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6740
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2601
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2858
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1629
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2572
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5130
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2959
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7638
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2898
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1401
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6686
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1543
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7647
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2520
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1549
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1199
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1376
getNVVMFAddUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFAddUpgrade(StringRef Name)
Definition AutoUpgrade.cpp:1509
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7655
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5200
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5563
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5290
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2837
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5505
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5664
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2552
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2584
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1671
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2786
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1174
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2920
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2706
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3230
shouldUpgradeNVPTXBulkG2SClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1275
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2952
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2982
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5515
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7533
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2868
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1653
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2819
shouldUpgradeNVPTXBulkG2SCTAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1303
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1242
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5576
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7034
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3438
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5611
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:540
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:776
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:936
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:788
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:792
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:813
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:852
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:806
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:796
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:802
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:856
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:844
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:824
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:820
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:848
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:983
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:511
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:359
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1597
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1717
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3109
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3392
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2463
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2395
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2449
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3481
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:140
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:218
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:602
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:594
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2908
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:357
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:555
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:353
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1771
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:703
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:352
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1525
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1081
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1578
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1245
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:733
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:615
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:597
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1495
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1524
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:107
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1766
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1575
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1567
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1563
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1582
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1862
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1573
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2097
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:592
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:993
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:718
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1252
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5328
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
Changed
Changed
Definition ObjCARCOpts.cpp:2468
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1423
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:801
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:649
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:636
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1404
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1397
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5725
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7349
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3221
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2434
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7718
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7898
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6599
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6838
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7133
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7305
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7548
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7936
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6666
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7750
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2453
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:711
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6644
llvm::RecurKind::FAdd
@ FAdd
Sum of floats.
Definition IVDescriptors.h:50
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6696
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7422
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6618
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6903
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7943
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0