LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 Intrinsic::ID MinMaxID =
992 StringSwitch<Intrinsic::ID>(Name.split('.').first)
993 .Case("smax", Intrinsic::smax)
994 .Case("smin", Intrinsic::smin)
995 .Case("umax", Intrinsic::umax)
996 .Case("umin", Intrinsic::umin)
997 .Default(Intrinsic::not_intrinsic);
998 if (MinMaxID != Intrinsic::not_intrinsic) {
999 if (F->arg_size() != 2 || !F->getReturnType()->isIntOrIntVectorTy())
1000 return false; // Invalid IR.
1001 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), MinMaxID,
1002 F->getReturnType());
1003 return true;
1004 }
1005
1006 if (Name.starts_with("addp")) {
1007 // 'aarch64.neon.addp*'.
1008 if (F->arg_size() != 2)
1009 return false; // Invalid IR.
1010 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
1011 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
1012 NewFn = Intrinsic::getOrInsertDeclaration(
1013 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
1014 return true;
1015 }
1016 }
1017
1018 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1019 if (Name.starts_with("bfcvt")) {
1020 NewFn = nullptr;
1021 return true;
1022 }
1023
1024 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1025 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1026 NewFn = nullptr;
1027 return true;
1028 }
1029
1030 return false; // No other 'aarch64.neon.*'.
1031 }
1032 if (Name.consume_front("sve.")) {
1033 // 'aarch64.sve.*'.
1034 if (Name.consume_front("bf")) {
1035 if (Name == "mmla") {
1036 Type *Tys[] = {F->getReturnType(),
1037 std::next(F->arg_begin())->getType()};
1038 NewFn = Intrinsic::getOrInsertDeclaration(
1039 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1040 return true;
1041 }
1042 if (Name.consume_back(".lane")) {
1043 // 'aarch64.sve.bf*.lane'.
1044 Intrinsic::ID ID =
1045 StringSwitch<Intrinsic::ID>(Name)
1046 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1047 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1048 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1049 .Default(Intrinsic::not_intrinsic);
1050 if (ID != Intrinsic::not_intrinsic) {
1051 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1052 return true;
1053 }
1054 return false; // No other 'aarch64.sve.bf*.lane'.
1055 }
1056 return false; // No other 'aarch64.sve.bf*'.
1057 }
1058
1059 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1060 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1061 NewFn = nullptr;
1062 return true;
1063 }
1064
1065 if (Name.consume_front("convert.from.svbool")) {
1066 // 'aarch64.sve.convert.from.svbool'
1067 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1068 if (!TTy || TTy->getName() != "aarch64.svcount")
1069 return false;
1070
1071 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1072 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1073 return true;
1074 }
1075
1076 if (Name.consume_front("convert.to.svbool")) {
1077 // 'aarch64.sve.convert.to.svbool'
1078 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1079 if (!TTy || TTy->getName() != "aarch64.svcount")
1080 return false;
1081
1082 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1083 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1084 return true;
1085 }
1086
1087 if (Name.consume_front("addqv")) {
1088 // 'aarch64.sve.addqv'.
1089 if (!F->getReturnType()->isFPOrFPVectorTy())
1090 return false;
1091
1092 auto Args = F->getFunctionType()->params();
1093 Type *Tys[] = {F->getReturnType(), Args[1]};
1094 NewFn = Intrinsic::getOrInsertDeclaration(
1095 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1096 return true;
1097 }
1098
1099 if (Name.consume_front("ld")) {
1100 // 'aarch64.sve.ld*'.
1101 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1102 if (LdRegex.match(Name)) {
1103 Type *ScalarTy =
1104 cast<VectorType>(F->getReturnType())->getElementType();
1105 ElementCount EC =
1106 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1107 assert(F->arg_size() == 2 &&
1108 "Expected 2 arguments for ld* intrinsic.");
1109 Type *PtrTy = F->getArg(1)->getType();
1110 Type *Ty = VectorType::get(ScalarTy, EC);
1111 static const Intrinsic::ID LoadIDs[] = {
1112 Intrinsic::aarch64_sve_ld2_sret,
1113 Intrinsic::aarch64_sve_ld3_sret,
1114 Intrinsic::aarch64_sve_ld4_sret,
1115 };
1116 NewFn = Intrinsic::getOrInsertDeclaration(
1117 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1118 return true;
1119 }
1120 return false; // No other 'aarch64.sve.ld*'.
1121 }
1122
1123 if (Name.consume_front("tuple.")) {
1124 // 'aarch64.sve.tuple.*'.
1125 if (Name.starts_with("get")) {
1126 // 'aarch64.sve.tuple.get*'.
1127 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1128 NewFn = Intrinsic::getOrInsertDeclaration(
1129 F->getParent(), Intrinsic::vector_extract, Tys);
1130 return true;
1131 }
1132
1133 if (Name.starts_with("set")) {
1134 // 'aarch64.sve.tuple.set*'.
1135 auto Args = F->getFunctionType()->params();
1136 Type *Tys[] = {Args[0], Args[2], Args[1]};
1137 NewFn = Intrinsic::getOrInsertDeclaration(
1138 F->getParent(), Intrinsic::vector_insert, Tys);
1139 return true;
1140 }
1141
1142 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1143 if (CreateTupleRegex.match(Name)) {
1144 // 'aarch64.sve.tuple.create*'.
1145 auto Args = F->getFunctionType()->params();
1146 Type *Tys[] = {F->getReturnType(), Args[1]};
1147 NewFn = Intrinsic::getOrInsertDeclaration(
1148 F->getParent(), Intrinsic::vector_insert, Tys);
1149 return true;
1150 }
1151 return false; // No other 'aarch64.sve.tuple.*'.
1152 }
1153
1154 if (Name.starts_with("rev.nxv")) {
1155 // 'aarch64.sve.rev.<Ty>'
1156 NewFn = Intrinsic::getOrInsertDeclaration(
1157 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1158 return true;
1159 }
1160
1161 return false; // No other 'aarch64.sve.*'.
1162 }
1163 if (Name.consume_front("sme.")) {
1164 // 'aarch64.sme.*'.
1165 if (Name.consume_front("ftmopa.")) {
1166 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1167 // intrinsics to model their FPMR dependency.
1168 Intrinsic::ID ID =
1169 StringSwitch<Intrinsic::ID>(Name)
1170 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1171 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1172 .Default(Intrinsic::not_intrinsic);
1173 if (ID != Intrinsic::not_intrinsic) {
1174 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1175 return true;
1176 }
1177 return false; // No other 'aarch64.sme.ftmopa.*'.
1178 }
1179
1180 return false; // No other 'aarch64.sme.*'.
1181 }
1182 }
1183 return false; // No other 'arm.*', 'aarch64.*'.
1184}
1185
1186// The TMA G2S (global-to-shared) tensor copy modes that have legacy
1187// declarations requiring an auto-upgrade. The same set applies to the
1188// cluster (g2s) and CTA (g2s_cta) variants.
1189#define NVVM_TMA_G2S_MODES(M) \
1190 M(tile_1d, "tile.1d") \
1191 M(tile_2d, "tile.2d") \
1192 M(tile_3d, "tile.3d") \
1193 M(tile_4d, "tile.4d") \
1194 M(tile_5d, "tile.5d") \
1195 M(tile_gather4_2d, "tile.gather4.2d") \
1196 M(im2col_3d, "im2col.3d") \
1197 M(im2col_4d, "im2col.4d") \
1198 M(im2col_5d, "im2col.5d") \
1199 M(im2col_w_3d, "im2col.w.3d") \
1200 M(im2col_w_4d, "im2col.w.4d") \
1201 M(im2col_w_5d, "im2col.w.5d") \
1202 M(im2col_w_128_3d, "im2col.w.128.3d") \
1203 M(im2col_w_128_4d, "im2col.w.128.4d") \
1204 M(im2col_w_128_5d, "im2col.w.128.5d")
1205
1206// Two legacy tails are:
1207//
1208// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch
1209// arg1, arg2, .. i64 %ch, i1 %flag_mc, i1 %flag_ch, i32 %cta_group
1210//
1211// The current tail appends a trailing i32 %flag_valid_pattern, so both
1212// legacy tails are recognized by an i1 at parameter N-2.
1213static Intrinsic::ID
1214shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name,
1215 SmallVectorImpl<Type *> &OvlTys) {
1216 if (!Name.consume_front("cp.async.bulk.tensor.g2s."))
1217 return Intrinsic::not_intrinsic;
1218
1219#define G2S_ID(ID_SUFFIX, NAME) \
1220 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)
1221 // clang-format off
1222 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1223 NVVM_TMA_G2S_MODES(G2S_ID)
1224 .Default(Intrinsic::not_intrinsic);
1225#undef G2S_ID
1226 // clang-format on
1227 if (ID == Intrinsic::not_intrinsic)
1228 return ID;
1229
1230 size_t NumParams = F->getFunctionType()->getNumParams();
1231
1232 // Parameter N-2 is i1 for both legacy tails; the current tail ends
1233 // with i32 %cta_group, i32 %flag_valid_pattern, for which N-2 is i32.
1234 if (!F->getFunctionType()->getParamType(NumParams - 2)->isIntegerTy(1))
1235 return Intrinsic::not_intrinsic;
1236
1237 // The multicast mask is the parameter immediately before the i64
1238 // cache-hint: N-4 for the 2-flag tail, N-5 for the 3-flag tail.
1239 ArrayRef<Type *> Params = F->getFunctionType()->params();
1240 size_t MaskIdx =
1241 Params[NumParams - 1]->isIntegerTy(1) ? NumParams - 4 : NumParams - 5;
1242 assert(Params[MaskIdx + 1]->isIntegerTy(64) &&
1243 "expected the i64 cache-hint after the multicast mask");
1244 Type *MaskTy = Params[MaskIdx];
1245 assert(MaskTy->isIntegerTy(16) && "unexpected multicast mask type");
1246 OvlTys.push_back(MaskTy);
1247
1248 return ID;
1249}
1250
1251// The legacy tail is:
1252//
1253// arg1, arg2, .. i64 %ch, i1 %flag_ch
1254//
1255// The current tail appends a trailing i32 %flag_valid_pattern, so the
1256// legacy tail is recognized by an i1 at parameter N-1.
1257static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F,
1258 StringRef Name) {
1259 if (!Name.consume_front("cp.async.bulk.tensor.g2s.cta."))
1260 return Intrinsic::not_intrinsic;
1261
1262#define G2S_CTA_ID(ID_SUFFIX, NAME) \
1263 .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)
1264 // clang-format off
1265 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1266 NVVM_TMA_G2S_MODES(G2S_CTA_ID)
1267 .Default(Intrinsic::not_intrinsic);
1268#undef G2S_CTA_ID
1269 // clang-format on
1270 if (ID == Intrinsic::not_intrinsic)
1271 return ID;
1272
1273 // Parameter N-1 is i1 for the legacy tail; the current tail ends
1274 // with i32 %flag_valid_pattern, for which N-1 is i32.
1275 if (!F->getFunctionType()
1276 ->getParamType(F->getFunctionType()->getNumParams() - 1)
1277 ->isIntegerTy(1))
1278 return Intrinsic::not_intrinsic;
1279
1280 return ID;
1281}
1282// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cluster is:
1283//
1284// ..., i16 %mc, i64 %ch, i1 %flag_mc, i1 %flag_ch
1285//
1286// The current intrinsic is overloaded on the multicast-mask type and takes a
1287// trailing i32 %flag_valid_pattern; the legacy tail is recognized by an i1 at
1288// parameter N-1.
1289static Intrinsic::ID
1290shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name,
1291 SmallVectorImpl<Type *> &OvlTys) {
1292 if (!Name.consume_front("cp.async.bulk.global.to.shared.cluster"))
1293 return Intrinsic::not_intrinsic;
1294
1295 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1296 // i32 %flag_valid_pattern, for which N-1 is i32.
1297 size_t NumParams = F->getFunctionType()->getNumParams();
1298 if (!F->getFunctionType()->getParamType(NumParams - 1)->isIntegerTy(1))
1299 return Intrinsic::not_intrinsic;
1300
1301 // The multicast mask is parameter 4; legacy IR only uses i16.
1302 Type *MaskTy = F->getFunctionType()->getParamType(NumParams - 4);
1303 if (!MaskTy->isIntegerTy(16))
1304 return Intrinsic::not_intrinsic;
1305 OvlTys.push_back(MaskTy);
1306
1307 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster;
1308}
1309
1310// The legacy tail of llvm.nvvm.cp.async.bulk.global.to.shared.cta is:
1311//
1312// ..., i64 %ch, i1 %flag_ch
1313//
1314// The current intrinsic adds %ignore_bytes_left/%ignore_bytes_right before
1315// %ch and trailing %flag_oob/%flag_valid_pattern; the legacy tail is
1316// recognized by an i1 at parameter N-1, whereas the current tail ends
1317// with an i32.
1318static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F,
1319 StringRef Name) {
1320 if (!Name.consume_front("cp.async.bulk.global.to.shared.cta"))
1321 return Intrinsic::not_intrinsic;
1322
1323 // Parameter N-1 is i1 for the legacy tail; the current tail ends with
1324 // i32 %flag_valid_pattern, for which N-1 is i32.
1325 if (!F->getFunctionType()->getParamType(5)->isIntegerTy(1))
1326 return Intrinsic::not_intrinsic;
1327
1328 return Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta;
1329}
1330
1331// The legacy TMA reduction intrinsics encode the reduction operator in their
1332// name, while the current ones take it as an immediate argument. Map the
1333// operator part of a legacy name to the corresponding immediate value.
1334static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1335 return StringSwitch<std::optional<unsigned>>(Name)
1336 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1337 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1338 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1339 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1340 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1341 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1342 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1343 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1344 .Default(std::nullopt);
1345}
1346
1347static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1348 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1349 return Intrinsic::not_intrinsic;
1350
1351 auto [RedOpName, ShapeName] = Name.split('.');
1352 if (!getNVPTXTMAReductionOp(RedOpName))
1353 return Intrinsic::not_intrinsic;
1354
1355 return StringSwitch<Intrinsic::ID>(ShapeName)
1356 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1357 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1358 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1359 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1360 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1361 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1362 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1363 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1364 .Default(Intrinsic::not_intrinsic);
1365}
1366
1367static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1368 StringRef Name) {
1369 if (Name.consume_front("mapa.shared.cluster"))
1370 if (F->getReturnType()->getPointerAddressSpace() ==
1371 NVPTXAS::ADDRESS_SPACE_SHARED)
1372 return Intrinsic::nvvm_mapa_shared_cluster;
1373
1374 if (Name.consume_front("cp.async.bulk.")) {
1375 Intrinsic::ID ID =
1376 StringSwitch<Intrinsic::ID>(Name)
1377 .Case("shared.cta.to.cluster",
1378 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1379 .Default(Intrinsic::not_intrinsic);
1380
1381 if (ID != Intrinsic::not_intrinsic)
1382 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1383 NVPTXAS::ADDRESS_SPACE_SHARED)
1384 return ID;
1385 }
1386
1387 return Intrinsic::not_intrinsic;
1388}
1389
1390static Intrinsic::ID
1391shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1392 if (!Name.consume_front("tcgen05.commit."))
1393 return Intrinsic::not_intrinsic;
1394
1395 if (Name.consume_front("shared."))
1396 return StringSwitch<Intrinsic::ID>(Name)
1397 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1398 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1399 .Default(Intrinsic::not_intrinsic);
1400
1401 if (Name.consume_front("mc.shared.")) {
1402 // Only upgrade older i16 mc variants.
1403 if (!F->getArg(1)->getType()->isIntegerTy(16))
1404 return Intrinsic::not_intrinsic;
1405
1406 return StringSwitch<Intrinsic::ID>(Name)
1407 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1408 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1409 .Default(Intrinsic::not_intrinsic);
1410 }
1411
1412 return Intrinsic::not_intrinsic;
1413}
1414
1415static Intrinsic::ID
1416shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1417 if (F->arg_size() != 2)
1418 return Intrinsic::not_intrinsic;
1419
1420 if (Name.consume_front("tcgen05.alloc.shared.") ||
1421 Name.consume_front("tcgen05.alloc."))
1422 return StringSwitch<Intrinsic::ID>(Name)
1423 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1424 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1425 .Default(Intrinsic::not_intrinsic);
1426
1427 if (Name.consume_front("tcgen05.dealloc."))
1428 return StringSwitch<Intrinsic::ID>(Name)
1429 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1430 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1431 .Default(Intrinsic::not_intrinsic);
1432
1433 return Intrinsic::not_intrinsic;
1434}
1435
1436static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1437 if (Name.consume_front("fma.rn."))
1438 return StringSwitch<Intrinsic::ID>(Name)
1439 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1440 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1441 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1442 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1443 .Default(Intrinsic::not_intrinsic);
1444
1445 if (Name.consume_front("fmax."))
1446 return StringSwitch<Intrinsic::ID>(Name)
1447 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1448 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1449 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1450 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1451 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1452 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1453 .Case("ftz.nan.xorsign.abs.bf16",
1454 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1455 .Case("ftz.nan.xorsign.abs.bf16x2",
1456 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1457 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1458 .Case("ftz.xorsign.abs.bf16x2",
1459 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1460 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1461 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1462 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1463 .Case("nan.xorsign.abs.bf16x2",
1464 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1465 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1466 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1467 .Default(Intrinsic::not_intrinsic);
1468
1469 if (Name.consume_front("fmin."))
1470 return StringSwitch<Intrinsic::ID>(Name)
1471 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1472 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1473 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1474 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1475 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1476 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1477 .Case("ftz.nan.xorsign.abs.bf16",
1478 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1479 .Case("ftz.nan.xorsign.abs.bf16x2",
1480 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1481 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1482 .Case("ftz.xorsign.abs.bf16x2",
1483 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1484 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1485 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1486 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1487 .Case("nan.xorsign.abs.bf16x2",
1488 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1489 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1490 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1491 .Default(Intrinsic::not_intrinsic);
1492
1493 if (Name.consume_front("neg."))
1494 return StringSwitch<Intrinsic::ID>(Name)
1495 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1496 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1497 .Default(Intrinsic::not_intrinsic);
1498
1499 return Intrinsic::not_intrinsic;
1500}
1501
1502static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1503 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1504 FunctionType *OldFnTy = F->getFunctionType();
1505 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1506 return OldTy->getScalarType()->isIntegerTy() &&
1507 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1508 };
1509
1510 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1511 return false;
1512
1513 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1514 return false;
1515
1516 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1517 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1518 return false;
1519
1520 return true;
1521}
1522
1523static std::optional<std::pair<Intrinsic::ID, RoundingMode>>
1524getNVVMFAddUpgrade(StringRef Name) {
1525 auto [Modifiers, Type] = Name.rsplit('.');
1526 if (!is_contained({"f", "d", "f16", "v2f16"}, Type))
1527 return std::nullopt;
1528
1529 std::optional<llvm::RoundingMode> RoundingMode =
1530 StringSwitch<std::optional<llvm::RoundingMode>>(Modifiers.take_front(2))
1531 .Case("rn", llvm::RoundingMode::NearestTiesToEven)
1532 .Case("rz", llvm::RoundingMode::TowardZero)
1533 .Case("rm", llvm::RoundingMode::TowardNegative)
1534 .Case("rp", llvm::RoundingMode::TowardPositive)
1535 .Default(std::nullopt);
1536 if (!RoundingMode)
1537 return std::nullopt;
1538
1539 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Modifiers.drop_front(2))
1540 .Case("", Intrinsic::nvvm_fadd)
1541 .Case(".ftz", Intrinsic::nvvm_fadd_ftz)
1542 .Case(".sat", Intrinsic::nvvm_fadd_sat)
1543 .Case(".ftz.sat", Intrinsic::nvvm_fadd_ftz_sat)
1544 .Default(Intrinsic::not_intrinsic);
1545 if (IID == Intrinsic::not_intrinsic)
1546 return std::nullopt;
1547
1548 return std::make_pair(IID, *RoundingMode);
1549}
1550
1551static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name) {
1552 if (Name != "mbarrier.init" && Name != "mbarrier.init.shared")
1553 return Intrinsic::not_intrinsic;
1554
1555 return Intrinsic::nvvm_mbarrier_init;
1556}
1557
1558static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1559 return Name.consume_front("local") || Name.consume_front("shared") ||
1560 Name.consume_front("global") || Name.consume_front("constant") ||
1561 Name.consume_front("param");
1562}
1563
1564static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1565 if (!Name.consume_front("vp."))
1566 return 0;
1567 return StringSwitch<unsigned>(Name)
1568 .StartsWith("select", Instruction::Select)
1569 .StartsWith("add", Instruction::Add)
1570 .StartsWith("sub", Instruction::Sub)
1571 .StartsWith("mul", Instruction::Mul)
1572 .StartsWith("ashr", Instruction::AShr)
1573 .StartsWith("lshr", Instruction::LShr)
1574 .StartsWith("shl", Instruction::Shl)
1575 .StartsWith("or", Instruction::Or)
1576 .StartsWith("and", Instruction::And)
1577 .StartsWith("xor", Instruction::Xor)
1578 .StartsWith("fadd", Instruction::FAdd)
1579 .StartsWith("fsub", Instruction::FSub)
1580 .StartsWith("fmuladd", 0)
1581 .StartsWith("fmul", Instruction::FMul)
1582 .StartsWith("fdiv", Instruction::FDiv)
1583 .StartsWith("frem", Instruction::FRem)
1584 .StartsWith("fneg", Instruction::FNeg)
1585 .StartsWith("trunc", Instruction::Trunc)
1586 .StartsWith("zext", Instruction::ZExt)
1587 .StartsWith("sext", Instruction::SExt)
1588 .StartsWith("fptrunc", Instruction::FPTrunc)
1589 .StartsWith("fpext", Instruction::FPExt)
1590 .StartsWith("fptoui", Instruction::FPToUI)
1591 .StartsWith("fptosi", Instruction::FPToSI)
1592 .StartsWith("uitofp", Instruction::UIToFP)
1593 .StartsWith("sitofp", Instruction::SIToFP)
1594 .StartsWith("ptrtoint", Instruction::PtrToInt)
1595 .StartsWith("inttoptr", Instruction::IntToPtr)
1596 .StartsWith("icmp", Instruction::ICmp)
1597 .StartsWith("fcmp", Instruction::FCmp)
1598 .Default(0);
1599}
1600
1601static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1602 if (!Name.consume_front("vp."))
1603 return 0;
1604 return StringSwitch<Intrinsic::ID>(Name)
1605 .StartsWith("abs", Intrinsic::abs)
1606 .StartsWith("smax", Intrinsic::smax)
1607 .StartsWith("smin", Intrinsic::smin)
1608 .StartsWith("umax", Intrinsic::umax)
1609 .StartsWith("umin", Intrinsic::umin)
1610 .StartsWith("copysign", Intrinsic::copysign)
1611 .StartsWith("minnum", Intrinsic::minnum)
1612 .StartsWith("maxnum", Intrinsic::maxnum)
1613 .StartsWith("minimum", Intrinsic::minimum)
1614 .StartsWith("maximum", Intrinsic::maximum)
1615 .StartsWith("fabs", Intrinsic::fabs)
1616 .StartsWith("sqrt", Intrinsic::sqrt)
1617 .StartsWith("fma", Intrinsic::fma)
1618 .StartsWith("fmuladd", Intrinsic::fmuladd)
1619 .StartsWith("ceil", Intrinsic::ceil)
1620 .StartsWith("floor", Intrinsic::floor)
1621 .StartsWith("rint", Intrinsic::rint)
1622 .StartsWith("nearbyint", Intrinsic::nearbyint)
1623 .StartsWith("roundeven", Intrinsic::roundeven)
1624 .StartsWith("roundtozero", Intrinsic::trunc)
1625 .StartsWith("round", Intrinsic::round)
1626 .StartsWith("lrint", Intrinsic::lrint)
1627 .StartsWith("llrint", Intrinsic::llrint)
1628 .StartsWith("bitreverse", Intrinsic::bitreverse)
1629 .StartsWith("bswap", Intrinsic::bswap)
1630 .StartsWith("ctpop", Intrinsic::ctpop)
1631 .StartsWith("ctlz", Intrinsic::ctlz)
1632 .StartsWith("cttz.elts", 0)
1633 .StartsWith("cttz", Intrinsic::cttz)
1634 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1635 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1636 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1637 .StartsWith("usub.sat", Intrinsic::usub_sat)
1638 .StartsWith("fshl", Intrinsic::fshl)
1639 .StartsWith("fshr", Intrinsic::fshr)
1640 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1641 .Default(0);
1642}
1643
1647
1648static bool convertIntrinsicValidType(StringRef Name,
1649 const FunctionType *FuncTy) {
1650 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1651 if (Name.starts_with("to.fp16")) {
1652 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1653 HalfTy) &&
1654 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1655 FuncTy->getReturnType());
1656 }
1657
1658 if (Name.starts_with("from.fp16")) {
1659 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1660 HalfTy) &&
1661 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1662 FuncTy->getReturnType());
1663 }
1664
1665 return false;
1666}
1667
1668static unsigned
1669getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID,
1670 SmallVectorImpl<Type *> &OverloadTys) {
1671 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1672 if (Defaults.empty())
1673 return 0;
1674
1675 unsigned FullArgCount = FirstDefault + Defaults.size();
1676
1677 // Only trailing default arguments can be missing.
1678 if (F->arg_size() < FirstDefault || F->arg_size() >= FullArgCount)
1679 return 0;
1680
1681 unsigned NumMissingTrailingParams = FullArgCount - F->arg_size();
1682 if (!Intrinsic::isSignatureValid(IID, F->getFunctionType(), OverloadTys,
1683 NumMissingTrailingParams))
1684 return 0;
1685
1686 return FullArgCount;
1687}
1688
1689static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn) {
1690 Intrinsic::ID IID = F->getIntrinsicID();
1691 SmallVector<Type *, 4> OverloadTys;
1692
1693 unsigned FullArgCount =
1694 getFullArgCountForDefaultArgUpgrade(F, IID, OverloadTys);
1695 if (FullArgCount == 0)
1696 return false;
1697
1698 rename(F);
1699 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OverloadTys);
1700 assert(NewFn->arg_size() == FullArgCount &&
1701 "total number of default args does not match intrinsic signature");
1702 return true;
1703}
1704
1705static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1706 bool CanUpgradeDebugIntrinsicsToRecords) {
1707 assert(F && "Illegal to upgrade a non-existent Function.");
1708
1709 StringRef Name = F->getName();
1710
1711 // Quickly eliminate it, if it's not a candidate.
1712 if (!Name.consume_front("llvm.") || Name.empty())
1713 return false;
1714
1715 switch (Name[0]) {
1716 default: break;
1717 case 'a': {
1718 bool IsArm = Name.consume_front("arm.");
1719 if (IsArm || Name.consume_front("aarch64.")) {
1720 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1721 return true;
1722 break;
1723 }
1724
1725 if (Name.consume_front("amdgcn.")) {
1726 if (Name == "alignbit") {
1727 // Target specific intrinsic became redundant
1728 NewFn = Intrinsic::getOrInsertDeclaration(
1729 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1730 return true;
1731 }
1732
1733 if (Name.consume_front("atomic.")) {
1734 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1735 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1736 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1737 // and usub_sat so there's no new declaration.
1738 NewFn = nullptr;
1739 return true;
1740 }
1741 break; // No other 'amdgcn.atomic.*'
1742 }
1743
1744 if (Name.starts_with("addrspacecast.nonnull")) {
1745 // Replaced with an addrspacecast instruction carrying the nonnull flag,
1746 // so there's no new declaration.
1747 NewFn = nullptr;
1748 return true;
1749 }
1750
1751 switch (F->getIntrinsicID()) {
1752 default:
1753 break;
1754 // Legacy wmma iu intrinsics without the optional clamp operand.
1755 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1756 if (F->arg_size() == 7) {
1757 NewFn = nullptr;
1758 return true;
1759 }
1760 break;
1761 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1762 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1763 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1764 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1765 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1766 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1767 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1768 if (F->arg_size() == 8) {
1769 NewFn = nullptr;
1770 return true;
1771 }
1772 break;
1773 }
1774
1775 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1776 Name.consume_front("flat.atomic.")) {
1777 if (Name.starts_with("fadd") ||
1778 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1779 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1780 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1781 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1782 // declaration.
1783 NewFn = nullptr;
1784 return true;
1785 }
1786 }
1787
1788 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1789 NewFn = nullptr;
1790 return true;
1791 }
1792
1793 if (Name.starts_with("ldexp.")) {
1794 // Target specific intrinsic became redundant
1795 NewFn = Intrinsic::getOrInsertDeclaration(
1796 F->getParent(), Intrinsic::ldexp,
1797 {F->getReturnType(), F->getArg(1)->getType()});
1798 return true;
1799 }
1800 break; // No other 'amdgcn.*'
1801 }
1802
1803 break;
1804 }
1805 case 'c': {
1806 if (F->arg_size() == 1) {
1807 if (Name.consume_front("convert.")) {
1808 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1809 NewFn = nullptr;
1810 return true;
1811 }
1812 }
1813
1814 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1815 .StartsWith("ctlz.", Intrinsic::ctlz)
1816 .StartsWith("cttz.", Intrinsic::cttz)
1817 .Default(Intrinsic::not_intrinsic);
1818 if (ID != Intrinsic::not_intrinsic) {
1819 rename(F);
1820 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1821 F->arg_begin()->getType());
1822 return true;
1823 }
1824 }
1825
1826 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1827 if (Name == "coro.end" &&
1828 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1829 CoroEndID = Intrinsic::coro_end;
1830 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1831 CoroEndID = Intrinsic::coro_end_async;
1832
1833 if (CoroEndID != Intrinsic::not_intrinsic) {
1834 rename(F);
1835 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1836 return true;
1837 }
1838
1839 break;
1840 }
1841 case 'd':
1842 if (Name.consume_front("dbg.")) {
1843 // Mark debug intrinsics for upgrade to new debug format.
1844 if (CanUpgradeDebugIntrinsicsToRecords) {
1845 if (Name == "addr" || Name == "value" || Name == "assign" ||
1846 Name == "declare" || Name == "label") {
1847 // There's no function to replace these with.
1848 NewFn = nullptr;
1849 // But we do want these to get upgraded.
1850 return true;
1851 }
1852 }
1853 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1854 // converted to DbgVariableRecords later.
1855 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1856 rename(F);
1857 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1858 Intrinsic::dbg_value);
1859 return true;
1860 }
1861 break; // No other 'dbg.*'.
1862 }
1863 break;
1864 case 'e':
1865 if (Name.consume_front("experimental.vector.")) {
1866 Intrinsic::ID ID =
1867 StringSwitch<Intrinsic::ID>(Name)
1868 // Skip over extract.last.active, otherwise it will be 'upgraded'
1869 // to a regular vector extract which is a different operation.
1870 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1871 .StartsWith("extract.", Intrinsic::vector_extract)
1872 .StartsWith("insert.", Intrinsic::vector_insert)
1873 .StartsWith("reverse.", Intrinsic::vector_reverse)
1874 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1875 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1876 .StartsWith("partial.reduce.add",
1877 Intrinsic::vector_partial_reduce_add)
1878 .Default(Intrinsic::not_intrinsic);
1879 if (ID != Intrinsic::not_intrinsic) {
1880 const auto *FT = F->getFunctionType();
1881 SmallVector<Type *, 2> Tys;
1882 if (ID == Intrinsic::vector_extract ||
1883 ID == Intrinsic::vector_interleave2)
1884 // Extracting overloads the return type.
1885 Tys.push_back(FT->getReturnType());
1886 if (ID != Intrinsic::vector_interleave2)
1887 Tys.push_back(FT->getParamType(0));
1888 if (ID == Intrinsic::vector_insert ||
1889 ID == Intrinsic::vector_partial_reduce_add)
1890 // Inserting overloads the inserted type.
1891 Tys.push_back(FT->getParamType(1));
1892 rename(F);
1893 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1894 return true;
1895 }
1896
1897 if (Name.consume_front("reduce.")) {
1898 SmallVector<StringRef, 2> Groups;
1899 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1900 if (R.match(Name, &Groups))
1901 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1902 .Case("add", Intrinsic::vector_reduce_add)
1903 .Case("mul", Intrinsic::vector_reduce_mul)
1904 .Case("and", Intrinsic::vector_reduce_and)
1905 .Case("or", Intrinsic::vector_reduce_or)
1906 .Case("xor", Intrinsic::vector_reduce_xor)
1907 .Case("smax", Intrinsic::vector_reduce_smax)
1908 .Case("smin", Intrinsic::vector_reduce_smin)
1909 .Case("umax", Intrinsic::vector_reduce_umax)
1910 .Case("umin", Intrinsic::vector_reduce_umin)
1911 .Case("fmax", Intrinsic::vector_reduce_fmax)
1912 .Case("fmin", Intrinsic::vector_reduce_fmin)
1913 .Default(Intrinsic::not_intrinsic);
1914
1915 bool V2 = false;
1916 if (ID == Intrinsic::not_intrinsic) {
1917 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1918 Groups.clear();
1919 V2 = true;
1920 if (R2.match(Name, &Groups))
1921 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1922 .Case("fadd", Intrinsic::vector_reduce_fadd)
1923 .Case("fmul", Intrinsic::vector_reduce_fmul)
1924 .Default(Intrinsic::not_intrinsic);
1925 }
1926 if (ID != Intrinsic::not_intrinsic) {
1927 rename(F);
1928 auto Args = F->getFunctionType()->params();
1929 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1930 {Args[V2 ? 1 : 0]});
1931 return true;
1932 }
1933 break; // No other 'expermental.vector.reduce.*'.
1934 }
1935
1936 if (Name.consume_front("splice"))
1937 return true;
1938 break; // No other 'experimental.vector.*'.
1939 }
1940 if (Name.consume_front("experimental.stepvector.")) {
1941 Intrinsic::ID ID = Intrinsic::stepvector;
1942 rename(F);
1943 NewFn = Intrinsic::getOrInsertDeclaration(
1944 F->getParent(), ID, F->getFunctionType()->getReturnType());
1945 return true;
1946 }
1947 break; // No other 'e*'.
1948 case 'f':
1949 if (Name.starts_with("flt.rounds")) {
1950 rename(F);
1951 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1952 Intrinsic::get_rounding);
1953 return true;
1954 }
1955 break;
1956 case 'i':
1957 if (Name.starts_with("invariant.group.barrier")) {
1958 // Rename invariant.group.barrier to launder.invariant.group
1959 auto Args = F->getFunctionType()->params();
1960 Type* ObjectPtr[1] = {Args[0]};
1961 rename(F);
1962 NewFn = Intrinsic::getOrInsertDeclaration(
1963 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1964 return true;
1965 }
1966 break;
1967 case 'l': {
1968 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1969 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1970 if (IsLifetimeStart || IsLifetimeEnd) {
1971 if (F->arg_size() == 2) {
1972 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1973 : Intrinsic::lifetime_end;
1974 rename(F);
1975 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1976 // arguments. Use the Ptr argument to create new declaration.
1977 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1978 F->getArg(1)->getType());
1979 return true;
1980 } else if (F->arg_size() == 1 && Name == ".i64") {
1981 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1982 // Autoupgrade prior to
1983 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1984 // intrinsic with no expected calls. To allow auto-upgrade process to
1985 // delete such invalid intrinsic declaration, set NewFn = nullptr
1986 // and return true here. If there are actual calls to this intrinsic
1987 // (which is not expected), they will be deleted in
1988 // UpgradeIntrinsicCall.
1989 NewFn = nullptr;
1990 return true;
1991 }
1992 }
1993 break;
1994 }
1995 case 'm': {
1996 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1997 // alignment parameter to embedding the alignment as an attribute of
1998 // the pointer args.
1999 if (unsigned ID = StringSwitch<unsigned>(Name)
2000 .StartsWith("memcpy.", Intrinsic::memcpy)
2001 .StartsWith("memmove.", Intrinsic::memmove)
2002 .Default(0)) {
2003 if (F->arg_size() == 5) {
2004 rename(F);
2005 // Get the types of dest, src, and len
2006 ArrayRef<Type *> ParamTypes =
2007 F->getFunctionType()->params().slice(0, 3);
2008 NewFn =
2009 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
2010 return true;
2011 }
2012 }
2013 if (Name.starts_with("memset.") && F->arg_size() == 5) {
2014 rename(F);
2015 // Get the types of dest, and len
2016 const auto *FT = F->getFunctionType();
2017 Type *ParamTypes[2] = {
2018 FT->getParamType(0), // Dest
2019 FT->getParamType(2) // len
2020 };
2021 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2022 Intrinsic::memset, ParamTypes);
2023 return true;
2024 }
2025
2026 unsigned MaskedID =
2027 StringSwitch<unsigned>(Name)
2028 .StartsWith("masked.load", Intrinsic::masked_load)
2029 .StartsWith("masked.gather", Intrinsic::masked_gather)
2030 .StartsWith("masked.store", Intrinsic::masked_store)
2031 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
2032 .Default(0);
2033 if (MaskedID && F->arg_size() == 4) {
2034 rename(F);
2035 if (MaskedID == Intrinsic::masked_load ||
2036 MaskedID == Intrinsic::masked_gather) {
2037 NewFn = Intrinsic::getOrInsertDeclaration(
2038 F->getParent(), MaskedID,
2039 {F->getReturnType(), F->getArg(0)->getType()});
2040 return true;
2041 }
2042 NewFn = Intrinsic::getOrInsertDeclaration(
2043 F->getParent(), MaskedID,
2044 {F->getArg(0)->getType(), F->getArg(1)->getType()});
2045 return true;
2046 }
2047 break;
2048 }
2049 case 'n': {
2050 if (Name.consume_front("nvvm.")) {
2051 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
2052 if (F->arg_size() == 1) {
2053 Intrinsic::ID IID =
2054 StringSwitch<Intrinsic::ID>(Name)
2055 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
2056 .Case("clz.i", Intrinsic::ctlz)
2057 .Case("popc.i", Intrinsic::ctpop)
2058 .Default(Intrinsic::not_intrinsic);
2059 if (IID != Intrinsic::not_intrinsic) {
2060 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2061 {F->getReturnType()});
2062 return true;
2063 }
2064 } else if (F->arg_size() == 2) {
2065 Intrinsic::ID IID =
2066 StringSwitch<Intrinsic::ID>(Name)
2067 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
2068 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
2069 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
2070 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
2071 .Default(Intrinsic::not_intrinsic);
2072 if (IID != Intrinsic::not_intrinsic) {
2073 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2074 {F->getReturnType()});
2075 return true;
2076 }
2077 }
2078
2079 // Check for nvvm intrinsics that need a return type adjustment.
2080 {
2081 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
2082 if (IID != Intrinsic::not_intrinsic &&
2083 isLegacyNVPTXBF16IntSignature(F, IID)) {
2084 NewFn = nullptr;
2085 return true;
2086 }
2087 }
2088
2089 // Upgrade Distributed Shared Memory Intrinsics
2090 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
2091 if (IID != Intrinsic::not_intrinsic) {
2092 rename(F);
2093 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2094 return true;
2095 }
2096
2097 // Upgrade TMA reduction intrinsics
2098 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
2099 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
2100 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
2101 if (IID != Intrinsic::not_intrinsic) {
2102 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2103 return true;
2104 }
2105
2106 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
2107 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
2108 if (IID != Intrinsic::not_intrinsic) {
2109 rename(F);
2110 NewFn = Intrinsic::getOrInsertDeclaration(
2111 F->getParent(), IID, F->getReturnType(),
2112 F->getFunctionType()->params());
2113 return true;
2114 }
2115
2116 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
2117 // tcgen05.alloc shared variants to anyptr intrinsics.
2118 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
2119 if (IID != Intrinsic::not_intrinsic) {
2120 rename(F);
2121 if (Intrinsic::isOverloaded(IID))
2122 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2123 {F->getArg(0)->getType()});
2124 else
2125 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2126 return true;
2127 }
2128
2129 // Upgrade TMA copy G2S CTA intrinsics.
2130 IID = shouldUpgradeNVPTXTMAG2SCTAIntrinsics(F, Name);
2131 if (IID != Intrinsic::not_intrinsic) {
2132 rename(F);
2133 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2134 return true;
2135 }
2136
2137 // Upgrade TMA copy G2S (cluster) intrinsics.
2138 SmallVector<Type *, 1> OvlTys;
2139 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name, OvlTys);
2140 if (IID != Intrinsic::not_intrinsic) {
2141 rename(F);
2142 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID, OvlTys);
2143 return true;
2144 }
2145
2146 // Upgrade the legacy cp.async.bulk.global.to.shared.cluster signature
2147 // (multicast-mask overloading + trailing flag_valid_pattern).
2148 SmallVector<Type *, 1> BulkG2SOvlTys;
2149 IID = shouldUpgradeNVPTXBulkG2SClusterIntrinsic(F, Name, BulkG2SOvlTys);
2150 if (IID != Intrinsic::not_intrinsic) {
2151 rename(F);
2152 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2153 BulkG2SOvlTys);
2154 return true;
2155 }
2156
2157 // Upgrade the legacy cp.async.bulk.global.to.shared.cta signature
2158 // (no ignore_bytes_left/right + trailing flag_valid_pattern).
2159 IID = shouldUpgradeNVPTXBulkG2SCTAIntrinsic(F, Name);
2160 if (IID != Intrinsic::not_intrinsic) {
2161 rename(F);
2162 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
2163 return true;
2164 }
2165
2166 // Upgrade mbarrier.init intrinsics missing the layout operand.
2167 IID = shouldUpgradeNVPTXMBarrierInitIntrinsic(Name);
2168 if (IID != Intrinsic::not_intrinsic) {
2169 rename(F);
2170 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
2171 F->getArg(0)->getType());
2172 return true;
2173 }
2174
2175 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
2176 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
2177 //
2178 // TODO: We could add lohi.i2d.
2179 bool Expand = false;
2180 if (Name.consume_front("abs."))
2181 // nvvm.abs.{i,ii}
2182 Expand =
2183 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2184 else if (Name.consume_front("fabs."))
2185 // nvvm.fabs.{f,ftz.f,d}
2186 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2187 else if (Name.consume_front("add."))
2188 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
2189 Expand = getNVVMFAddUpgrade(Name).has_value();
2190 else if (Name.consume_front("ex2.approx."))
2191 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2192 Expand =
2193 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2194 else if (Name.consume_front("atomic.load."))
2195 // nvvm.atomic.load.add.{f32,f64}.p
2196 // nvvm.atomic.load.{inc,dec}.32.p
2197 Expand = StringSwitch<bool>(Name)
2198 .StartsWith("add.f32.p", true)
2199 .StartsWith("add.f64.p", true)
2200 .StartsWith("inc.32.p", true)
2201 .StartsWith("dec.32.p", true)
2202 .Default(false);
2203 else if (Name.consume_front("atomic."))
2204 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2205 // nvvm.atomic.cas.gen.i.{cta,sys}
2206 Expand = StringSwitch<bool>(Name)
2207 .StartsWith("add.gen.", true)
2208 .StartsWith("exch.gen.", true)
2209 .StartsWith("max.gen.", true)
2210 .StartsWith("min.gen.", true)
2211 .StartsWith("inc.gen.", true)
2212 .StartsWith("dec.gen.", true)
2213 .StartsWith("and.gen.", true)
2214 .StartsWith("or.gen.", true)
2215 .StartsWith("xor.gen.", true)
2216 .StartsWith("cas.gen.", true)
2217 .Default(false);
2218 else if (Name.consume_front("bitcast."))
2219 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2220 Expand =
2221 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2222 else if (Name.consume_front("rotate."))
2223 // nvvm.rotate.{b32,b64,right.b64}
2224 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2225 else if (Name.consume_front("ptr.gen.to."))
2226 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2227 Expand = consumeNVVMPtrAddrSpace(Name);
2228 else if (Name.consume_front("ptr."))
2229 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2230 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2231 else if (Name.consume_front("ldg.global."))
2232 // nvvm.ldg.global.{i,p,f}
2233 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2234 Name.starts_with("p."));
2235 else
2236 Expand = StringSwitch<bool>(Name)
2237 .Case("barrier0", true)
2238 .Case("barrier.n", true)
2239 .Case("barrier.sync.cnt", true)
2240 .Case("barrier.sync", true)
2241 .Case("barrier", true)
2242 .Case("bar.sync", true)
2243 .Case("barrier0.popc", true)
2244 .Case("barrier0.and", true)
2245 .Case("barrier0.or", true)
2246 .Case("clz.ll", true)
2247 .Case("popc.ll", true)
2248 .Case("h2f", true)
2249 .Case("swap.lo.hi.b64", true)
2250 .Case("tanh.approx.f32", true)
2251 .Default(false);
2252
2253 if (Expand) {
2254 NewFn = nullptr;
2255 return true;
2256 }
2257 break; // No other 'nvvm.*'.
2258 }
2259 break;
2260 }
2261 case 'o':
2262 if (Name.starts_with("objectsize.")) {
2263 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2264 if (F->arg_size() == 2 || F->arg_size() == 3) {
2265 rename(F);
2266 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2267 Intrinsic::objectsize, Tys);
2268 return true;
2269 }
2270 }
2271 break;
2272
2273 case 'p':
2274 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2275 rename(F);
2276 NewFn = Intrinsic::getOrInsertDeclaration(
2277 F->getParent(), Intrinsic::ptr_annotation,
2278 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2279 return true;
2280 }
2281 break;
2282
2283 case 'r': {
2284 if (Name.consume_front("riscv.")) {
2285 Intrinsic::ID ID;
2286 ID = StringSwitch<Intrinsic::ID>(Name)
2287 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2288 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2289 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2290 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2291 .Default(Intrinsic::not_intrinsic);
2292 if (ID != Intrinsic::not_intrinsic) {
2293 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2294 rename(F);
2295 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2296 return true;
2297 }
2298 break; // No other applicable upgrades.
2299 }
2300
2301 ID = StringSwitch<Intrinsic::ID>(Name)
2302 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2303 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2304 .Default(Intrinsic::not_intrinsic);
2305 if (ID != Intrinsic::not_intrinsic) {
2306 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2307 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2308 rename(F);
2309 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2310 return true;
2311 }
2312 break; // No other applicable upgrades.
2313 }
2314
2315 ID = StringSwitch<Intrinsic::ID>(Name)
2316 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2317 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2318 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2319 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2320 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2321 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2322 .Default(Intrinsic::not_intrinsic);
2323 if (ID != Intrinsic::not_intrinsic) {
2324 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2325 rename(F);
2326 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2327 return true;
2328 }
2329 break; // No other applicable upgrades.
2330 }
2331
2332 // Replace llvm.riscv.clmul with llvm.clmul.
2333 if (Name == "clmul.i32" || Name == "clmul.i64") {
2334 NewFn = Intrinsic::getOrInsertDeclaration(
2335 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2336 return true;
2337 }
2338
2339 break; // No other 'riscv.*' intrinsics
2340 }
2341 } break;
2342
2343 case 's':
2344 if (Name == "stackprotectorcheck") {
2345 NewFn = nullptr;
2346 return true;
2347 }
2348 if (Name.starts_with("strip.invariant.group")) {
2349 // For clang's usage it would be safe to just drop the
2350 // strip.invariant.group, but to be conservative replace with the
2351 // stronger launder.invariant.group instead.
2352 NewFn = Intrinsic::getOrInsertDeclaration(
2353 F->getParent(), Intrinsic::launder_invariant_group,
2354 F->getReturnType());
2355 return true;
2356 }
2357 break;
2358
2359 case 't':
2360 if (Name == "thread.pointer") {
2361 NewFn = Intrinsic::getOrInsertDeclaration(
2362 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2363 return true;
2364 }
2365 break;
2366
2367 case 'v': {
2368 if (Name == "var.annotation" && F->arg_size() == 4) {
2369 rename(F);
2370 NewFn = Intrinsic::getOrInsertDeclaration(
2371 F->getParent(), Intrinsic::var_annotation,
2372 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2373 return true;
2374 }
2375 if (Name.consume_front("vector.splice")) {
2376 if (Name.starts_with(".left") || Name.starts_with(".right"))
2377 break;
2378 return true;
2379 }
2380 if (shouldUpgradeVPIntrinsic(Name))
2381 return true;
2382 break;
2383 }
2384
2385 case 'w':
2386 if (Name.consume_front("wasm.")) {
2387 Intrinsic::ID ID =
2388 StringSwitch<Intrinsic::ID>(Name)
2389 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2390 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2391 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2392 .Default(Intrinsic::not_intrinsic);
2393 if (ID != Intrinsic::not_intrinsic) {
2394 rename(F);
2395 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2396 F->getReturnType());
2397 return true;
2398 }
2399
2400 if (Name.consume_front("dot.i8x16.i7x16.")) {
2401 ID = StringSwitch<Intrinsic::ID>(Name)
2402 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2403 .Case("add.signed",
2404 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2405 .Default(Intrinsic::not_intrinsic);
2406 if (ID != Intrinsic::not_intrinsic) {
2407 rename(F);
2408 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2409 return true;
2410 }
2411 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2412 }
2413 break; // No other 'wasm.*'.
2414 }
2415 break;
2416
2417 case 'x':
2418 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2419 return true;
2420 }
2421
2422 auto *ST = dyn_cast<StructType>(F->getReturnType());
2423 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2424 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2425 // Replace return type with literal non-packed struct. Only do this for
2426 // intrinsics declared to return a struct, not for intrinsics with
2427 // overloaded return type, in which case the exact struct type will be
2428 // mangled into the name.
2429 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2430 FunctionType *FT = F->getFunctionType();
2431 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2432 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2433 std::string Name = F->getName().str();
2434 rename(F);
2435 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2436 Name, F->getParent());
2437
2438 // The new function may also need remangling.
2439 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2440 NewFn = *Result;
2441 return true;
2442 }
2443 }
2444
2445 // Remangle our intrinsic since we upgrade the mangling
2446 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2447 if (Result != std::nullopt) {
2448 NewFn = *Result;
2449 return true;
2450 }
2451
2452 if (upgradeIntrinsicWithDefaultArgs(F, NewFn))
2453 return true;
2454
2455 // This may not belong here. This function is effectively being overloaded
2456 // to both detect an intrinsic which needs upgrading, and to provide the
2457 // upgraded form of the intrinsic. We should perhaps have two separate
2458 // functions for this.
2459
2460 return false;
2461}
2462
2463bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2464 bool CanUpgradeDebugIntrinsicsToRecords) {
2465 NewFn = nullptr;
2466 bool Upgraded =
2467 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2468
2469 // Upgrade intrinsic attributes. This does not change the function.
2470 if (NewFn)
2471 F = NewFn;
2472 if (Intrinsic::ID id = F->getIntrinsicID()) {
2473 // Only do this if the intrinsic signature is valid.
2474 SmallVector<Type *> OverloadTys;
2475 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2476 F->setAttributes(
2477 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2478 }
2479 return Upgraded;
2480}
2481
2482GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2483 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2484 GV->getName() == "llvm.global_dtors")) ||
2485 !GV->hasInitializer())
2486 return nullptr;
2487 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2488 if (!ATy)
2489 return nullptr;
2490 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2491 if (!STy || STy->getNumElements() != 2)
2492 return nullptr;
2493
2494 LLVMContext &C = GV->getContext();
2495 IRBuilder<> IRB(C);
2496 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2497 IRB.getPtrTy());
2498 Constant *Init = GV->getInitializer();
2499 unsigned N = Init->getNumOperands();
2500 std::vector<Constant *> NewCtors(N);
2501 for (unsigned i = 0; i != N; ++i) {
2502 auto Ctor = cast<Constant>(Init->getOperand(i));
2503 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2504 Ctor->getAggregateElement(1),
2505 ConstantPointerNull::get(IRB.getPtrTy()));
2506 }
2507 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2508
2509 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2510 NewInit, GV->getName());
2511}
2512
2513// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2514// to byte shuffles.
2515static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2516 unsigned Shift) {
2517 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2518 unsigned NumElts = ResultTy->getNumElements() * 8;
2519
2520 // Bitcast from a 64-bit element type to a byte element type.
2521 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2522 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2523
2524 // We'll be shuffling in zeroes.
2525 Value *Res = Constant::getNullValue(VecTy);
2526
2527 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2528 // we'll just return the zero vector.
2529 if (Shift < 16) {
2530 int Idxs[64];
2531 // 256/512-bit version is split into 2/4 16-byte lanes.
2532 for (unsigned l = 0; l != NumElts; l += 16)
2533 for (unsigned i = 0; i != 16; ++i) {
2534 unsigned Idx = NumElts + i - Shift;
2535 if (Idx < NumElts)
2536 Idx -= NumElts - 16; // end of lane, switch operand.
2537 Idxs[l + i] = Idx + l;
2538 }
2539
2540 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2541 }
2542
2543 // Bitcast back to a 64-bit element type.
2544 return Builder.CreateBitCast(Res, ResultTy, "cast");
2545}
2546
2547// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2548// to byte shuffles.
2549static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2550 unsigned Shift) {
2551 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2552 unsigned NumElts = ResultTy->getNumElements() * 8;
2553
2554 // Bitcast from a 64-bit element type to a byte element type.
2555 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2556 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2557
2558 // We'll be shuffling in zeroes.
2559 Value *Res = Constant::getNullValue(VecTy);
2560
2561 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2562 // we'll just return the zero vector.
2563 if (Shift < 16) {
2564 int Idxs[64];
2565 // 256/512-bit version is split into 2/4 16-byte lanes.
2566 for (unsigned l = 0; l != NumElts; l += 16)
2567 for (unsigned i = 0; i != 16; ++i) {
2568 unsigned Idx = i + Shift;
2569 if (Idx >= 16)
2570 Idx += NumElts - 16; // end of lane, switch operand.
2571 Idxs[l + i] = Idx + l;
2572 }
2573
2574 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2575 }
2576
2577 // Bitcast back to a 64-bit element type.
2578 return Builder.CreateBitCast(Res, ResultTy, "cast");
2579}
2580
2581static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2582 unsigned NumElts) {
2583 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2584 llvm::VectorType *MaskTy = FixedVectorType::get(
2585 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2586 Mask = Builder.CreateBitCast(Mask, MaskTy);
2587
2588 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2589 // i8 and we need to extract down to the right number of elements.
2590 if (NumElts <= 4) {
2591 int Indices[4];
2592 for (unsigned i = 0; i != NumElts; ++i)
2593 Indices[i] = i;
2594 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2595 "extract");
2596 }
2597
2598 return Mask;
2599}
2600
2601static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2602 Value *Op1) {
2603 // If the mask is all ones just emit the first operation.
2604 if (const auto *C = dyn_cast<Constant>(Mask))
2605 if (C->isAllOnesValue())
2606 return Op0;
2607
2608 Mask = getX86MaskVec(Builder, Mask,
2609 cast<FixedVectorType>(Op0->getType())->getNumElements());
2610 return Builder.CreateSelect(Mask, Op0, Op1);
2611}
2612
2613static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2614 Value *Op1) {
2615 // If the mask is all ones just emit the first operation.
2616 if (const auto *C = dyn_cast<Constant>(Mask))
2617 if (C->isAllOnesValue())
2618 return Op0;
2619
2620 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2621 Mask->getType()->getIntegerBitWidth());
2622 Mask = Builder.CreateBitCast(Mask, MaskTy);
2623 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2624 return Builder.CreateSelect(Mask, Op0, Op1);
2625}
2626
2627// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2628// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2629// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2630static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2631 Value *Op1, Value *Shift,
2632 Value *Passthru, Value *Mask,
2633 bool IsVALIGN) {
2634 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2635
2636 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2637 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2638 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2639 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2640
2641 // Mask the immediate for VALIGN.
2642 if (IsVALIGN)
2643 ShiftVal &= (NumElts - 1);
2644
2645 // If palignr is shifting the pair of vectors more than the size of two
2646 // lanes, emit zero.
2647 if (ShiftVal >= 32)
2648 return llvm::Constant::getNullValue(Op0->getType());
2649
2650 // If palignr is shifting the pair of input vectors more than one lane,
2651 // but less than two lanes, convert to shifting in zeroes.
2652 if (ShiftVal > 16) {
2653 ShiftVal -= 16;
2654 Op1 = Op0;
2655 Op0 = llvm::Constant::getNullValue(Op0->getType());
2656 }
2657
2658 int Indices[64];
2659 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2660 for (unsigned l = 0; l < NumElts; l += 16) {
2661 for (unsigned i = 0; i != 16; ++i) {
2662 unsigned Idx = ShiftVal + i;
2663 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2664 Idx += NumElts - 16; // End of lane, switch operand.
2665 Indices[l + i] = Idx + l;
2666 }
2667 }
2668
2669 Value *Align = Builder.CreateShuffleVector(
2670 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2671
2672 return emitX86Select(Builder, Mask, Align, Passthru);
2673}
2674
2675static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2676 bool ZeroMask, bool IndexForm) {
2677 Type *Ty = CI.getType();
2678 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2679 unsigned EltWidth = Ty->getScalarSizeInBits();
2680 bool IsFloat = Ty->isFPOrFPVectorTy();
2681 Intrinsic::ID IID;
2682 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2683 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2684 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2685 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2686 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2687 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2688 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2689 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2690 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2691 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2692 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2693 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2694 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2695 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2696 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2697 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2698 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2699 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2700 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2701 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2702 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2703 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2704 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2705 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2706 else if (VecWidth == 128 && EltWidth == 16)
2707 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2708 else if (VecWidth == 256 && EltWidth == 16)
2709 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2710 else if (VecWidth == 512 && EltWidth == 16)
2711 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2712 else if (VecWidth == 128 && EltWidth == 8)
2713 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2714 else if (VecWidth == 256 && EltWidth == 8)
2715 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2716 else if (VecWidth == 512 && EltWidth == 8)
2717 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2718 else
2719 llvm_unreachable("Unexpected intrinsic");
2720
2721 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2722 CI.getArgOperand(2) };
2723
2724 // If this isn't index form we need to swap operand 0 and 1.
2725 if (!IndexForm)
2726 std::swap(Args[0], Args[1]);
2727
2728 Value *V = Builder.CreateIntrinsic(IID, Args);
2729 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2730 : Builder.CreateBitCast(CI.getArgOperand(1),
2731 Ty);
2732 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2733}
2734
2735static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2736 Intrinsic::ID IID) {
2737 Type *Ty = CI.getType();
2738 Value *Op0 = CI.getOperand(0);
2739 Value *Op1 = CI.getOperand(1);
2740 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2741
2742 if (CI.arg_size() == 4) { // For masked intrinsics.
2743 Value *VecSrc = CI.getOperand(2);
2744 Value *Mask = CI.getOperand(3);
2745 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2746 }
2747 return Res;
2748}
2749
2750static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2751 bool IsRotateRight) {
2752 Type *Ty = CI.getType();
2753 Value *Src = CI.getArgOperand(0);
2754 Value *Amt = CI.getArgOperand(1);
2755
2756 // Amount may be scalar immediate, in which case create a splat vector.
2757 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2758 // we only care about the lowest log2 bits anyway.
2759 if (Amt->getType() != Ty) {
2760 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2761 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2762 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2763 }
2764
2765 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2766 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2767
2768 if (CI.arg_size() == 4) { // For masked intrinsics.
2769 Value *VecSrc = CI.getOperand(2);
2770 Value *Mask = CI.getOperand(3);
2771 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2772 }
2773 return Res;
2774}
2775
2776static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2777 bool IsSigned) {
2778 Type *Ty = CI.getType();
2779 Value *LHS = CI.getArgOperand(0);
2780 Value *RHS = CI.getArgOperand(1);
2781
2782 CmpInst::Predicate Pred;
2783 switch (Imm) {
2784 case 0x0:
2785 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2786 break;
2787 case 0x1:
2788 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2789 break;
2790 case 0x2:
2791 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2792 break;
2793 case 0x3:
2794 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2795 break;
2796 case 0x4:
2797 Pred = ICmpInst::ICMP_EQ;
2798 break;
2799 case 0x5:
2800 Pred = ICmpInst::ICMP_NE;
2801 break;
2802 case 0x6:
2803 return Constant::getNullValue(Ty); // FALSE
2804 case 0x7:
2805 return Constant::getAllOnesValue(Ty); // TRUE
2806 default:
2807 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2808 }
2809
2810 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2811 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2812 return Ext;
2813}
2814
2815static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2816 bool IsShiftRight, bool ZeroMask) {
2817 Type *Ty = CI.getType();
2818 Value *Op0 = CI.getArgOperand(0);
2819 Value *Op1 = CI.getArgOperand(1);
2820 Value *Amt = CI.getArgOperand(2);
2821
2822 if (IsShiftRight)
2823 std::swap(Op0, Op1);
2824
2825 // Amount may be scalar immediate, in which case create a splat vector.
2826 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2827 // we only care about the lowest log2 bits anyway.
2828 if (Amt->getType() != Ty) {
2829 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2830 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2831 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2832 }
2833
2834 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2835 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2836
2837 unsigned NumArgs = CI.arg_size();
2838 if (NumArgs >= 4) { // For masked intrinsics.
2839 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2840 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2841 CI.getArgOperand(0);
2842 Value *Mask = CI.getOperand(NumArgs - 1);
2843 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2844 }
2845 return Res;
2846}
2847
2848static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2849 Value *Mask, bool Aligned) {
2850 const Align Alignment =
2851 Aligned
2852 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2853 : Align(1);
2854
2855 // If the mask is all ones just emit a regular store.
2856 if (const auto *C = dyn_cast<Constant>(Mask))
2857 if (C->isAllOnesValue())
2858 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2859
2860 // Convert the mask from an integer type to a vector of i1.
2861 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2862 Mask = getX86MaskVec(Builder, Mask, NumElts);
2863 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2864}
2865
2866static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2867 Value *Passthru, Value *Mask, bool Aligned) {
2868 Type *ValTy = Passthru->getType();
2869 const Align Alignment =
2870 Aligned
2871 ? Align(
2872 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2873 8)
2874 : Align(1);
2875
2876 // If the mask is all ones just emit a regular store.
2877 if (const auto *C = dyn_cast<Constant>(Mask))
2878 if (C->isAllOnesValue())
2879 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2880
2881 // Convert the mask from an integer type to a vector of i1.
2882 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2883 Mask = getX86MaskVec(Builder, Mask, NumElts);
2884 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2885}
2886
2887static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2888 Type *Ty = CI.getType();
2889 Value *Op0 = CI.getArgOperand(0);
2890 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2891 {Op0, Builder.getInt1(false)});
2892 if (CI.arg_size() == 3)
2893 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2894 return Res;
2895}
2896
2897static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2898 Type *Ty = CI.getType();
2899
2900 // Arguments have a vXi32 type so cast to vXi64.
2901 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2902 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2903
2904 if (IsSigned) {
2905 // Shift left then arithmetic shift right.
2906 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2907 LHS = Builder.CreateShl(LHS, ShiftAmt);
2908 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2909 RHS = Builder.CreateShl(RHS, ShiftAmt);
2910 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2911 } else {
2912 // Clear the upper bits.
2913 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2914 LHS = Builder.CreateAnd(LHS, Mask);
2915 RHS = Builder.CreateAnd(RHS, Mask);
2916 }
2917
2918 Value *Res = Builder.CreateMul(LHS, RHS);
2919
2920 if (CI.arg_size() == 4)
2921 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2922
2923 return Res;
2924}
2925
2926// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2927static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2928 Value *Mask) {
2929 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2930 if (Mask) {
2931 const auto *C = dyn_cast<Constant>(Mask);
2932 if (!C || !C->isAllOnesValue())
2933 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2934 }
2935
2936 if (NumElts < 8) {
2937 int Indices[8];
2938 for (unsigned i = 0; i != NumElts; ++i)
2939 Indices[i] = i;
2940 for (unsigned i = NumElts; i != 8; ++i)
2941 Indices[i] = NumElts + i % NumElts;
2942 Vec = Builder.CreateShuffleVector(Vec,
2943 Constant::getNullValue(Vec->getType()),
2944 Indices);
2945 }
2946 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2947}
2948
2949static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2950 unsigned CC, bool Signed) {
2951 Value *Op0 = CI.getArgOperand(0);
2952 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2953
2954 Value *Cmp;
2955 if (CC == 3) {
2956 Cmp = Constant::getNullValue(
2957 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2958 } else if (CC == 7) {
2959 Cmp = Constant::getAllOnesValue(
2960 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2961 } else {
2962 ICmpInst::Predicate Pred;
2963 switch (CC) {
2964 default: llvm_unreachable("Unknown condition code");
2965 case 0: Pred = ICmpInst::ICMP_EQ; break;
2966 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2967 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2968 case 4: Pred = ICmpInst::ICMP_NE; break;
2969 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2970 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2971 }
2972 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2973 }
2974
2975 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2976
2977 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2978}
2979
2980// Replace a masked intrinsic with an older unmasked intrinsic.
2981static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2982 Intrinsic::ID IID) {
2983 Value *Rep =
2984 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2985 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2986}
2987
2988static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2989 Value* A = CI.getArgOperand(0);
2990 Value* B = CI.getArgOperand(1);
2991 Value* Src = CI.getArgOperand(2);
2992 Value* Mask = CI.getArgOperand(3);
2993
2994 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2995 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2996 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2997 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2998 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2999 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
3000}
3001
3002static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
3003 Value* Op = CI.getArgOperand(0);
3004 Type* ReturnOp = CI.getType();
3005 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
3006 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
3007 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
3008}
3009
3010// Replace intrinsic with unmasked version and a select.
3011static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
3012 CallBase &CI, Value *&Rep) {
3013 Name = Name.substr(12); // Remove avx512.mask.
3014
3015 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
3016 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
3017 Intrinsic::ID IID;
3018 if (Name.starts_with("max.p")) {
3019 if (VecWidth == 128 && EltWidth == 32)
3020 IID = Intrinsic::x86_sse_max_ps;
3021 else if (VecWidth == 128 && EltWidth == 64)
3022 IID = Intrinsic::x86_sse2_max_pd;
3023 else if (VecWidth == 256 && EltWidth == 32)
3024 IID = Intrinsic::x86_avx_max_ps_256;
3025 else if (VecWidth == 256 && EltWidth == 64)
3026 IID = Intrinsic::x86_avx_max_pd_256;
3027 else
3028 llvm_unreachable("Unexpected intrinsic");
3029 } else if (Name.starts_with("min.p")) {
3030 if (VecWidth == 128 && EltWidth == 32)
3031 IID = Intrinsic::x86_sse_min_ps;
3032 else if (VecWidth == 128 && EltWidth == 64)
3033 IID = Intrinsic::x86_sse2_min_pd;
3034 else if (VecWidth == 256 && EltWidth == 32)
3035 IID = Intrinsic::x86_avx_min_ps_256;
3036 else if (VecWidth == 256 && EltWidth == 64)
3037 IID = Intrinsic::x86_avx_min_pd_256;
3038 else
3039 llvm_unreachable("Unexpected intrinsic");
3040 } else if (Name.starts_with("pshuf.b.")) {
3041 if (VecWidth == 128)
3042 IID = Intrinsic::x86_ssse3_pshuf_b_128;
3043 else if (VecWidth == 256)
3044 IID = Intrinsic::x86_avx2_pshuf_b;
3045 else if (VecWidth == 512)
3046 IID = Intrinsic::x86_avx512_pshuf_b_512;
3047 else
3048 llvm_unreachable("Unexpected intrinsic");
3049 } else if (Name.starts_with("pmul.hr.sw.")) {
3050 if (VecWidth == 128)
3051 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
3052 else if (VecWidth == 256)
3053 IID = Intrinsic::x86_avx2_pmul_hr_sw;
3054 else if (VecWidth == 512)
3055 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
3056 else
3057 llvm_unreachable("Unexpected intrinsic");
3058 } else if (Name.starts_with("pmulh.w.")) {
3059 if (VecWidth == 128)
3060 IID = Intrinsic::x86_sse2_pmulh_w;
3061 else if (VecWidth == 256)
3062 IID = Intrinsic::x86_avx2_pmulh_w;
3063 else if (VecWidth == 512)
3064 IID = Intrinsic::x86_avx512_pmulh_w_512;
3065 else
3066 llvm_unreachable("Unexpected intrinsic");
3067 } else if (Name.starts_with("pmulhu.w.")) {
3068 if (VecWidth == 128)
3069 IID = Intrinsic::x86_sse2_pmulhu_w;
3070 else if (VecWidth == 256)
3071 IID = Intrinsic::x86_avx2_pmulhu_w;
3072 else if (VecWidth == 512)
3073 IID = Intrinsic::x86_avx512_pmulhu_w_512;
3074 else
3075 llvm_unreachable("Unexpected intrinsic");
3076 } else if (Name.starts_with("pmaddw.d.")) {
3077 if (VecWidth == 128)
3078 IID = Intrinsic::x86_sse2_pmadd_wd;
3079 else if (VecWidth == 256)
3080 IID = Intrinsic::x86_avx2_pmadd_wd;
3081 else if (VecWidth == 512)
3082 IID = Intrinsic::x86_avx512_pmaddw_d_512;
3083 else
3084 llvm_unreachable("Unexpected intrinsic");
3085 } else if (Name.starts_with("pmaddubs.w.")) {
3086 if (VecWidth == 128)
3087 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
3088 else if (VecWidth == 256)
3089 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
3090 else if (VecWidth == 512)
3091 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
3092 else
3093 llvm_unreachable("Unexpected intrinsic");
3094 } else if (Name.starts_with("packsswb.")) {
3095 if (VecWidth == 128)
3096 IID = Intrinsic::x86_sse2_packsswb_128;
3097 else if (VecWidth == 256)
3098 IID = Intrinsic::x86_avx2_packsswb;
3099 else if (VecWidth == 512)
3100 IID = Intrinsic::x86_avx512_packsswb_512;
3101 else
3102 llvm_unreachable("Unexpected intrinsic");
3103 } else if (Name.starts_with("packssdw.")) {
3104 if (VecWidth == 128)
3105 IID = Intrinsic::x86_sse2_packssdw_128;
3106 else if (VecWidth == 256)
3107 IID = Intrinsic::x86_avx2_packssdw;
3108 else if (VecWidth == 512)
3109 IID = Intrinsic::x86_avx512_packssdw_512;
3110 else
3111 llvm_unreachable("Unexpected intrinsic");
3112 } else if (Name.starts_with("packuswb.")) {
3113 if (VecWidth == 128)
3114 IID = Intrinsic::x86_sse2_packuswb_128;
3115 else if (VecWidth == 256)
3116 IID = Intrinsic::x86_avx2_packuswb;
3117 else if (VecWidth == 512)
3118 IID = Intrinsic::x86_avx512_packuswb_512;
3119 else
3120 llvm_unreachable("Unexpected intrinsic");
3121 } else if (Name.starts_with("packusdw.")) {
3122 if (VecWidth == 128)
3123 IID = Intrinsic::x86_sse41_packusdw;
3124 else if (VecWidth == 256)
3125 IID = Intrinsic::x86_avx2_packusdw;
3126 else if (VecWidth == 512)
3127 IID = Intrinsic::x86_avx512_packusdw_512;
3128 else
3129 llvm_unreachable("Unexpected intrinsic");
3130 } else if (Name.starts_with("vpermilvar.")) {
3131 if (VecWidth == 128 && EltWidth == 32)
3132 IID = Intrinsic::x86_avx_vpermilvar_ps;
3133 else if (VecWidth == 128 && EltWidth == 64)
3134 IID = Intrinsic::x86_avx_vpermilvar_pd;
3135 else if (VecWidth == 256 && EltWidth == 32)
3136 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
3137 else if (VecWidth == 256 && EltWidth == 64)
3138 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
3139 else if (VecWidth == 512 && EltWidth == 32)
3140 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
3141 else if (VecWidth == 512 && EltWidth == 64)
3142 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
3143 else
3144 llvm_unreachable("Unexpected intrinsic");
3145 } else if (Name == "cvtpd2dq.256") {
3146 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
3147 } else if (Name == "cvtpd2ps.256") {
3148 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
3149 } else if (Name == "cvttpd2dq.256") {
3150 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
3151 } else if (Name == "cvttps2dq.128") {
3152 IID = Intrinsic::x86_sse2_cvttps2dq;
3153 } else if (Name == "cvttps2dq.256") {
3154 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
3155 } else if (Name.starts_with("permvar.")) {
3156 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
3157 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
3158 IID = Intrinsic::x86_avx2_permps;
3159 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
3160 IID = Intrinsic::x86_avx2_permd;
3161 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
3162 IID = Intrinsic::x86_avx512_permvar_df_256;
3163 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
3164 IID = Intrinsic::x86_avx512_permvar_di_256;
3165 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
3166 IID = Intrinsic::x86_avx512_permvar_sf_512;
3167 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
3168 IID = Intrinsic::x86_avx512_permvar_si_512;
3169 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
3170 IID = Intrinsic::x86_avx512_permvar_df_512;
3171 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
3172 IID = Intrinsic::x86_avx512_permvar_di_512;
3173 else if (VecWidth == 128 && EltWidth == 16)
3174 IID = Intrinsic::x86_avx512_permvar_hi_128;
3175 else if (VecWidth == 256 && EltWidth == 16)
3176 IID = Intrinsic::x86_avx512_permvar_hi_256;
3177 else if (VecWidth == 512 && EltWidth == 16)
3178 IID = Intrinsic::x86_avx512_permvar_hi_512;
3179 else if (VecWidth == 128 && EltWidth == 8)
3180 IID = Intrinsic::x86_avx512_permvar_qi_128;
3181 else if (VecWidth == 256 && EltWidth == 8)
3182 IID = Intrinsic::x86_avx512_permvar_qi_256;
3183 else if (VecWidth == 512 && EltWidth == 8)
3184 IID = Intrinsic::x86_avx512_permvar_qi_512;
3185 else
3186 llvm_unreachable("Unexpected intrinsic");
3187 } else if (Name.starts_with("dbpsadbw.")) {
3188 if (VecWidth == 128)
3189 IID = Intrinsic::x86_avx512_dbpsadbw_128;
3190 else if (VecWidth == 256)
3191 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3192 else if (VecWidth == 512)
3193 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3194 else
3195 llvm_unreachable("Unexpected intrinsic");
3196 } else if (Name.starts_with("pmultishift.qb.")) {
3197 if (VecWidth == 128)
3198 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3199 else if (VecWidth == 256)
3200 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3201 else if (VecWidth == 512)
3202 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3203 else
3204 llvm_unreachable("Unexpected intrinsic");
3205 } else if (Name.starts_with("conflict.")) {
3206 if (Name[9] == 'd' && VecWidth == 128)
3207 IID = Intrinsic::x86_avx512_conflict_d_128;
3208 else if (Name[9] == 'd' && VecWidth == 256)
3209 IID = Intrinsic::x86_avx512_conflict_d_256;
3210 else if (Name[9] == 'd' && VecWidth == 512)
3211 IID = Intrinsic::x86_avx512_conflict_d_512;
3212 else if (Name[9] == 'q' && VecWidth == 128)
3213 IID = Intrinsic::x86_avx512_conflict_q_128;
3214 else if (Name[9] == 'q' && VecWidth == 256)
3215 IID = Intrinsic::x86_avx512_conflict_q_256;
3216 else if (Name[9] == 'q' && VecWidth == 512)
3217 IID = Intrinsic::x86_avx512_conflict_q_512;
3218 else
3219 llvm_unreachable("Unexpected intrinsic");
3220 } else if (Name.starts_with("pavg.")) {
3221 if (Name[5] == 'b' && VecWidth == 128)
3222 IID = Intrinsic::x86_sse2_pavg_b;
3223 else if (Name[5] == 'b' && VecWidth == 256)
3224 IID = Intrinsic::x86_avx2_pavg_b;
3225 else if (Name[5] == 'b' && VecWidth == 512)
3226 IID = Intrinsic::x86_avx512_pavg_b_512;
3227 else if (Name[5] == 'w' && VecWidth == 128)
3228 IID = Intrinsic::x86_sse2_pavg_w;
3229 else if (Name[5] == 'w' && VecWidth == 256)
3230 IID = Intrinsic::x86_avx2_pavg_w;
3231 else if (Name[5] == 'w' && VecWidth == 512)
3232 IID = Intrinsic::x86_avx512_pavg_w_512;
3233 else
3234 llvm_unreachable("Unexpected intrinsic");
3235 } else
3236 return false;
3237
3238 SmallVector<Value *, 4> Args(CI.args());
3239 Args.pop_back();
3240 Args.pop_back();
3241 Rep = Builder.CreateIntrinsic(IID, Args);
3242 unsigned NumArgs = CI.arg_size();
3243 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3244 CI.getArgOperand(NumArgs - 2));
3245 return true;
3246}
3247
3248/// Upgrade comment in call to inline asm that represents an objc retain release
3249/// marker.
3250void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3251 size_t Pos;
3252 if (AsmStr->find("mov\tfp") == 0 &&
3253 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3254 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3255 AsmStr->replace(Pos, 1, ";");
3256 }
3257}
3258
3259static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3260 Function *F, IRBuilder<> &Builder) {
3261 Value *Rep = nullptr;
3262
3263 if (Name == "abs.i" || Name == "abs.ll") {
3264 Value *Arg = CI->getArgOperand(0);
3265 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3266 {Arg, Builder.getTrue()},
3267 /*FMFSource=*/nullptr, "abs");
3268 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3269 Type *Ty = (Name == "abs.bf16")
3270 ? Builder.getBFloatTy()
3271 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3272 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3273 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3274 Rep = Builder.CreateBitCast(Abs, CI->getType());
3275 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3276 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3277 : Intrinsic::nvvm_fabs;
3278 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3279 } else if (Name.consume_front("add.")) {
3280 // nvvm.add.<rnd>{.ftz}{.sat}.{f,d,f16,v2f16}
3281 auto FAdd = getNVVMFAddUpgrade(Name);
3282 assert(FAdd && "unsupported nvvm.add.* intrinsic");
3283 auto [IID, RoundingMode] = *FAdd;
3284 Value *A = CI->getArgOperand(0);
3285 Rep = Builder.CreateIntrinsic(
3286 A->getType(), IID,
3287 {A, CI->getArgOperand(1),
3288 Builder.getInt32(static_cast<int>(RoundingMode))});
3289 } else if (Name.consume_front("ex2.approx.")) {
3290 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3291 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3292 : Intrinsic::nvvm_ex2_approx;
3293 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3294 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3295 Name.starts_with("atomic.load.add.f64.p")) {
3296 Value *Ptr = CI->getArgOperand(0);
3297 Value *Val = CI->getArgOperand(1);
3298 Rep = Builder.CreateAtomicRMW(
3299 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3300 CI->getContext().getOrInsertSyncScopeID("device"));
3301 // The default scope for atomic.load.* intrinsics is device
3302 // (= gpu scope in ptx), but the default LLVM atomic scope is
3303 // "system"
3304 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3305 Name.starts_with("atomic.load.dec.32.p")) {
3306 Value *Ptr = CI->getArgOperand(0);
3307 Value *Val = CI->getArgOperand(1);
3308 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3309 : AtomicRMWInst::UDecWrap;
3310 Rep = Builder.CreateAtomicRMW(
3311 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3312 CI->getContext().getOrInsertSyncScopeID("device"));
3313 // See comment above.
3314 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3315 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3316 StringRef Op = Name.substr(StringRef("atomic.").size());
3317 Value *Ptr = CI->getArgOperand(0);
3318 Value *Val = CI->getArgOperand(1);
3319 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3320 Op.contains(".cta.") ? "block" : "");
3321 if (Op.starts_with("cas.")) {
3322 Value *New = CI->getArgOperand(2);
3323 Value *Pair = Builder.CreateAtomicCmpXchg(
3324 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3325 AtomicOrdering::Monotonic, SSID);
3326 Rep = Builder.CreateExtractValue(Pair, 0);
3327 } else {
3328 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3329 // because we were actually missing those intrinsics!
3330 AtomicRMWInst::BinOp BinOp =
3331 StringSwitch<AtomicRMWInst::BinOp>(Op)
3332 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3333 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3334 .StartsWith("exch.", AtomicRMWInst::Xchg)
3335 .StartsWith("max.", AtomicRMWInst::Max)
3336 .StartsWith("min.", AtomicRMWInst::Min)
3337 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3338 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3339 .StartsWith("and.", AtomicRMWInst::And)
3340 .StartsWith("or.", AtomicRMWInst::Or)
3341 .StartsWith("xor.", AtomicRMWInst::Xor)
3342 .Default(AtomicRMWInst::BAD_BINOP);
3343 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3344 "unexpected nvvm scoped atomic intrinsic");
3345 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3346 AtomicOrdering::Monotonic, SSID);
3347 }
3348 } else if (Name == "clz.ll") {
3349 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3350 Value *Arg = CI->getArgOperand(0);
3351 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3352 {Arg, Builder.getFalse()},
3353 /*FMFSource=*/nullptr, "ctlz");
3354 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3355 } else if (Name == "popc.ll") {
3356 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3357 // i64.
3358 Value *Arg = CI->getArgOperand(0);
3359 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3360 Arg, /*FMFSource=*/nullptr, "ctpop");
3361 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3362 } else if (Name == "h2f") {
3363 Value *Cast =
3364 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3365 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3366 } else if (Name.consume_front("bitcast.") &&
3367 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3368 Name == "d2ll")) {
3369 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3370 } else if (Name == "rotate.b32") {
3371 Value *Arg = CI->getOperand(0);
3372 Value *ShiftAmt = CI->getOperand(1);
3373 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3374 {Arg, Arg, ShiftAmt});
3375 } else if (Name == "rotate.b64") {
3376 Type *Int64Ty = Builder.getInt64Ty();
3377 Value *Arg = CI->getOperand(0);
3378 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3379 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3380 {Arg, Arg, ZExtShiftAmt});
3381 } else if (Name == "rotate.right.b64") {
3382 Type *Int64Ty = Builder.getInt64Ty();
3383 Value *Arg = CI->getOperand(0);
3384 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3385 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3386 {Arg, Arg, ZExtShiftAmt});
3387 } else if (Name == "swap.lo.hi.b64") {
3388 Type *Int64Ty = Builder.getInt64Ty();
3389 Value *Arg = CI->getOperand(0);
3390 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3391 {Arg, Arg, Builder.getInt64(32)});
3392 } else if ((Name.consume_front("ptr.gen.to.") &&
3393 consumeNVVMPtrAddrSpace(Name)) ||
3394 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3395 Name.starts_with(".to.gen"))) {
3396 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3397 } else if (Name.consume_front("ldg.global")) {
3398 Value *Ptr = CI->getArgOperand(0);
3399 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3400 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3401 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3402 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3403 MDNode *MD = MDNode::get(Builder.getContext(), {});
3404 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3405 return LD;
3406 } else if (Name == "tanh.approx.f32") {
3407 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3408 FastMathFlags FMF;
3409 FMF.setApproxFunc();
3410 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3411 FMF);
3412 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3413 Value *Arg =
3414 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3415 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3416 {}, {Arg});
3417 } else if (Name == "barrier") {
3418 Rep = Builder.CreateIntrinsic(
3419 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3420 {CI->getArgOperand(0), CI->getArgOperand(1)});
3421 } else if (Name == "barrier.sync") {
3422 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3423 {CI->getArgOperand(0)});
3424 } else if (Name == "barrier.sync.cnt") {
3425 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3426 {CI->getArgOperand(0), CI->getArgOperand(1)});
3427 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3428 Name == "barrier0.or") {
3429 Value *C = CI->getArgOperand(0);
3430 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3431
3432 Intrinsic::ID IID =
3433 StringSwitch<Intrinsic::ID>(Name)
3434 .Case("barrier0.popc",
3435 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3436 .Case("barrier0.and",
3437 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3438 .Case("barrier0.or",
3439 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3440 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3441 Rep = Builder.CreateZExt(Bar, CI->getType());
3442 } else {
3443 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3444 if (IID != Intrinsic::not_intrinsic &&
3445 isLegacyNVPTXBF16IntSignature(F, IID)) {
3446 rename(F);
3447 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3448 SmallVector<Value *, 2> Args;
3449 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3450 Value *Arg = CI->getArgOperand(I);
3451 Type *OldType = Arg->getType();
3452 Type *NewType = NewFn->getArg(I)->getType();
3453 Args.push_back(
3454 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3455 ? Builder.CreateBitCast(Arg, NewType)
3456 : Arg);
3457 }
3458 Rep = Builder.CreateCall(NewFn, Args);
3459 if (F->getReturnType()->isIntegerTy())
3460 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3461 }
3462 }
3463
3464 return Rep;
3465}
3466
3467static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3468 IRBuilder<> &Builder) {
3469 LLVMContext &C = F->getContext();
3470 Value *Rep = nullptr;
3471
3472 if (Name.starts_with("sse4a.movnt.")) {
3473 SmallVector<Metadata *, 1> Elts;
3474 Elts.push_back(
3475 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3476 MDNode *Node = MDNode::get(C, Elts);
3477
3478 Value *Arg0 = CI->getArgOperand(0);
3479 Value *Arg1 = CI->getArgOperand(1);
3480
3481 // Nontemporal (unaligned) store of the 0'th element of the float/double
3482 // vector.
3483 Value *Extract =
3484 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3485
3486 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3487 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3488 } else if (Name.starts_with("avx.movnt.") ||
3489 Name.starts_with("avx512.storent.")) {
3490 SmallVector<Metadata *, 1> Elts;
3491 Elts.push_back(
3492 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3493 MDNode *Node = MDNode::get(C, Elts);
3494
3495 Value *Arg0 = CI->getArgOperand(0);
3496 Value *Arg1 = CI->getArgOperand(1);
3497
3498 StoreInst *SI = Builder.CreateAlignedStore(
3499 Arg1, Arg0,
3500 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3501 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3502 } else if (Name == "sse2.storel.dq") {
3503 Value *Arg0 = CI->getArgOperand(0);
3504 Value *Arg1 = CI->getArgOperand(1);
3505
3506 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3507 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3508 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3509 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3510 } else if (Name.starts_with("sse.storeu.") ||
3511 Name.starts_with("sse2.storeu.") ||
3512 Name.starts_with("avx.storeu.")) {
3513 Value *Arg0 = CI->getArgOperand(0);
3514 Value *Arg1 = CI->getArgOperand(1);
3515 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3516 } else if (Name == "avx512.mask.store.ss") {
3517 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3518 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3519 Mask, false);
3520 } else if (Name.starts_with("avx512.mask.store")) {
3521 // "avx512.mask.storeu." or "avx512.mask.store."
3522 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3523 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3524 CI->getArgOperand(2), Aligned);
3525 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3526 // Upgrade packed integer vector compare intrinsics to compare instructions.
3527 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3528 bool CmpEq = Name[9] == 'e';
3529 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3530 CI->getArgOperand(0), CI->getArgOperand(1));
3531 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3532 } else if (Name.starts_with("avx512.broadcastm")) {
3533 Type *ExtTy = Type::getInt32Ty(C);
3534 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3535 ExtTy = Type::getInt64Ty(C);
3536 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3537 ExtTy->getPrimitiveSizeInBits();
3538 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3539 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3540 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3541 Value *Vec = CI->getArgOperand(0);
3542 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3543 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3544 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3545 } else if (Name.starts_with("avx.sqrt.p") ||
3546 Name.starts_with("sse2.sqrt.p") ||
3547 Name.starts_with("sse.sqrt.p")) {
3548 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3549 {CI->getArgOperand(0)});
3550 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3551 if (CI->arg_size() == 4 &&
3552 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3553 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3554 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3555 : Intrinsic::x86_avx512_sqrt_pd_512;
3556
3557 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3558 Rep = Builder.CreateIntrinsic(IID, Args);
3559 } else {
3560 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3561 {CI->getArgOperand(0)});
3562 }
3563 Rep =
3564 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3565 } else if (Name.starts_with("avx512.ptestm") ||
3566 Name.starts_with("avx512.ptestnm")) {
3567 Value *Op0 = CI->getArgOperand(0);
3568 Value *Op1 = CI->getArgOperand(1);
3569 Value *Mask = CI->getArgOperand(2);
3570 Rep = Builder.CreateAnd(Op0, Op1);
3571 llvm::Type *Ty = Op0->getType();
3572 Value *Zero = llvm::Constant::getNullValue(Ty);
3573 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3574 ? ICmpInst::ICMP_NE
3575 : ICmpInst::ICMP_EQ;
3576 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3577 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3578 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3579 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3580 ->getNumElements();
3581 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3582 Rep =
3583 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3584 } else if (Name.starts_with("avx512.kunpck")) {
3585 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3586 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3587 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3588 int Indices[64];
3589 for (unsigned i = 0; i != NumElts; ++i)
3590 Indices[i] = i;
3591
3592 // First extract half of each vector. This gives better codegen than
3593 // doing it in a single shuffle.
3594 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3595 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3596 // Concat the vectors.
3597 // NOTE: Operands have to be swapped to match intrinsic definition.
3598 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3599 Rep = Builder.CreateBitCast(Rep, CI->getType());
3600 } else if (Name == "avx512.kand.w") {
3601 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3602 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3603 Rep = Builder.CreateAnd(LHS, RHS);
3604 Rep = Builder.CreateBitCast(Rep, CI->getType());
3605 } else if (Name == "avx512.kandn.w") {
3606 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3607 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3608 LHS = Builder.CreateNot(LHS);
3609 Rep = Builder.CreateAnd(LHS, RHS);
3610 Rep = Builder.CreateBitCast(Rep, CI->getType());
3611 } else if (Name == "avx512.kor.w") {
3612 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3613 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3614 Rep = Builder.CreateOr(LHS, RHS);
3615 Rep = Builder.CreateBitCast(Rep, CI->getType());
3616 } else if (Name == "avx512.kxor.w") {
3617 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3618 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3619 Rep = Builder.CreateXor(LHS, RHS);
3620 Rep = Builder.CreateBitCast(Rep, CI->getType());
3621 } else if (Name == "avx512.kxnor.w") {
3622 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3623 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3624 LHS = Builder.CreateNot(LHS);
3625 Rep = Builder.CreateXor(LHS, RHS);
3626 Rep = Builder.CreateBitCast(Rep, CI->getType());
3627 } else if (Name == "avx512.knot.w") {
3628 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3629 Rep = Builder.CreateNot(Rep);
3630 Rep = Builder.CreateBitCast(Rep, CI->getType());
3631 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3632 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3633 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3634 Rep = Builder.CreateOr(LHS, RHS);
3635 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3636 Value *C;
3637 if (Name[14] == 'c')
3638 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3639 else
3640 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3641 Rep = Builder.CreateICmpEQ(Rep, C);
3642 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3643 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3644 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3645 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3646 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3647 Type *I32Ty = Type::getInt32Ty(C);
3648 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3649 ConstantInt::get(I32Ty, 0));
3650 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3651 ConstantInt::get(I32Ty, 0));
3652 Value *EltOp;
3653 if (Name.contains(".add."))
3654 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3655 else if (Name.contains(".sub."))
3656 EltOp = Builder.CreateFSub(Elt0, Elt1);
3657 else if (Name.contains(".mul."))
3658 EltOp = Builder.CreateFMul(Elt0, Elt1);
3659 else
3660 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3661 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3662 ConstantInt::get(I32Ty, 0));
3663 } else if (Name.starts_with("avx512.mask.pcmp")) {
3664 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3665 bool CmpEq = Name[16] == 'e';
3666 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3667 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3668 Type *OpTy = CI->getArgOperand(0)->getType();
3669 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3670 Intrinsic::ID IID;
3671 switch (VecWidth) {
3672 default:
3673 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3674 break;
3675 case 128:
3676 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3677 break;
3678 case 256:
3679 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3680 break;
3681 case 512:
3682 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3683 break;
3684 }
3685
3686 Rep =
3687 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3688 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3689 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3690 Type *OpTy = CI->getArgOperand(0)->getType();
3691 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3692 unsigned EltWidth = OpTy->getScalarSizeInBits();
3693 Intrinsic::ID IID;
3694 if (VecWidth == 128 && EltWidth == 32)
3695 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3696 else if (VecWidth == 256 && EltWidth == 32)
3697 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3698 else if (VecWidth == 512 && EltWidth == 32)
3699 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3700 else if (VecWidth == 128 && EltWidth == 64)
3701 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3702 else if (VecWidth == 256 && EltWidth == 64)
3703 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3704 else if (VecWidth == 512 && EltWidth == 64)
3705 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3706 else
3707 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3708
3709 Rep =
3710 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3711 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3712 } else if (Name.starts_with("avx512.cmp.p")) {
3713 SmallVector<Value *, 4> Args(CI->args());
3714 Type *OpTy = Args[0]->getType();
3715 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3716 unsigned EltWidth = OpTy->getScalarSizeInBits();
3717 Intrinsic::ID IID;
3718 if (VecWidth == 128 && EltWidth == 32)
3719 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3720 else if (VecWidth == 256 && EltWidth == 32)
3721 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3722 else if (VecWidth == 512 && EltWidth == 32)
3723 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3724 else if (VecWidth == 128 && EltWidth == 64)
3725 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3726 else if (VecWidth == 256 && EltWidth == 64)
3727 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3728 else if (VecWidth == 512 && EltWidth == 64)
3729 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3730 else
3731 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3732
3733 Value *Mask = Constant::getAllOnesValue(CI->getType());
3734 if (VecWidth == 512)
3735 std::swap(Mask, Args.back());
3736 Args.push_back(Mask);
3737
3738 Rep = Builder.CreateIntrinsic(IID, Args);
3739 } else if (Name.starts_with("avx512.mask.cmp.")) {
3740 // Integer compare intrinsics.
3741 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3742 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3743 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3744 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3745 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3746 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3747 Name.starts_with("avx512.cvtw2mask.") ||
3748 Name.starts_with("avx512.cvtd2mask.") ||
3749 Name.starts_with("avx512.cvtq2mask.")) {
3750 Value *Op = CI->getArgOperand(0);
3751 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3752 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3753 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3754 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3755 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3756 Name.starts_with("avx512.mask.pabs")) {
3757 Rep = upgradeAbs(Builder, *CI);
3758 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3759 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3760 Name.starts_with("avx512.mask.pmaxs")) {
3761 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3762 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3763 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3764 Name.starts_with("avx512.mask.pmaxu")) {
3765 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3766 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3767 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3768 Name.starts_with("avx512.mask.pmins")) {
3769 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3770 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3771 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3772 Name.starts_with("avx512.mask.pminu")) {
3773 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3774 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3775 Name == "avx512.pmulu.dq.512" ||
3776 Name.starts_with("avx512.mask.pmulu.dq.")) {
3777 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3778 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3779 Name == "avx512.pmul.dq.512" ||
3780 Name.starts_with("avx512.mask.pmul.dq.")) {
3781 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3782 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3783 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3784 Rep =
3785 Builder.CreateSIToFP(CI->getArgOperand(1),
3786 cast<VectorType>(CI->getType())->getElementType());
3787 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3788 } else if (Name == "avx512.cvtusi2sd") {
3789 Rep =
3790 Builder.CreateUIToFP(CI->getArgOperand(1),
3791 cast<VectorType>(CI->getType())->getElementType());
3792 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3793 } else if (Name == "sse2.cvtss2sd") {
3794 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3795 Rep = Builder.CreateFPExt(
3796 Rep, cast<VectorType>(CI->getType())->getElementType());
3797 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3798 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3799 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3800 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3801 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3802 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3803 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3804 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3805 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3806 Name == "avx512.mask.cvtqq2ps.256" ||
3807 Name == "avx512.mask.cvtqq2ps.512" ||
3808 Name == "avx512.mask.cvtuqq2ps.256" ||
3809 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3810 Name == "avx.cvt.ps2.pd.256" ||
3811 Name == "avx512.mask.cvtps2pd.128" ||
3812 Name == "avx512.mask.cvtps2pd.256") {
3813 auto *DstTy = cast<FixedVectorType>(CI->getType());
3814 Rep = CI->getArgOperand(0);
3815 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3816
3817 unsigned NumDstElts = DstTy->getNumElements();
3818 if (NumDstElts < SrcTy->getNumElements()) {
3819 assert(NumDstElts == 2 && "Unexpected vector size");
3820 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3821 }
3822
3823 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3824 bool IsUnsigned = Name.contains("cvtu");
3825 if (IsPS2PD)
3826 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3827 else if (CI->arg_size() == 4 &&
3828 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3829 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3830 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3831 : Intrinsic::x86_avx512_sitofp_round;
3832 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3833 {Rep, CI->getArgOperand(3)});
3834 } else {
3835 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3836 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3837 }
3838
3839 if (CI->arg_size() >= 3)
3840 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3841 CI->getArgOperand(1));
3842 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3843 Name.starts_with("vcvtph2ps.")) {
3844 auto *DstTy = cast<FixedVectorType>(CI->getType());
3845 Rep = CI->getArgOperand(0);
3846 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3847 unsigned NumDstElts = DstTy->getNumElements();
3848 if (NumDstElts != SrcTy->getNumElements()) {
3849 assert(NumDstElts == 4 && "Unexpected vector size");
3850 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3851 }
3852 Rep = Builder.CreateBitCast(
3853 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3854 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3855 if (CI->arg_size() >= 3)
3856 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3857 CI->getArgOperand(1));
3858 } else if (Name.starts_with("avx512.mask.load")) {
3859 // "avx512.mask.loadu." or "avx512.mask.load."
3860 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3861 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3862 CI->getArgOperand(2), Aligned);
3863 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3864 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3865 auto *PtrTy = CI->getOperand(0)->getType();
3866 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3867 ResultTy->getNumElements());
3868 Rep = Builder.CreateIntrinsic(
3869 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3870 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3871 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3872 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3873 auto *PtrTy = CI->getArgOperand(0)->getType();
3874 Value *MaskVec =
3875 getX86MaskVec(Builder, CI->getArgOperand(2),
3876 cast<FixedVectorType>(ResultTy)->getNumElements());
3877 Rep = Builder.CreateIntrinsic(
3878 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3879 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3880 } else if (Name.starts_with("avx512.mask.compress.") ||
3881 Name.starts_with("avx512.mask.expand.")) {
3882 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3883
3884 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3885 ResultTy->getNumElements());
3886
3887 bool IsCompress = Name[12] == 'c';
3888 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3889 : Intrinsic::x86_avx512_mask_expand;
3890 Rep = Builder.CreateIntrinsic(
3891 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3892 } else if (Name.starts_with("xop.vpcom")) {
3893 bool IsSigned;
3894 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3895 Name.ends_with("uq"))
3896 IsSigned = false;
3897 else if (Name.ends_with("b") || Name.ends_with("w") ||
3898 Name.ends_with("d") || Name.ends_with("q"))
3899 IsSigned = true;
3900 else
3901 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3902
3903 unsigned Imm;
3904 if (CI->arg_size() == 3) {
3905 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3906 } else {
3907 Name = Name.substr(9); // strip off "xop.vpcom"
3908 if (Name.starts_with("lt"))
3909 Imm = 0;
3910 else if (Name.starts_with("le"))
3911 Imm = 1;
3912 else if (Name.starts_with("gt"))
3913 Imm = 2;
3914 else if (Name.starts_with("ge"))
3915 Imm = 3;
3916 else if (Name.starts_with("eq"))
3917 Imm = 4;
3918 else if (Name.starts_with("ne"))
3919 Imm = 5;
3920 else if (Name.starts_with("false"))
3921 Imm = 6;
3922 else if (Name.starts_with("true"))
3923 Imm = 7;
3924 else
3925 llvm_unreachable("Unknown condition");
3926 }
3927
3928 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3929 } else if (Name.starts_with("xop.vpcmov")) {
3930 Value *Sel = CI->getArgOperand(2);
3931 Value *NotSel = Builder.CreateNot(Sel);
3932 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3933 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3934 Rep = Builder.CreateOr(Sel0, Sel1);
3935 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3936 Name.starts_with("avx512.mask.prol")) {
3937 Rep = upgradeX86Rotate(Builder, *CI, false);
3938 } else if (Name.starts_with("avx512.pror") ||
3939 Name.starts_with("avx512.mask.pror")) {
3940 Rep = upgradeX86Rotate(Builder, *CI, true);
3941 } else if (Name.starts_with("avx512.vpshld.") ||
3942 Name.starts_with("avx512.mask.vpshld") ||
3943 Name.starts_with("avx512.maskz.vpshld")) {
3944 bool ZeroMask = Name[11] == 'z';
3945 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3946 } else if (Name.starts_with("avx512.vpshrd.") ||
3947 Name.starts_with("avx512.mask.vpshrd") ||
3948 Name.starts_with("avx512.maskz.vpshrd")) {
3949 bool ZeroMask = Name[11] == 'z';
3950 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3951 } else if (Name == "sse42.crc32.64.8") {
3952 Value *Trunc0 =
3953 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3954 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3955 {Trunc0, CI->getArgOperand(1)});
3956 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3957 } else if (Name.starts_with("avx.vbroadcast.s") ||
3958 Name.starts_with("avx512.vbroadcast.s")) {
3959 // Replace broadcasts with a series of insertelements.
3960 auto *VecTy = cast<FixedVectorType>(CI->getType());
3961 Type *EltTy = VecTy->getElementType();
3962 unsigned EltNum = VecTy->getNumElements();
3963 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3964 Type *I32Ty = Type::getInt32Ty(C);
3965 Rep = PoisonValue::get(VecTy);
3966 for (unsigned I = 0; I < EltNum; ++I)
3967 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3968 } else if (Name.starts_with("sse41.pmovsx") ||
3969 Name.starts_with("sse41.pmovzx") ||
3970 Name.starts_with("avx2.pmovsx") ||
3971 Name.starts_with("avx2.pmovzx") ||
3972 Name.starts_with("avx512.mask.pmovsx") ||
3973 Name.starts_with("avx512.mask.pmovzx")) {
3974 auto *DstTy = cast<FixedVectorType>(CI->getType());
3975 unsigned NumDstElts = DstTy->getNumElements();
3976
3977 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3978 SmallVector<int, 8> ShuffleMask(NumDstElts);
3979 for (unsigned i = 0; i != NumDstElts; ++i)
3980 ShuffleMask[i] = i;
3981
3982 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3983
3984 bool DoSext = Name.contains("pmovsx");
3985 Rep =
3986 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3987 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3988 if (CI->arg_size() == 3)
3989 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3990 CI->getArgOperand(1));
3991 } else if (Name == "avx512.mask.pmov.qd.256" ||
3992 Name == "avx512.mask.pmov.qd.512" ||
3993 Name == "avx512.mask.pmov.wb.256" ||
3994 Name == "avx512.mask.pmov.wb.512") {
3995 Type *Ty = CI->getArgOperand(1)->getType();
3996 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3997 Rep =
3998 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3999 } else if (Name.starts_with("avx.vbroadcastf128") ||
4000 Name == "avx2.vbroadcasti128") {
4001 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
4002 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
4003 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
4004 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
4005 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
4006 if (NumSrcElts == 2)
4007 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
4008 else
4009 Rep = Builder.CreateShuffleVector(Load,
4010 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
4011 } else if (Name.starts_with("avx512.mask.shuf.i") ||
4012 Name.starts_with("avx512.mask.shuf.f")) {
4013 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4014 Type *VT = CI->getType();
4015 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
4016 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
4017 unsigned ControlBitsMask = NumLanes - 1;
4018 unsigned NumControlBits = NumLanes / 2;
4019 SmallVector<int, 8> ShuffleMask(0);
4020
4021 for (unsigned l = 0; l != NumLanes; ++l) {
4022 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
4023 // We actually need the other source.
4024 if (l >= NumLanes / 2)
4025 LaneMask += NumLanes;
4026 for (unsigned i = 0; i != NumElementsInLane; ++i)
4027 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
4028 }
4029 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4030 CI->getArgOperand(1), ShuffleMask);
4031 Rep =
4032 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4033 } else if (Name.starts_with("avx512.mask.broadcastf") ||
4034 Name.starts_with("avx512.mask.broadcasti")) {
4035 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
4036 ->getNumElements();
4037 unsigned NumDstElts =
4038 cast<FixedVectorType>(CI->getType())->getNumElements();
4039
4040 SmallVector<int, 8> ShuffleMask(NumDstElts);
4041 for (unsigned i = 0; i != NumDstElts; ++i)
4042 ShuffleMask[i] = i % NumSrcElts;
4043
4044 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
4045 CI->getArgOperand(0), ShuffleMask);
4046 Rep =
4047 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4048 } else if (Name.starts_with("avx2.pbroadcast") ||
4049 Name.starts_with("avx2.vbroadcast") ||
4050 Name.starts_with("avx512.pbroadcast") ||
4051 Name.starts_with("avx512.mask.broadcast.s")) {
4052 // Replace vp?broadcasts with a vector shuffle.
4053 Value *Op = CI->getArgOperand(0);
4054 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
4055 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
4056 SmallVector<int, 8> M;
4057 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
4058 Rep = Builder.CreateShuffleVector(Op, M);
4059
4060 if (CI->arg_size() == 3)
4061 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
4062 CI->getArgOperand(1));
4063 } else if (Name.starts_with("sse2.padds.") ||
4064 Name.starts_with("avx2.padds.") ||
4065 Name.starts_with("avx512.padds.") ||
4066 Name.starts_with("avx512.mask.padds.")) {
4067 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
4068 } else if (Name.starts_with("sse2.psubs.") ||
4069 Name.starts_with("avx2.psubs.") ||
4070 Name.starts_with("avx512.psubs.") ||
4071 Name.starts_with("avx512.mask.psubs.")) {
4072 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
4073 } else if (Name.starts_with("sse2.paddus.") ||
4074 Name.starts_with("avx2.paddus.") ||
4075 Name.starts_with("avx512.mask.paddus.")) {
4076 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
4077 } else if (Name.starts_with("sse2.psubus.") ||
4078 Name.starts_with("avx2.psubus.") ||
4079 Name.starts_with("avx512.mask.psubus.")) {
4080 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
4081 } else if (Name.starts_with("avx512.mask.palignr.")) {
4082 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
4083 CI->getArgOperand(1), CI->getArgOperand(2),
4084 CI->getArgOperand(3), CI->getArgOperand(4),
4085 false);
4086 } else if (Name.starts_with("avx512.mask.valign.")) {
4087 Rep = upgradeX86ALIGNIntrinsics(
4088 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
4089 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
4090 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
4091 // 128/256-bit shift left specified in bits.
4092 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4093 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
4094 Shift / 8); // Shift is in bits.
4095 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
4096 // 128/256-bit shift right specified in bits.
4097 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4098 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
4099 Shift / 8); // Shift is in bits.
4100 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
4101 Name == "avx512.psll.dq.512") {
4102 // 128/256/512-bit shift left specified in bytes.
4103 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4104 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4105 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
4106 Name == "avx512.psrl.dq.512") {
4107 // 128/256/512-bit shift right specified in bytes.
4108 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4109 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
4110 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
4111 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
4112 Name.starts_with("avx2.pblendd.")) {
4113 Value *Op0 = CI->getArgOperand(0);
4114 Value *Op1 = CI->getArgOperand(1);
4115 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4116 auto *VecTy = cast<FixedVectorType>(CI->getType());
4117 unsigned NumElts = VecTy->getNumElements();
4118
4119 SmallVector<int, 16> Idxs(NumElts);
4120 for (unsigned i = 0; i != NumElts; ++i)
4121 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
4122
4123 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4124 } else if (Name.starts_with("avx.vinsertf128.") ||
4125 Name == "avx2.vinserti128" ||
4126 Name.starts_with("avx512.mask.insert")) {
4127 Value *Op0 = CI->getArgOperand(0);
4128 Value *Op1 = CI->getArgOperand(1);
4129 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4130 unsigned DstNumElts =
4131 cast<FixedVectorType>(CI->getType())->getNumElements();
4132 unsigned SrcNumElts =
4133 cast<FixedVectorType>(Op1->getType())->getNumElements();
4134 unsigned Scale = DstNumElts / SrcNumElts;
4135
4136 // Mask off the high bits of the immediate value; hardware ignores those.
4137 Imm = Imm % Scale;
4138
4139 // Extend the second operand into a vector the size of the destination.
4140 SmallVector<int, 8> Idxs(DstNumElts);
4141 for (unsigned i = 0; i != SrcNumElts; ++i)
4142 Idxs[i] = i;
4143 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
4144 Idxs[i] = SrcNumElts;
4145 Rep = Builder.CreateShuffleVector(Op1, Idxs);
4146
4147 // Insert the second operand into the first operand.
4148
4149 // Note that there is no guarantee that instruction lowering will actually
4150 // produce a vinsertf128 instruction for the created shuffles. In
4151 // particular, the 0 immediate case involves no lane changes, so it can
4152 // be handled as a blend.
4153
4154 // Example of shuffle mask for 32-bit elements:
4155 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
4156 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
4157
4158 // First fill with identify mask.
4159 for (unsigned i = 0; i != DstNumElts; ++i)
4160 Idxs[i] = i;
4161 // Then replace the elements where we need to insert.
4162 for (unsigned i = 0; i != SrcNumElts; ++i)
4163 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
4164 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
4165
4166 // If the intrinsic has a mask operand, handle that.
4167 if (CI->arg_size() == 5)
4168 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
4169 CI->getArgOperand(3));
4170 } else if (Name.starts_with("avx.vextractf128.") ||
4171 Name == "avx2.vextracti128" ||
4172 Name.starts_with("avx512.mask.vextract")) {
4173 Value *Op0 = CI->getArgOperand(0);
4174 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4175 unsigned DstNumElts =
4176 cast<FixedVectorType>(CI->getType())->getNumElements();
4177 unsigned SrcNumElts =
4178 cast<FixedVectorType>(Op0->getType())->getNumElements();
4179 unsigned Scale = SrcNumElts / DstNumElts;
4180
4181 // Mask off the high bits of the immediate value; hardware ignores those.
4182 Imm = Imm % Scale;
4183
4184 // Get indexes for the subvector of the input vector.
4185 SmallVector<int, 8> Idxs(DstNumElts);
4186 for (unsigned i = 0; i != DstNumElts; ++i) {
4187 Idxs[i] = i + (Imm * DstNumElts);
4188 }
4189 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4190
4191 // If the intrinsic has a mask operand, handle that.
4192 if (CI->arg_size() == 4)
4193 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4194 CI->getArgOperand(2));
4195 } else if (Name.starts_with("avx512.mask.perm.df.") ||
4196 Name.starts_with("avx512.mask.perm.di.")) {
4197 Value *Op0 = CI->getArgOperand(0);
4198 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4199 auto *VecTy = cast<FixedVectorType>(CI->getType());
4200 unsigned NumElts = VecTy->getNumElements();
4201
4202 SmallVector<int, 8> Idxs(NumElts);
4203 for (unsigned i = 0; i != NumElts; ++i)
4204 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4205
4206 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4207
4208 if (CI->arg_size() == 4)
4209 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4210 CI->getArgOperand(2));
4211 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4212 // The immediate permute control byte looks like this:
4213 // [1:0] - select 128 bits from sources for low half of destination
4214 // [2] - ignore
4215 // [3] - zero low half of destination
4216 // [5:4] - select 128 bits from sources for high half of destination
4217 // [6] - ignore
4218 // [7] - zero high half of destination
4219
4220 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4221
4222 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4223 unsigned HalfSize = NumElts / 2;
4224 SmallVector<int, 8> ShuffleMask(NumElts);
4225
4226 // Determine which operand(s) are actually in use for this instruction.
4227 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4228 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4229
4230 // If needed, replace operands based on zero mask.
4231 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4232 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4233
4234 // Permute low half of result.
4235 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4236 for (unsigned i = 0; i < HalfSize; ++i)
4237 ShuffleMask[i] = StartIndex + i;
4238
4239 // Permute high half of result.
4240 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4241 for (unsigned i = 0; i < HalfSize; ++i)
4242 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4243
4244 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4245
4246 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4247 Name.starts_with("avx512.mask.vpermil.p") ||
4248 Name.starts_with("avx512.mask.pshuf.d.")) {
4249 Value *Op0 = CI->getArgOperand(0);
4250 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4251 auto *VecTy = cast<FixedVectorType>(CI->getType());
4252 unsigned NumElts = VecTy->getNumElements();
4253 // Calculate the size of each index in the immediate.
4254 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4255 unsigned IdxMask = ((1 << IdxSize) - 1);
4256
4257 SmallVector<int, 8> Idxs(NumElts);
4258 // Lookup the bits for this element, wrapping around the immediate every
4259 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4260 // to offset by the first index of each group.
4261 for (unsigned i = 0; i != NumElts; ++i)
4262 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4263
4264 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4265
4266 if (CI->arg_size() == 4)
4267 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4268 CI->getArgOperand(2));
4269 } else if (Name == "sse2.pshufl.w" ||
4270 Name.starts_with("avx512.mask.pshufl.w.")) {
4271 Value *Op0 = CI->getArgOperand(0);
4272 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4273 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4274
4275 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4276 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4277
4278 SmallVector<int, 16> Idxs(NumElts);
4279 for (unsigned l = 0; l != NumElts; l += 8) {
4280 for (unsigned i = 0; i != 4; ++i)
4281 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4282 for (unsigned i = 4; i != 8; ++i)
4283 Idxs[i + l] = i + l;
4284 }
4285
4286 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4287
4288 if (CI->arg_size() == 4)
4289 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4290 CI->getArgOperand(2));
4291 } else if (Name == "sse2.pshufh.w" ||
4292 Name.starts_with("avx512.mask.pshufh.w.")) {
4293 Value *Op0 = CI->getArgOperand(0);
4294 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4295 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4296
4297 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4298 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4299
4300 SmallVector<int, 16> Idxs(NumElts);
4301 for (unsigned l = 0; l != NumElts; l += 8) {
4302 for (unsigned i = 0; i != 4; ++i)
4303 Idxs[i + l] = i + l;
4304 for (unsigned i = 0; i != 4; ++i)
4305 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4306 }
4307
4308 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4309
4310 if (CI->arg_size() == 4)
4311 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4312 CI->getArgOperand(2));
4313 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4314 Value *Op0 = CI->getArgOperand(0);
4315 Value *Op1 = CI->getArgOperand(1);
4316 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4317 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4318
4319 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4320 unsigned HalfLaneElts = NumLaneElts / 2;
4321
4322 SmallVector<int, 16> Idxs(NumElts);
4323 for (unsigned i = 0; i != NumElts; ++i) {
4324 // Base index is the starting element of the lane.
4325 Idxs[i] = i - (i % NumLaneElts);
4326 // If we are half way through the lane switch to the other source.
4327 if ((i % NumLaneElts) >= HalfLaneElts)
4328 Idxs[i] += NumElts;
4329 // Now select the specific element. By adding HalfLaneElts bits from
4330 // the immediate. Wrapping around the immediate every 8-bits.
4331 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4332 }
4333
4334 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4335
4336 Rep =
4337 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4338 } else if (Name.starts_with("avx512.mask.movddup") ||
4339 Name.starts_with("avx512.mask.movshdup") ||
4340 Name.starts_with("avx512.mask.movsldup")) {
4341 Value *Op0 = CI->getArgOperand(0);
4342 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4343 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4344
4345 unsigned Offset = 0;
4346 if (Name.starts_with("avx512.mask.movshdup."))
4347 Offset = 1;
4348
4349 SmallVector<int, 16> Idxs(NumElts);
4350 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4351 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4352 Idxs[i + l + 0] = i + l + Offset;
4353 Idxs[i + l + 1] = i + l + Offset;
4354 }
4355
4356 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4357
4358 Rep =
4359 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4360 } else if (Name.starts_with("avx512.mask.punpckl") ||
4361 Name.starts_with("avx512.mask.unpckl.")) {
4362 Value *Op0 = CI->getArgOperand(0);
4363 Value *Op1 = CI->getArgOperand(1);
4364 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4365 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4366
4367 SmallVector<int, 64> Idxs(NumElts);
4368 for (int l = 0; l != NumElts; l += NumLaneElts)
4369 for (int i = 0; i != NumLaneElts; ++i)
4370 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4371
4372 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4373
4374 Rep =
4375 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4376 } else if (Name.starts_with("avx512.mask.punpckh") ||
4377 Name.starts_with("avx512.mask.unpckh.")) {
4378 Value *Op0 = CI->getArgOperand(0);
4379 Value *Op1 = CI->getArgOperand(1);
4380 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4381 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4382
4383 SmallVector<int, 64> Idxs(NumElts);
4384 for (int l = 0; l != NumElts; l += NumLaneElts)
4385 for (int i = 0; i != NumLaneElts; ++i)
4386 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4387
4388 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4389
4390 Rep =
4391 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4392 } else if (Name.starts_with("avx512.mask.and.") ||
4393 Name.starts_with("avx512.mask.pand.")) {
4394 VectorType *FTy = cast<VectorType>(CI->getType());
4395 VectorType *ITy = VectorType::getInteger(FTy);
4396 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4397 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4398 Rep = Builder.CreateBitCast(Rep, FTy);
4399 Rep =
4400 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4401 } else if (Name.starts_with("avx512.mask.andn.") ||
4402 Name.starts_with("avx512.mask.pandn.")) {
4403 VectorType *FTy = cast<VectorType>(CI->getType());
4404 VectorType *ITy = VectorType::getInteger(FTy);
4405 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4406 Rep = Builder.CreateAnd(Rep,
4407 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4408 Rep = Builder.CreateBitCast(Rep, FTy);
4409 Rep =
4410 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4411 } else if (Name.starts_with("avx512.mask.or.") ||
4412 Name.starts_with("avx512.mask.por.")) {
4413 VectorType *FTy = cast<VectorType>(CI->getType());
4414 VectorType *ITy = VectorType::getInteger(FTy);
4415 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4416 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4417 Rep = Builder.CreateBitCast(Rep, FTy);
4418 Rep =
4419 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4420 } else if (Name.starts_with("avx512.mask.xor.") ||
4421 Name.starts_with("avx512.mask.pxor.")) {
4422 VectorType *FTy = cast<VectorType>(CI->getType());
4423 VectorType *ITy = VectorType::getInteger(FTy);
4424 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4425 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4426 Rep = Builder.CreateBitCast(Rep, FTy);
4427 Rep =
4428 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4429 } else if (Name.starts_with("avx512.mask.padd.")) {
4430 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4431 Rep =
4432 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4433 } else if (Name.starts_with("avx512.mask.psub.")) {
4434 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4435 Rep =
4436 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4437 } else if (Name.starts_with("avx512.mask.pmull.")) {
4438 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4439 Rep =
4440 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4441 } else if (Name.starts_with("avx512.mask.add.p")) {
4442 if (Name.ends_with(".512")) {
4443 Intrinsic::ID IID;
4444 if (Name[17] == 's')
4445 IID = Intrinsic::x86_avx512_add_ps_512;
4446 else
4447 IID = Intrinsic::x86_avx512_add_pd_512;
4448
4449 Rep = Builder.CreateIntrinsic(
4450 IID,
4451 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4452 } else {
4453 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4454 }
4455 Rep =
4456 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4457 } else if (Name.starts_with("avx512.mask.div.p")) {
4458 if (Name.ends_with(".512")) {
4459 Intrinsic::ID IID;
4460 if (Name[17] == 's')
4461 IID = Intrinsic::x86_avx512_div_ps_512;
4462 else
4463 IID = Intrinsic::x86_avx512_div_pd_512;
4464
4465 Rep = Builder.CreateIntrinsic(
4466 IID,
4467 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4468 } else {
4469 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4470 }
4471 Rep =
4472 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4473 } else if (Name.starts_with("avx512.mask.mul.p")) {
4474 if (Name.ends_with(".512")) {
4475 Intrinsic::ID IID;
4476 if (Name[17] == 's')
4477 IID = Intrinsic::x86_avx512_mul_ps_512;
4478 else
4479 IID = Intrinsic::x86_avx512_mul_pd_512;
4480
4481 Rep = Builder.CreateIntrinsic(
4482 IID,
4483 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4484 } else {
4485 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4486 }
4487 Rep =
4488 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4489 } else if (Name.starts_with("avx512.mask.sub.p")) {
4490 if (Name.ends_with(".512")) {
4491 Intrinsic::ID IID;
4492 if (Name[17] == 's')
4493 IID = Intrinsic::x86_avx512_sub_ps_512;
4494 else
4495 IID = Intrinsic::x86_avx512_sub_pd_512;
4496
4497 Rep = Builder.CreateIntrinsic(
4498 IID,
4499 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4500 } else {
4501 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4502 }
4503 Rep =
4504 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4505 } else if ((Name.starts_with("avx512.mask.max.p") ||
4506 Name.starts_with("avx512.mask.min.p")) &&
4507 Name.drop_front(18) == ".512") {
4508 bool IsDouble = Name[17] == 'd';
4509 bool IsMin = Name[13] == 'i';
4510 static const Intrinsic::ID MinMaxTbl[2][2] = {
4511 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4512 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4513 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4514
4515 Rep = Builder.CreateIntrinsic(
4516 IID,
4517 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4518 Rep =
4519 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4520 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4521 Rep =
4522 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4523 {CI->getArgOperand(0), Builder.getInt1(false)});
4524 Rep =
4525 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4526 } else if (Name.starts_with("avx512.mask.psll")) {
4527 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4528 bool IsVariable = Name[16] == 'v';
4529 char Size = Name[16] == '.' ? Name[17]
4530 : Name[17] == '.' ? Name[18]
4531 : Name[18] == '.' ? Name[19]
4532 : Name[20];
4533
4534 Intrinsic::ID IID;
4535 if (IsVariable && Name[17] != '.') {
4536 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4537 IID = Intrinsic::x86_avx2_psllv_q;
4538 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4539 IID = Intrinsic::x86_avx2_psllv_q_256;
4540 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4541 IID = Intrinsic::x86_avx2_psllv_d;
4542 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4543 IID = Intrinsic::x86_avx2_psllv_d_256;
4544 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4545 IID = Intrinsic::x86_avx512_psllv_w_128;
4546 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4547 IID = Intrinsic::x86_avx512_psllv_w_256;
4548 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4549 IID = Intrinsic::x86_avx512_psllv_w_512;
4550 else
4551 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4552 } else if (Name.ends_with(".128")) {
4553 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4554 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4555 : Intrinsic::x86_sse2_psll_d;
4556 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4557 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4558 : Intrinsic::x86_sse2_psll_q;
4559 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4560 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4561 : Intrinsic::x86_sse2_psll_w;
4562 else
4563 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4564 } else if (Name.ends_with(".256")) {
4565 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4566 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4567 : Intrinsic::x86_avx2_psll_d;
4568 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4569 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4570 : Intrinsic::x86_avx2_psll_q;
4571 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4572 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4573 : Intrinsic::x86_avx2_psll_w;
4574 else
4575 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4576 } else {
4577 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4578 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4579 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4580 : Intrinsic::x86_avx512_psll_d_512;
4581 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4582 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4583 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4584 : Intrinsic::x86_avx512_psll_q_512;
4585 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4586 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4587 : Intrinsic::x86_avx512_psll_w_512;
4588 else
4589 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4590 }
4591
4592 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4593 } else if (Name.starts_with("avx512.mask.psrl")) {
4594 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4595 bool IsVariable = Name[16] == 'v';
4596 char Size = Name[16] == '.' ? Name[17]
4597 : Name[17] == '.' ? Name[18]
4598 : Name[18] == '.' ? Name[19]
4599 : Name[20];
4600
4601 Intrinsic::ID IID;
4602 if (IsVariable && Name[17] != '.') {
4603 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4604 IID = Intrinsic::x86_avx2_psrlv_q;
4605 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4606 IID = Intrinsic::x86_avx2_psrlv_q_256;
4607 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4608 IID = Intrinsic::x86_avx2_psrlv_d;
4609 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4610 IID = Intrinsic::x86_avx2_psrlv_d_256;
4611 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4612 IID = Intrinsic::x86_avx512_psrlv_w_128;
4613 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4614 IID = Intrinsic::x86_avx512_psrlv_w_256;
4615 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4616 IID = Intrinsic::x86_avx512_psrlv_w_512;
4617 else
4618 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4619 } else if (Name.ends_with(".128")) {
4620 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4621 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4622 : Intrinsic::x86_sse2_psrl_d;
4623 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4624 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4625 : Intrinsic::x86_sse2_psrl_q;
4626 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4627 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4628 : Intrinsic::x86_sse2_psrl_w;
4629 else
4630 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4631 } else if (Name.ends_with(".256")) {
4632 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4633 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4634 : Intrinsic::x86_avx2_psrl_d;
4635 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4636 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4637 : Intrinsic::x86_avx2_psrl_q;
4638 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4639 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4640 : Intrinsic::x86_avx2_psrl_w;
4641 else
4642 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4643 } else {
4644 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4645 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4646 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4647 : Intrinsic::x86_avx512_psrl_d_512;
4648 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4649 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4650 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4651 : Intrinsic::x86_avx512_psrl_q_512;
4652 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4653 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4654 : Intrinsic::x86_avx512_psrl_w_512;
4655 else
4656 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4657 }
4658
4659 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4660 } else if (Name.starts_with("avx512.mask.psra")) {
4661 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4662 bool IsVariable = Name[16] == 'v';
4663 char Size = Name[16] == '.' ? Name[17]
4664 : Name[17] == '.' ? Name[18]
4665 : Name[18] == '.' ? Name[19]
4666 : Name[20];
4667
4668 Intrinsic::ID IID;
4669 if (IsVariable && Name[17] != '.') {
4670 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4671 IID = Intrinsic::x86_avx2_psrav_d;
4672 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4673 IID = Intrinsic::x86_avx2_psrav_d_256;
4674 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4675 IID = Intrinsic::x86_avx512_psrav_w_128;
4676 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4677 IID = Intrinsic::x86_avx512_psrav_w_256;
4678 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4679 IID = Intrinsic::x86_avx512_psrav_w_512;
4680 else
4681 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4682 } else if (Name.ends_with(".128")) {
4683 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4684 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4685 : Intrinsic::x86_sse2_psra_d;
4686 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4687 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4688 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4689 : Intrinsic::x86_avx512_psra_q_128;
4690 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4691 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4692 : Intrinsic::x86_sse2_psra_w;
4693 else
4694 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4695 } else if (Name.ends_with(".256")) {
4696 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4697 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4698 : Intrinsic::x86_avx2_psra_d;
4699 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4700 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4701 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4702 : Intrinsic::x86_avx512_psra_q_256;
4703 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4704 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4705 : Intrinsic::x86_avx2_psra_w;
4706 else
4707 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4708 } else {
4709 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4710 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4711 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4712 : Intrinsic::x86_avx512_psra_d_512;
4713 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4714 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4715 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4716 : Intrinsic::x86_avx512_psra_q_512;
4717 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4718 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4719 : Intrinsic::x86_avx512_psra_w_512;
4720 else
4721 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4722 }
4723
4724 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4725 } else if (Name.starts_with("avx512.mask.move.s")) {
4726 Rep = upgradeMaskedMove(Builder, *CI);
4727 } else if (Name.starts_with("avx512.cvtmask2")) {
4728 Rep = upgradeMaskToInt(Builder, *CI);
4729 } else if (Name.ends_with(".movntdqa")) {
4730 MDNode *Node = MDNode::get(
4731 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4732
4733 LoadInst *LI = Builder.CreateAlignedLoad(
4734 CI->getType(), CI->getArgOperand(0),
4735 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4736 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4737 Rep = LI;
4738 } else if (Name.starts_with("fma.vfmadd.") ||
4739 Name.starts_with("fma.vfmsub.") ||
4740 Name.starts_with("fma.vfnmadd.") ||
4741 Name.starts_with("fma.vfnmsub.")) {
4742 bool NegMul = Name[6] == 'n';
4743 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4744 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4745
4746 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4747 CI->getArgOperand(2)};
4748
4749 if (IsScalar) {
4750 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4751 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4752 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4753 }
4754
4755 if (NegMul && !IsScalar)
4756 Ops[0] = Builder.CreateFNeg(Ops[0]);
4757 if (NegMul && IsScalar)
4758 Ops[1] = Builder.CreateFNeg(Ops[1]);
4759 if (NegAcc)
4760 Ops[2] = Builder.CreateFNeg(Ops[2]);
4761
4762 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4763
4764 if (IsScalar)
4765 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4766 } else if (Name.starts_with("fma4.vfmadd.s")) {
4767 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4768 CI->getArgOperand(2)};
4769
4770 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4771 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4772 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4773
4774 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4775
4776 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4777 Rep, (uint64_t)0);
4778 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4779 Name.starts_with("avx512.maskz.vfmadd.s") ||
4780 Name.starts_with("avx512.mask3.vfmadd.s") ||
4781 Name.starts_with("avx512.mask3.vfmsub.s") ||
4782 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4783 bool IsMask3 = Name[11] == '3';
4784 bool IsMaskZ = Name[11] == 'z';
4785 // Drop the "avx512.mask." to make it easier.
4786 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4787 bool NegMul = Name[2] == 'n';
4788 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4789
4790 Value *A = CI->getArgOperand(0);
4791 Value *B = CI->getArgOperand(1);
4792 Value *C = CI->getArgOperand(2);
4793
4794 if (NegMul && (IsMask3 || IsMaskZ))
4795 A = Builder.CreateFNeg(A);
4796 if (NegMul && !(IsMask3 || IsMaskZ))
4797 B = Builder.CreateFNeg(B);
4798 if (NegAcc)
4799 C = Builder.CreateFNeg(C);
4800
4801 A = Builder.CreateExtractElement(A, (uint64_t)0);
4802 B = Builder.CreateExtractElement(B, (uint64_t)0);
4803 C = Builder.CreateExtractElement(C, (uint64_t)0);
4804
4805 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4806 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4807 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4808
4809 Intrinsic::ID IID;
4810 if (Name.back() == 'd')
4811 IID = Intrinsic::x86_avx512_vfmadd_f64;
4812 else
4813 IID = Intrinsic::x86_avx512_vfmadd_f32;
4814 Rep = Builder.CreateIntrinsic(IID, Ops);
4815 } else {
4816 Rep = Builder.CreateFMA(A, B, C);
4817 }
4818
4819 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4820 : IsMask3 ? C
4821 : A;
4822
4823 // For Mask3 with NegAcc, we need to create a new extractelement that
4824 // avoids the negation above.
4825 if (NegAcc && IsMask3)
4826 PassThru =
4827 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4828
4829 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4830 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4831 (uint64_t)0);
4832 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4833 Name.starts_with("avx512.mask.vfnmadd.p") ||
4834 Name.starts_with("avx512.mask.vfnmsub.p") ||
4835 Name.starts_with("avx512.mask3.vfmadd.p") ||
4836 Name.starts_with("avx512.mask3.vfmsub.p") ||
4837 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4838 Name.starts_with("avx512.maskz.vfmadd.p")) {
4839 bool IsMask3 = Name[11] == '3';
4840 bool IsMaskZ = Name[11] == 'z';
4841 // Drop the "avx512.mask." to make it easier.
4842 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4843 bool NegMul = Name[2] == 'n';
4844 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4845
4846 Value *A = CI->getArgOperand(0);
4847 Value *B = CI->getArgOperand(1);
4848 Value *C = CI->getArgOperand(2);
4849
4850 if (NegMul && (IsMask3 || IsMaskZ))
4851 A = Builder.CreateFNeg(A);
4852 if (NegMul && !(IsMask3 || IsMaskZ))
4853 B = Builder.CreateFNeg(B);
4854 if (NegAcc)
4855 C = Builder.CreateFNeg(C);
4856
4857 if (CI->arg_size() == 5 &&
4858 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4859 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4860 Intrinsic::ID IID;
4861 // Check the character before ".512" in string.
4862 if (Name[Name.size() - 5] == 's')
4863 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4864 else
4865 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4866
4867 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4868 } else {
4869 Rep = Builder.CreateFMA(A, B, C);
4870 }
4871
4872 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4873 : IsMask3 ? CI->getArgOperand(2)
4874 : CI->getArgOperand(0);
4875
4876 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4877 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4878 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4879 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4880 Intrinsic::ID IID;
4881 if (VecWidth == 128 && EltWidth == 32)
4882 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4883 else if (VecWidth == 256 && EltWidth == 32)
4884 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4885 else if (VecWidth == 128 && EltWidth == 64)
4886 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4887 else if (VecWidth == 256 && EltWidth == 64)
4888 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4889 else
4890 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4891
4892 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4893 CI->getArgOperand(2)};
4894 Ops[2] = Builder.CreateFNeg(Ops[2]);
4895 Rep = Builder.CreateIntrinsic(IID, Ops);
4896 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4897 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4898 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4899 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4900 bool IsMask3 = Name[11] == '3';
4901 bool IsMaskZ = Name[11] == 'z';
4902 // Drop the "avx512.mask." to make it easier.
4903 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4904 bool IsSubAdd = Name[3] == 's';
4905 if (CI->arg_size() == 5) {
4906 Intrinsic::ID IID;
4907 // Check the character before ".512" in string.
4908 if (Name[Name.size() - 5] == 's')
4909 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4910 else
4911 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4912
4913 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4914 CI->getArgOperand(2), CI->getArgOperand(4)};
4915 if (IsSubAdd)
4916 Ops[2] = Builder.CreateFNeg(Ops[2]);
4917
4918 Rep = Builder.CreateIntrinsic(IID, Ops);
4919 } else {
4920 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4921
4922 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4923 CI->getArgOperand(2)};
4924
4925 Function *FMA = Intrinsic::getOrInsertDeclaration(
4926 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4927 Value *Odd = Builder.CreateCall(FMA, Ops);
4928 Ops[2] = Builder.CreateFNeg(Ops[2]);
4929 Value *Even = Builder.CreateCall(FMA, Ops);
4930
4931 if (IsSubAdd)
4932 std::swap(Even, Odd);
4933
4934 SmallVector<int, 32> Idxs(NumElts);
4935 for (int i = 0; i != NumElts; ++i)
4936 Idxs[i] = i + (i % 2) * NumElts;
4937
4938 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4939 }
4940
4941 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4942 : IsMask3 ? CI->getArgOperand(2)
4943 : CI->getArgOperand(0);
4944
4945 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4946 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4947 Name.starts_with("avx512.maskz.pternlog.")) {
4948 bool ZeroMask = Name[11] == 'z';
4949 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4950 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4951 Intrinsic::ID IID;
4952 if (VecWidth == 128 && EltWidth == 32)
4953 IID = Intrinsic::x86_avx512_pternlog_d_128;
4954 else if (VecWidth == 256 && EltWidth == 32)
4955 IID = Intrinsic::x86_avx512_pternlog_d_256;
4956 else if (VecWidth == 512 && EltWidth == 32)
4957 IID = Intrinsic::x86_avx512_pternlog_d_512;
4958 else if (VecWidth == 128 && EltWidth == 64)
4959 IID = Intrinsic::x86_avx512_pternlog_q_128;
4960 else if (VecWidth == 256 && EltWidth == 64)
4961 IID = Intrinsic::x86_avx512_pternlog_q_256;
4962 else if (VecWidth == 512 && EltWidth == 64)
4963 IID = Intrinsic::x86_avx512_pternlog_q_512;
4964 else
4965 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4966
4967 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4968 CI->getArgOperand(2), CI->getArgOperand(3)};
4969 Rep = Builder.CreateIntrinsic(IID, Args);
4970 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4971 : CI->getArgOperand(0);
4972 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4973 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4974 Name.starts_with("avx512.maskz.vpmadd52")) {
4975 bool ZeroMask = Name[11] == 'z';
4976 bool High = Name[20] == 'h' || Name[21] == 'h';
4977 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4978 Intrinsic::ID IID;
4979 if (VecWidth == 128 && !High)
4980 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4981 else if (VecWidth == 256 && !High)
4982 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4983 else if (VecWidth == 512 && !High)
4984 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4985 else if (VecWidth == 128 && High)
4986 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4987 else if (VecWidth == 256 && High)
4988 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4989 else if (VecWidth == 512 && High)
4990 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4991 else
4992 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4993
4994 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4995 CI->getArgOperand(2)};
4996 Rep = Builder.CreateIntrinsic(IID, Args);
4997 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4998 : CI->getArgOperand(0);
4999 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5000 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
5001 Name.starts_with("avx512.mask.vpermt2var.") ||
5002 Name.starts_with("avx512.maskz.vpermt2var.")) {
5003 bool ZeroMask = Name[11] == 'z';
5004 bool IndexForm = Name[17] == 'i';
5005 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
5006 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
5007 Name.starts_with("avx512.maskz.vpdpbusd.") ||
5008 Name.starts_with("avx512.mask.vpdpbusds.") ||
5009 Name.starts_with("avx512.maskz.vpdpbusds.")) {
5010 bool ZeroMask = Name[11] == 'z';
5011 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5012 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5013 Intrinsic::ID IID;
5014 if (VecWidth == 128 && !IsSaturating)
5015 IID = Intrinsic::x86_avx512_vpdpbusd_128;
5016 else if (VecWidth == 256 && !IsSaturating)
5017 IID = Intrinsic::x86_avx512_vpdpbusd_256;
5018 else if (VecWidth == 512 && !IsSaturating)
5019 IID = Intrinsic::x86_avx512_vpdpbusd_512;
5020 else if (VecWidth == 128 && IsSaturating)
5021 IID = Intrinsic::x86_avx512_vpdpbusds_128;
5022 else if (VecWidth == 256 && IsSaturating)
5023 IID = Intrinsic::x86_avx512_vpdpbusds_256;
5024 else if (VecWidth == 512 && IsSaturating)
5025 IID = Intrinsic::x86_avx512_vpdpbusds_512;
5026 else
5027 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5028
5029 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5030 CI->getArgOperand(2)};
5031
5032 // Input arguments types were incorrectly set to vectors of i32 before but
5033 // they should be vectors of i8. Insert bit cast when encountering the old
5034 // types
5035 if (Args[1]->getType()->isVectorTy() &&
5036 cast<VectorType>(Args[1]->getType())
5037 ->getElementType()
5038 ->isIntegerTy(32) &&
5039 Args[2]->getType()->isVectorTy() &&
5040 cast<VectorType>(Args[2]->getType())
5041 ->getElementType()
5042 ->isIntegerTy(32)) {
5043 Type *NewArgType = nullptr;
5044 if (VecWidth == 128)
5045 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
5046 else if (VecWidth == 256)
5047 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
5048 else if (VecWidth == 512)
5049 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
5050 else
5051 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5052 CI);
5053
5054 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5055 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5056 }
5057
5058 Rep = Builder.CreateIntrinsic(IID, Args);
5059 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5060 : CI->getArgOperand(0);
5061 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5062 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
5063 Name.starts_with("avx512.maskz.vpdpwssd.") ||
5064 Name.starts_with("avx512.mask.vpdpwssds.") ||
5065 Name.starts_with("avx512.maskz.vpdpwssds.")) {
5066 bool ZeroMask = Name[11] == 'z';
5067 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
5068 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
5069 Intrinsic::ID IID;
5070 if (VecWidth == 128 && !IsSaturating)
5071 IID = Intrinsic::x86_avx512_vpdpwssd_128;
5072 else if (VecWidth == 256 && !IsSaturating)
5073 IID = Intrinsic::x86_avx512_vpdpwssd_256;
5074 else if (VecWidth == 512 && !IsSaturating)
5075 IID = Intrinsic::x86_avx512_vpdpwssd_512;
5076 else if (VecWidth == 128 && IsSaturating)
5077 IID = Intrinsic::x86_avx512_vpdpwssds_128;
5078 else if (VecWidth == 256 && IsSaturating)
5079 IID = Intrinsic::x86_avx512_vpdpwssds_256;
5080 else if (VecWidth == 512 && IsSaturating)
5081 IID = Intrinsic::x86_avx512_vpdpwssds_512;
5082 else
5083 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5084
5085 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5086 CI->getArgOperand(2)};
5087
5088 // Input arguments types were incorrectly set to vectors of i32 before but
5089 // they should be vectors of i16. Insert bit cast when encountering the old
5090 // types
5091 if (Args[1]->getType()->isVectorTy() &&
5092 cast<VectorType>(Args[1]->getType())
5093 ->getElementType()
5094 ->isIntegerTy(32) &&
5095 Args[2]->getType()->isVectorTy() &&
5096 cast<VectorType>(Args[2]->getType())
5097 ->getElementType()
5098 ->isIntegerTy(32)) {
5099 Type *NewArgType = nullptr;
5100 if (VecWidth == 128)
5101 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
5102 else if (VecWidth == 256)
5103 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
5104 else if (VecWidth == 512)
5105 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
5106 else
5107 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
5108 CI);
5109
5110 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
5111 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
5112 }
5113
5114 Rep = Builder.CreateIntrinsic(IID, Args);
5115 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
5116 : CI->getArgOperand(0);
5117 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
5118 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
5119 Name == "addcarry.u32" || Name == "addcarry.u64" ||
5120 Name == "subborrow.u32" || Name == "subborrow.u64") {
5121 Intrinsic::ID IID;
5122 if (Name[0] == 'a' && Name.back() == '2')
5123 IID = Intrinsic::x86_addcarry_32;
5124 else if (Name[0] == 'a' && Name.back() == '4')
5125 IID = Intrinsic::x86_addcarry_64;
5126 else if (Name[0] == 's' && Name.back() == '2')
5127 IID = Intrinsic::x86_subborrow_32;
5128 else if (Name[0] == 's' && Name.back() == '4')
5129 IID = Intrinsic::x86_subborrow_64;
5130 else
5131 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5132
5133 // Make a call with 3 operands.
5134 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
5135 CI->getArgOperand(2)};
5136 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
5137
5138 // Extract the second result and store it.
5139 Value *Data = Builder.CreateExtractValue(NewCall, 1);
5140 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
5141 // Replace the original call result with the first result of the new call.
5142 Value *CF = Builder.CreateExtractValue(NewCall, 0);
5143
5144 CI->replaceAllUsesWith(CF);
5145 Rep = nullptr;
5146 } else if (Name.starts_with("avx512.mask.") &&
5147 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
5148 // Rep will be updated by the call in the condition.
5149 } else if (Name.starts_with("bmi.pdep.")) {
5150 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
5151 } else if (Name.starts_with("bmi.pext.")) {
5152 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
5153 } else
5154 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
5155
5156 return Rep;
5157}
5158
5159static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
5160 Function *F, IRBuilder<> &Builder) {
5161 if (Name.starts_with("neon.bfcvt")) {
5162 if (Name.starts_with("neon.bfcvtn2")) {
5163 SmallVector<int, 32> LoMask(4);
5164 std::iota(LoMask.begin(), LoMask.end(), 0);
5165 SmallVector<int, 32> ConcatMask(8);
5166 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5167 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
5168 Value *Trunc =
5169 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
5170 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
5171 } else if (Name.starts_with("neon.bfcvtn")) {
5172 SmallVector<int, 32> ConcatMask(8);
5173 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
5174 Type *V4BF16 =
5175 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
5176 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
5177 dbgs() << "Trunc: " << *Trunc << "\n";
5178 return Builder.CreateShuffleVector(
5179 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
5180 } else {
5181 return Builder.CreateFPTrunc(CI->getOperand(0),
5182 Type::getBFloatTy(F->getContext()));
5183 }
5184 } else if (Name.starts_with("sve.fcvt")) {
5185 Intrinsic::ID NewID =
5186 StringSwitch<Intrinsic::ID>(Name)
5187 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
5188 .Case("sve.fcvtnt.bf16f32",
5189 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
5190 .Default(Intrinsic::not_intrinsic);
5191 if (NewID == Intrinsic::not_intrinsic)
5192 llvm_unreachable("Unhandled Intrinsic!");
5193
5194 SmallVector<Value *, 3> Args(CI->args());
5195
5196 // The original intrinsics incorrectly used a predicate based on the
5197 // smallest element type rather than the largest.
5198 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
5199 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
5200
5201 if (Args[1]->getType() != BadPredTy)
5202 llvm_unreachable("Unexpected predicate type!");
5203
5204 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5205 BadPredTy, Args[1]);
5206 Args[1] = Builder.CreateIntrinsic(
5207 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5208
5209 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5210 CI->getName());
5211 }
5212
5213 if (Name == "neon.vcvtfp2hf")
5214 return Builder.CreateBitCast(
5215 Builder.CreateFPTrunc(
5216 CI->getOperand(0),
5217 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5218 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5219 if (Name == "neon.vcvthf2fp")
5220 return Builder.CreateFPExt(
5221 Builder.CreateBitCast(
5222 CI->getOperand(0),
5223 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5224 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5225
5226 llvm_unreachable("Unhandled Intrinsic!");
5227}
5228
5229static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5230 IRBuilder<> &Builder) {
5231 if (Name == "mve.vctp64.old") {
5232 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5233 // correct type.
5234 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5235 CI->getArgOperand(0),
5236 /*FMFSource=*/nullptr, CI->getName());
5237 Value *C1 = Builder.CreateIntrinsic(
5238 Intrinsic::arm_mve_pred_v2i,
5239 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5240 return Builder.CreateIntrinsic(
5241 Intrinsic::arm_mve_pred_i2v,
5242 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5243 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5244 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5245 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5246 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5247 Name ==
5248 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5249 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5250 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5251 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5252 Name ==
5253 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5254 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5255 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5256 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5257 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5258 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5259 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5260 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5261 std::vector<Type *> Tys;
5262 unsigned ID = CI->getIntrinsicID();
5263 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5264 switch (ID) {
5265 case Intrinsic::arm_mve_mull_int_predicated:
5266 case Intrinsic::arm_mve_vqdmull_predicated:
5267 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5268 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5269 break;
5270 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5271 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5272 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5273 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5274 V2I1Ty};
5275 break;
5276 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5277 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5278 CI->getOperand(1)->getType(), V2I1Ty};
5279 break;
5280 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5281 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5282 CI->getOperand(2)->getType(), V2I1Ty};
5283 break;
5284 case Intrinsic::arm_cde_vcx1q_predicated:
5285 case Intrinsic::arm_cde_vcx1qa_predicated:
5286 case Intrinsic::arm_cde_vcx2q_predicated:
5287 case Intrinsic::arm_cde_vcx2qa_predicated:
5288 case Intrinsic::arm_cde_vcx3q_predicated:
5289 case Intrinsic::arm_cde_vcx3qa_predicated:
5290 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5291 break;
5292 default:
5293 llvm_unreachable("Unhandled Intrinsic!");
5294 }
5295
5296 std::vector<Value *> Ops;
5297 for (Value *Op : CI->args()) {
5298 Type *Ty = Op->getType();
5299 if (Ty->getScalarSizeInBits() == 1) {
5300 Value *C1 = Builder.CreateIntrinsic(
5301 Intrinsic::arm_mve_pred_v2i,
5302 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5303 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5304 }
5305 Ops.push_back(Op);
5306 }
5307
5308 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5309 CI->getName());
5310 }
5311 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5312}
5313
5314// These are expected to have the arguments:
5315// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5316//
5317// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5318//
5319static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5320 Function *F, IRBuilder<> &Builder) {
5321 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5322 // for compatibility.
5323 auto UpgradeLegacyWMMAIUIntrinsicCall =
5324 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5325 ArrayRef<Type *> OverloadTys) -> Value * {
5326 // Prepare arguments, append clamp=0 for compatibility
5327 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5328 Args.push_back(Builder.getFalse());
5329
5330 // Insert the declaration for the right overload types
5331 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5332 F->getParent(), F->getIntrinsicID(), OverloadTys);
5333
5334 // Copy operand bundles if any
5335 SmallVector<OperandBundleDef, 1> Bundles;
5336 CI->getOperandBundlesAsDefs(Bundles);
5337
5338 // Create the new call and copy calling properties
5339 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5340 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5341 NewCall->setCallingConv(CI->getCallingConv());
5342 NewCall->setAttributes(CI->getAttributes());
5343 NewCall->copyMetadata(*CI);
5344 return NewCall;
5345 };
5346
5347 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5348 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5349 "intrinsic should have 7 arguments");
5350 Type *T1 = CI->getArgOperand(4)->getType();
5351 Type *T2 = CI->getArgOperand(1)->getType();
5352 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5353 }
5354 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5355 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5356 "intrinsic should have 8 arguments");
5357 Type *T1 = CI->getArgOperand(4)->getType();
5358 Type *T2 = CI->getArgOperand(1)->getType();
5359 Type *T3 = CI->getArgOperand(3)->getType();
5360 Type *T4 = CI->getArgOperand(5)->getType();
5361 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5362 }
5363
5364 switch (F->getIntrinsicID()) {
5365 default:
5366 break;
5367 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5368 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5369 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5370 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5371 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5372 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5373 // Drop src0 and src1 modifiers.
5374 const Value *Op0 = CI->getArgOperand(0);
5375 const Value *Op2 = CI->getArgOperand(2);
5376 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5377 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5378 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5379 if (!ModA->isZero() || !ModB->isZero())
5380 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5381
5382 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5383 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5384 Args.push_back(CI->getArgOperand(I));
5385
5386 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5387 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5388 Overloads.push_back(Args[3]->getType());
5389 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5390 F->getParent(), F->getIntrinsicID(), Overloads);
5391
5392 SmallVector<OperandBundleDef, 1> Bundles;
5393 CI->getOperandBundlesAsDefs(Bundles);
5394
5395 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5396 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5397 NewCall->setCallingConv(CI->getCallingConv());
5398 NewCall->setAttributes(CI->getAttributes());
5399 NewCall->copyMetadata(*CI);
5400 NewCall->takeName(CI);
5401 return NewCall;
5402 }
5403 }
5404
5405 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5406 Value *LHS = CI->getArgOperand(0);
5407 Value *RHS = CI->getArgOperand(1);
5408 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5409 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5410 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5411 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5412 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5413 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5414 NewCall->setCallingConv(CI->getCallingConv());
5415 NewCall->copyMetadata(*CI);
5416 NewCall->takeName(CI);
5417 return NewCall;
5418 }
5419
5420 if (Name.starts_with("addrspacecast.nonnull")) {
5421 if (CI->getNumOperands() < 2) // Malformed bitcode.
5422 return nullptr;
5423 Value *ASC = Builder.CreateAddrSpaceCast(
5424 CI->getArgOperand(0), CI->getType(), "", /*IsNonNull=*/true);
5425 ASC->takeName(CI);
5426 return ASC;
5427 }
5428
5429 AtomicRMWInst::BinOp RMWOp =
5430 StringSwitch<AtomicRMWInst::BinOp>(Name)
5431 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5432 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5433 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5434 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5435 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5436 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5437 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5438 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5439 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5440 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5441 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5442 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5443 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5444
5445 unsigned NumOperands = CI->getNumOperands();
5446 if (NumOperands < 3) // Malformed bitcode.
5447 return nullptr;
5448
5449 Value *Ptr = CI->getArgOperand(0);
5450 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5451 if (!PtrTy) // Malformed.
5452 return nullptr;
5453
5454 Value *Val = CI->getArgOperand(1);
5455 if (Val->getType() != CI->getType()) // Malformed.
5456 return nullptr;
5457
5458 ConstantInt *OrderArg = nullptr;
5459 bool IsVolatile = false;
5460
5461 // These should have 5 arguments (plus the callee). A separate version of the
5462 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5463 if (NumOperands > 3)
5464 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5465
5466 // Ignore scope argument at 3
5467
5468 if (NumOperands > 5) {
5469 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5470 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5471 }
5472
5473 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5474 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5475 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5476 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5477 Order = AtomicOrdering::SequentiallyConsistent;
5478
5479 LLVMContext &Ctx = F->getContext();
5480
5481 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5482 Type *RetTy = CI->getType();
5483 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5484 if (VT->getElementType()->isIntegerTy(16)) {
5485 VectorType *AsBF16 =
5486 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5487 Val = Builder.CreateBitCast(Val, AsBF16);
5488 }
5489 }
5490
5491 // The scope argument never really worked correctly. Use agent as the most
5492 // conservative option which should still always produce the instruction.
5493 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5494 AtomicRMWInst *RMW =
5495 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5496
5497 unsigned AddrSpace = PtrTy->getAddressSpace();
5498 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5499 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5500 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5501 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5502 RMW->setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, EmptyMD);
5503 }
5504
5505 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5506 MDBuilder MDB(F->getContext());
5507 MDNode *RangeNotPrivate =
5508 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5509 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5510 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5511 }
5512
5513 if (IsVolatile)
5514 RMW->setVolatile(true);
5515
5516 return Builder.CreateBitCast(RMW, RetTy);
5517}
5518
5519/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5520/// plain MDNode, as it's the verifier's job to check these are the correct
5521/// types later.
5522static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5523 if (Op < CI->arg_size()) {
5524 if (MetadataAsValue *MAV =
5525 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5526 Metadata *MD = MAV->getMetadata();
5527 return dyn_cast_if_present<MDNode>(MD);
5528 }
5529 }
5530 return nullptr;
5531}
5532
5533/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5534static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5535 if (Op < CI->arg_size())
5536 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5537 return MAV->getMetadata();
5538 return nullptr;
5539}
5540
5541/// Convert debug intrinsic calls to non-instruction debug records.
5542/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5543/// \p CI - The debug intrinsic call.
5544static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5545 DbgRecord *DR = nullptr;
5546 if (Name == "label") {
5547 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5548 } else if (Name == "assign") {
5549 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5550 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5551 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5552 unwrapMAVMetadataOp(CI, 4),
5553 /*The address is a Value ref, it will be stored as a Metadata */
5554 unwrapMAVOp(CI, 5));
5555 } else if (Name == "declare") {
5556 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5557 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5558 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5559 } else if (Name == "addr") {
5560 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5561 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5562 // Don't try to add something to the expression if it's not an expression.
5563 // Instead, allow the verifier to fail later.
5564 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5565 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5566 }
5567 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5568 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5569 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5570 } else if (Name == "value") {
5571 // An old version of dbg.value had an extra offset argument.
5572 unsigned VarOp = 1;
5573 unsigned ExprOp = 2;
5574 if (CI->arg_size() == 4) {
5575 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5576 // Nonzero offset dbg.values get dropped without a replacement.
5577 if (!Offset || !Offset->isNullValue())
5578 return;
5579 VarOp = 2;
5580 ExprOp = 3;
5581 }
5582 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5583 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5584 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5585 nullptr);
5586 }
5587 DR->setDebugLoc(CI->getDebugLoc());
5588 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5589 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5590}
5591
5592static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5593 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5594 if (!Offset)
5595 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5596 int64_t OffsetVal = Offset->getSExtValue();
5597 return Builder.CreateIntrinsic(OffsetVal >= 0
5598 ? Intrinsic::vector_splice_left
5599 : Intrinsic::vector_splice_right,
5600 CI->getType(),
5601 {CI->getArgOperand(0), CI->getArgOperand(1),
5602 Builder.getInt32(std::abs(OffsetVal))});
5603}
5604
5605static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5606 Function *F, IRBuilder<> &Builder) {
5607 if (Name.starts_with("to.fp16")) {
5608 Value *Cast =
5609 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5610 return Builder.CreateBitCast(Cast, CI->getType());
5611 }
5612
5613 if (Name.starts_with("from.fp16")) {
5614 Value *Cast =
5615 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5616 return Builder.CreateFPExt(Cast, CI->getType());
5617 }
5618
5619 return nullptr;
5620}
5621
5622static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5623 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5624 if (!MD || !isa<MDString>(MD))
5625 return ICmpInst::BAD_ICMP_PREDICATE;
5626 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5627 .Case("eq", ICmpInst::ICMP_EQ)
5628 .Case("ne", ICmpInst::ICMP_NE)
5629 .Case("ugt", ICmpInst::ICMP_UGT)
5630 .Case("uge", ICmpInst::ICMP_UGE)
5631 .Case("ult", ICmpInst::ICMP_ULT)
5632 .Case("ule", ICmpInst::ICMP_ULE)
5633 .Case("sgt", ICmpInst::ICMP_SGT)
5634 .Case("sge", ICmpInst::ICMP_SGE)
5635 .Case("slt", ICmpInst::ICMP_SLT)
5636 .Case("sle", ICmpInst::ICMP_SLE)
5637 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5638}
5639
5640static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5641 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5642 if (!MD || !isa<MDString>(MD))
5643 return FCmpInst::BAD_FCMP_PREDICATE;
5644 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5645 .Case("oeq", FCmpInst::FCMP_OEQ)
5646 .Case("ogt", FCmpInst::FCMP_OGT)
5647 .Case("oge", FCmpInst::FCMP_OGE)
5648 .Case("olt", FCmpInst::FCMP_OLT)
5649 .Case("ole", FCmpInst::FCMP_OLE)
5650 .Case("one", FCmpInst::FCMP_ONE)
5651 .Case("ord", FCmpInst::FCMP_ORD)
5652 .Case("uno", FCmpInst::FCMP_UNO)
5653 .Case("ueq", FCmpInst::FCMP_UEQ)
5654 .Case("ugt", FCmpInst::FCMP_UGT)
5655 .Case("uge", FCmpInst::FCMP_UGE)
5656 .Case("ult", FCmpInst::FCMP_ULT)
5657 .Case("ule", FCmpInst::FCMP_ULE)
5658 .Case("une", FCmpInst::FCMP_UNE)
5659 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5660}
5661
5662static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5663 IRBuilder<> &Builder) {
5664 Value *Rep;
5665 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5666 if (Opcode && Instruction::isUnaryOp(Opcode))
5667 Rep =
5668 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5669 else if (Opcode && Instruction::isBinaryOp(Opcode))
5670 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5671 CI->getArgOperand(0), CI->getArgOperand(1));
5672 else if (Opcode && Instruction::isCast(Opcode))
5673 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5674 CI->getType());
5675 else if (Opcode == Instruction::ICmp)
5676 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5677 CI->getArgOperand(0), CI->getArgOperand(1));
5678 else if (Opcode == Instruction::FCmp)
5679 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5680 CI->getArgOperand(0), CI->getArgOperand(1));
5681 else if (Opcode == Instruction::Select)
5682 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5683 CI->getArgOperand(2));
5684 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5685 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5686 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5687 } else
5688 llvm_unreachable("Unexpected vp intrinsic");
5689 Rep->takeName(CI);
5690 return Rep;
5691}
5692
5693static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5694 IRBuilder<> &Builder) {
5695 Intrinsic::ID IID = NewFn->getIntrinsicID();
5696
5697 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5698 if (Defaults.empty())
5699 return false;
5700
5701 unsigned OldArgCount = CI->arg_size();
5702 unsigned NewArgCount = NewFn->arg_size();
5703
5704 if (OldArgCount < FirstDefault)
5705 return false;
5706
5707 // More arguments than the new intrinsic accepts, cannot upgrade.
5708 if (OldArgCount > NewArgCount)
5709 return false;
5710
5711 // The call already passes the defaulted arguments explicitly; only the
5712 // callee is still the old, shorter declaration, so retarget it.
5713 if (OldArgCount == NewArgCount) {
5714 if (CI->getFunctionType() != NewFn->getFunctionType())
5715 return false;
5716 CI->setCalledFunction(NewFn);
5717 return true;
5718 }
5719
5720 // OldArgCount < NewArgCount: Fill in each missing trailing default
5721 // argument from the table.
5722 SmallVector<Value *, 8> NewArgs(CI->args());
5723
5724 FunctionType *NewFT = NewFn->getFunctionType();
5725 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5726 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5727 "missing argument outside the default range");
5728 Type *ParamTy = NewFT->getParamType(Idx);
5729
5730 // Only integer types are supported (i1, i8, i16, i32, i64).
5731 if (!ParamTy->isIntegerTy())
5732 return false;
5733 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5734 }
5735
5736 // Preserve operand bundles by creating the call with them.
5737 SmallVector<OperandBundleDef, 1> OpBundles;
5738 CI->getOperandBundlesAsDefs(OpBundles);
5739 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5740
5741 NewCall->takeName(CI);
5742 NewCall->setCallingConv(CI->getCallingConv());
5743 NewCall->copyMetadata(*CI);
5744 if (auto *OldCI = dyn_cast<CallInst>(CI))
5745 NewCall->setTailCallKind(OldCI->getTailCallKind());
5746
5747 CI->replaceAllUsesWith(NewCall);
5748 CI->eraseFromParent();
5749 return true;
5750}
5751
5752/// Upgrade a call to an old intrinsic. All argument and return casting must be
5753/// provided to seamlessly integrate with existing context.
5754void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5755 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5756 // checks the callee's function type matches. It's likely we need to handle
5757 // type changes here.
5758 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5759 if (!F)
5760 return;
5761
5762 LLVMContext &C = CI->getContext();
5763 IRBuilder<> Builder(C);
5764 if (isa<FPMathOperator>(CI))
5765 Builder.setFastMathFlags(CI->getFastMathFlags());
5766 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5767
5768 if (!NewFn) {
5769 // Get the Function's name.
5770 StringRef Name = F->getName();
5771 if (!Name.consume_front("llvm."))
5772 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5773
5774 bool IsX86 = Name.consume_front("x86.");
5775 bool IsNVVM = Name.consume_front("nvvm.");
5776 bool IsAArch64 = Name.consume_front("aarch64.");
5777 bool IsARM = Name.consume_front("arm.");
5778 bool IsAMDGCN = Name.consume_front("amdgcn.");
5779 bool IsDbg = Name.consume_front("dbg.");
5780 bool IsOldSplice =
5781 (Name.consume_front("experimental.vector.splice") ||
5782 Name.consume_front("vector.splice")) &&
5783 !(Name.starts_with(".left") || Name.starts_with(".right"));
5784 Value *Rep = nullptr;
5785
5786 if (!IsX86 && Name == "stackprotectorcheck") {
5787 Rep = nullptr;
5788 } else if (IsNVVM) {
5789 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5790 } else if (IsX86) {
5791 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5792 } else if (IsAArch64) {
5793 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5794 } else if (IsARM) {
5795 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5796 } else if (IsAMDGCN) {
5797 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5798 } else if (IsDbg) {
5799 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5800 } else if (IsOldSplice) {
5801 Rep = upgradeVectorSplice(CI, Builder);
5802 } else if (Name.consume_front("convert.")) {
5803 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5804 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5805 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5806 Rep = nullptr;
5807 } else if (shouldUpgradeVPIntrinsic(Name)) {
5808 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5809 } else {
5810 llvm_unreachable("Unknown function for CallBase upgrade.");
5811 }
5812
5813 if (Rep)
5814 CI->replaceAllUsesWith(Rep);
5815 CI->eraseFromParent();
5816 return;
5817 }
5818
5819 const auto &DefaultCase = [&]() -> void {
5820 if (F == NewFn)
5821 return;
5822
5823 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5824 // Handle generic mangling change.
5825 assert(
5826 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5827 "Unknown function for CallBase upgrade and isn't just a name change");
5828 CI->setCalledFunction(NewFn);
5829 return;
5830 }
5831
5832 // This must be an upgrade from a named to a literal struct.
5833 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5834 assert(OldST != NewFn->getReturnType() &&
5835 "Return type must have changed");
5836 assert(OldST->getNumElements() ==
5837 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5838 "Must have same number of elements");
5839
5840 SmallVector<Value *> Args(CI->args());
5841 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5842 NewCI->setAttributes(CI->getAttributes());
5843 Value *Res = PoisonValue::get(OldST);
5844 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5845 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5846 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5847 }
5848 CI->replaceAllUsesWith(Res);
5849 CI->eraseFromParent();
5850 return;
5851 }
5852
5853 // We're probably about to produce something invalid. Let the verifier catch
5854 // it instead of dying here.
5855 CI->setCalledOperand(
5856 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5857 return;
5858 };
5859 CallInst *NewCall = nullptr;
5860 switch (NewFn->getIntrinsicID()) {
5861 default: {
5862 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5863 return;
5864 DefaultCase();
5865 return;
5866 }
5867 case Intrinsic::arm_neon_vst1:
5868 case Intrinsic::arm_neon_vst2:
5869 case Intrinsic::arm_neon_vst3:
5870 case Intrinsic::arm_neon_vst4:
5871 case Intrinsic::arm_neon_vst2lane:
5872 case Intrinsic::arm_neon_vst3lane:
5873 case Intrinsic::arm_neon_vst4lane: {
5874 SmallVector<Value *, 4> Args(CI->args());
5875 NewCall = Builder.CreateCall(NewFn, Args);
5876 break;
5877 }
5878 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5879 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5880 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5881 LLVMContext &Ctx = F->getParent()->getContext();
5882 SmallVector<Value *, 4> Args(CI->args());
5883 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5884 cast<ConstantInt>(Args[3])->getZExtValue());
5885 NewCall = Builder.CreateCall(NewFn, Args);
5886 break;
5887 }
5888 case Intrinsic::aarch64_sve_ld3_sret:
5889 case Intrinsic::aarch64_sve_ld4_sret:
5890 case Intrinsic::aarch64_sve_ld2_sret: {
5891 // Is this a trivial remangle of the name to support ptr address spaces?
5892 if (isa<StructType>(F->getReturnType())) {
5893 DefaultCase();
5894 return;
5895 }
5896
5897 StringRef Name = F->getName();
5898 Name = Name.substr(5);
5899 unsigned N = StringSwitch<unsigned>(Name)
5900 .StartsWith("aarch64.sve.ld2", 2)
5901 .StartsWith("aarch64.sve.ld3", 3)
5902 .StartsWith("aarch64.sve.ld4", 4)
5903 .Default(0);
5904 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5905 unsigned MinElts = RetTy->getMinNumElements() / N;
5906 SmallVector<Value *, 2> Args(CI->args());
5907 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5908 Value *Ret = llvm::PoisonValue::get(RetTy);
5909 for (unsigned I = 0; I < N; I++) {
5910 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5911 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5912 }
5913 NewCall = dyn_cast<CallInst>(Ret);
5914 break;
5915 }
5916
5917 case Intrinsic::coro_end_async:
5918 case Intrinsic::coro_end: {
5919 SmallVector<Value *, 3> Args(CI->args());
5920 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5921 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5922 NewCall = Builder.CreateCall(NewFn, Args);
5923
5924 if (!CI->getType()->isVoidTy()) {
5925 if (!CI->use_empty()) {
5926 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5927 CI->getModule(), Intrinsic::coro_is_in_ramp);
5928 Value *InRamp = Builder.CreateCall(IsInRamp);
5929 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5930 }
5931 CI->eraseFromParent();
5932 return;
5933 }
5934
5935 break;
5936 }
5937
5938 case Intrinsic::vector_extract: {
5939 StringRef Name = F->getName();
5940 Name = Name.substr(5); // Strip llvm
5941 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5942 DefaultCase();
5943 return;
5944 }
5945 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5946 unsigned MinElts = RetTy->getMinNumElements();
5947 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5948 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5949 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5950 break;
5951 }
5952
5953 case Intrinsic::vector_insert: {
5954 StringRef Name = F->getName();
5955 Name = Name.substr(5);
5956 if (!Name.starts_with("aarch64.sve.tuple")) {
5957 DefaultCase();
5958 return;
5959 }
5960 if (Name.starts_with("aarch64.sve.tuple.set")) {
5961 uint64_t I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5962 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5963 Value *NewIdx =
5964 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5965 NewCall = Builder.CreateCall(
5966 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5967 break;
5968 }
5969 if (Name.starts_with("aarch64.sve.tuple.create")) {
5970 unsigned N = StringSwitch<unsigned>(Name)
5971 .StartsWith("aarch64.sve.tuple.create2", 2)
5972 .StartsWith("aarch64.sve.tuple.create3", 3)
5973 .StartsWith("aarch64.sve.tuple.create4", 4)
5974 .Default(0);
5975 assert(N > 1 && "Create is expected to be between 2-4");
5976 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5977 Value *Ret = llvm::PoisonValue::get(RetTy);
5978 unsigned MinElts = RetTy->getMinNumElements() / N;
5979 for (unsigned I = 0; I < N; I++) {
5980 Value *V = CI->getArgOperand(I);
5981 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5982 }
5983 NewCall = dyn_cast<CallInst>(Ret);
5984 }
5985 break;
5986 }
5987
5988 case Intrinsic::arm_neon_bfdot:
5989 case Intrinsic::arm_neon_bfmmla:
5990 case Intrinsic::arm_neon_bfmlalb:
5991 case Intrinsic::arm_neon_bfmlalt:
5992 case Intrinsic::aarch64_neon_bfdot:
5993 case Intrinsic::aarch64_neon_bfmmla:
5994 case Intrinsic::aarch64_neon_bfmlalb:
5995 case Intrinsic::aarch64_neon_bfmlalt: {
5996 SmallVector<Value *, 3> Args;
5997 assert(CI->arg_size() == 3 &&
5998 "Mismatch between function args and call args");
5999 size_t OperandWidth =
6000 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
6001 assert((OperandWidth == 64 || OperandWidth == 128) &&
6002 "Unexpected operand width");
6003 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
6004 auto Iter = CI->args().begin();
6005 Args.push_back(*Iter++);
6006 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6007 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
6008 NewCall = Builder.CreateCall(NewFn, Args);
6009 break;
6010 }
6011
6012 case Intrinsic::bitreverse:
6013 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6014 break;
6015
6016 case Intrinsic::ctlz:
6017 case Intrinsic::cttz: {
6018 if (CI->arg_size() != 1) {
6019 DefaultCase();
6020 return;
6021 }
6022
6023 NewCall =
6024 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
6025 break;
6026 }
6027
6028 case Intrinsic::objectsize: {
6029 Value *NullIsUnknownSize =
6030 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
6031 Value *Dynamic =
6032 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
6033 NewCall = Builder.CreateCall(
6034 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
6035 break;
6036 }
6037
6038 case Intrinsic::ctpop:
6039 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
6040 break;
6041 case Intrinsic::dbg_value: {
6042 StringRef Name = F->getName();
6043 Name = Name.substr(5); // Strip llvm.
6044 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
6045 if (Name.starts_with("dbg.addr")) {
6046 DIExpression *Expr = cast<DIExpression>(
6047 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
6048 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
6049 NewCall =
6050 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6051 MetadataAsValue::get(C, Expr)});
6052 break;
6053 }
6054
6055 // Upgrade from the old version that had an extra offset argument.
6056 assert(CI->arg_size() == 4);
6057 // Drop nonzero offsets instead of attempting to upgrade them.
6058 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
6059 if (Offset->isNullValue()) {
6060 NewCall = Builder.CreateCall(
6061 NewFn,
6062 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
6063 break;
6064 }
6065 CI->eraseFromParent();
6066 return;
6067 }
6068
6069 case Intrinsic::ptr_annotation:
6070 // Upgrade from versions that lacked the annotation attribute argument.
6071 if (CI->arg_size() != 4) {
6072 DefaultCase();
6073 return;
6074 }
6075
6076 // Create a new call with an added null annotation attribute argument.
6077 NewCall = Builder.CreateCall(
6078 NewFn,
6079 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6080 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6081 NewCall->takeName(CI);
6082 CI->replaceAllUsesWith(NewCall);
6083 CI->eraseFromParent();
6084 return;
6085
6086 case Intrinsic::var_annotation:
6087 // Upgrade from versions that lacked the annotation attribute argument.
6088 if (CI->arg_size() != 4) {
6089 DefaultCase();
6090 return;
6091 }
6092 // Create a new call with an added null annotation attribute argument.
6093 NewCall = Builder.CreateCall(
6094 NewFn,
6095 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
6096 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
6097 NewCall->takeName(CI);
6098 CI->replaceAllUsesWith(NewCall);
6099 CI->eraseFromParent();
6100 return;
6101
6102 case Intrinsic::riscv_aes32dsi:
6103 case Intrinsic::riscv_aes32dsmi:
6104 case Intrinsic::riscv_aes32esi:
6105 case Intrinsic::riscv_aes32esmi:
6106 case Intrinsic::riscv_sm4ks:
6107 case Intrinsic::riscv_sm4ed: {
6108 // The last argument to these intrinsics used to be i8 and changed to i32.
6109 // The type overload for sm4ks and sm4ed was removed.
6110 Value *Arg2 = CI->getArgOperand(2);
6111 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
6112 return;
6113
6114 Value *Arg0 = CI->getArgOperand(0);
6115 Value *Arg1 = CI->getArgOperand(1);
6116 if (CI->getType()->isIntegerTy(64)) {
6117 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
6118 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
6119 }
6120
6121 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
6122 cast<ConstantInt>(Arg2)->getZExtValue());
6123
6124 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
6125 Value *Res = NewCall;
6126 if (Res->getType() != CI->getType())
6127 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6128 NewCall->takeName(CI);
6129 CI->replaceAllUsesWith(Res);
6130 CI->eraseFromParent();
6131 return;
6132 }
6133 case Intrinsic::nvvm_mapa_shared_cluster: {
6134 // Create a new call with the correct address space.
6135 NewCall =
6136 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
6137 Value *Res = NewCall;
6138 Res = Builder.CreateAddrSpaceCast(
6139 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
6140 NewCall->takeName(CI);
6141 CI->replaceAllUsesWith(Res);
6142 CI->eraseFromParent();
6143 return;
6144 }
6145 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster: {
6146 SmallVector<Value *, 4> Args(CI->args());
6147 unsigned AS = Args[0]->getType()->getPointerAddressSpace();
6148 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6149 Args[0] = Builder.CreateAddrSpaceCast(
6150 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6151
6152 // Append the missing trailing flag_valid_pattern (0 = disabled).
6153 Args.push_back(Builder.getInt32(0));
6154
6155 NewCall = Builder.CreateCall(NewFn, Args);
6156 NewCall->takeName(CI);
6157 CI->replaceAllUsesWith(NewCall);
6158 CI->eraseFromParent();
6159 return;
6160 }
6161 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cta: {
6162 // (dst, mbar, src, size, ch, flag_ch)
6163 // -> (dst, mbar, src, size, i32 0, i32 0, ch, flag_ch, i1 false,
6164 // i32 0 /* flag_valid_pattern=disabled */)
6165 SmallVector<Value *, 10> Args;
6166 for (unsigned I = 0; I < 4; ++I)
6167 Args.push_back(CI->getArgOperand(I));
6168 Args.push_back(Builder.getInt32(0)); // ignore_bytes_left
6169 Args.push_back(Builder.getInt32(0)); // ignore_bytes_right
6170 Args.push_back(CI->getArgOperand(4)); // cache_hint
6171 Args.push_back(CI->getArgOperand(5)); // flag_ch
6172 Args.push_back(Builder.getInt1(false)); // flag_oob
6173 Args.push_back(Builder.getInt32(0)); // flag_valid_pattern
6174
6175 NewCall = Builder.CreateCall(NewFn, Args);
6176 NewCall->takeName(CI);
6177 CI->replaceAllUsesWith(NewCall);
6178 CI->eraseFromParent();
6179 return;
6180 }
6181 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
6182 // Create a new call with the correct address space.
6183 SmallVector<Value *, 4> Args(CI->args());
6184 Args[0] = Builder.CreateAddrSpaceCast(
6185 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6186
6187 NewCall = Builder.CreateCall(NewFn, Args);
6188 NewCall->takeName(CI);
6189 CI->replaceAllUsesWith(NewCall);
6190 CI->eraseFromParent();
6191 return;
6192 }
6193 // clang-format off
6194#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME) \
6195 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:
6196 NVVM_TMA_G2S_MODES(G2S_CLUSTER_CASE)
6197#undef G2S_CLUSTER_CASE
6198 {
6199 SmallVector<Value *, 16> Args(CI->args());
6200 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
6201 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
6202 Args[0] = Builder.CreateAddrSpaceCast(
6203 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
6204
6205 // Append the missing trailing arguments with default values (cta_group,
6206 // flag_valid_pattern).
6207 while (Args.size() < NewFn->getFunctionType()->getNumParams())
6208 Args.push_back(Builder.getInt32(0));
6209
6210 NewCall = Builder.CreateCall(NewFn, Args);
6211 NewCall->takeName(CI);
6212 CI->replaceAllUsesWith(NewCall);
6213 CI->eraseFromParent();
6214 return;
6215 }
6216
6217#define G2S_CTA_CASE(ID_SUFFIX, NAME) \
6218 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:
6219 NVVM_TMA_G2S_MODES(G2S_CTA_CASE)
6220#undef G2S_CTA_CASE
6221 {
6222 SmallVector<Value *, 16> Args(CI->args());
6223 // Append the missing trailing flag_valid_pattern argument with default
6224 // value 0.
6225 assert(Args.size() + 1 == NewFn->getFunctionType()->getNumParams() &&
6226 "expected only the trailing flag_valid_pattern to be missing");
6227 Args.push_back(Builder.getInt32(0));
6228
6229 NewCall = Builder.CreateCall(NewFn, Args);
6230 NewCall->takeName(CI);
6231 CI->replaceAllUsesWith(NewCall);
6232 CI->eraseFromParent();
6233 return;
6234 }
6235#undef NVVM_TMA_G2S_MODES
6236 // clang-format on
6237
6238 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
6239 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
6240 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
6241 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
6242 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
6243 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
6244 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
6245 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
6246 StringRef Name = F->getName();
6247 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
6248 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
6249
6250 SmallVector<Value *, 16> Args(CI->args());
6251 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
6252 NewCall = Builder.CreateCall(NewFn, Args);
6253 break;
6254 }
6255 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6256 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6257 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6258 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6259 NewCall =
6260 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6261 Builder.getFalse()});
6262 break;
6263 case Intrinsic::nvvm_mbarrier_init: {
6264 SmallVector<Value *, 3> Args(CI->args());
6265 // The .shared variant folded into the overloaded form without gaining an
6266 // operand, so only the pre-layout two-argument form needs one appended.
6267 if (Args.size() == 2)
6268 Args.push_back(Builder.getInt32(0)); // layout = default(0)
6269 NewCall = Builder.CreateCall(NewFn, Args);
6270 break;
6271 }
6272 case Intrinsic::riscv_sha256sig0:
6273 case Intrinsic::riscv_sha256sig1:
6274 case Intrinsic::riscv_sha256sum0:
6275 case Intrinsic::riscv_sha256sum1:
6276 case Intrinsic::riscv_sm3p0:
6277 case Intrinsic::riscv_sm3p1: {
6278 // The last argument to these intrinsics used to be i8 and changed to i32.
6279 // The type overload for sm4ks and sm4ed was removed.
6280 if (!CI->getType()->isIntegerTy(64))
6281 return;
6282
6283 Value *Arg =
6284 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6285
6286 NewCall = Builder.CreateCall(NewFn, Arg);
6287 Value *Res =
6288 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6289 NewCall->takeName(CI);
6290 CI->replaceAllUsesWith(Res);
6291 CI->eraseFromParent();
6292 return;
6293 }
6294
6295 case Intrinsic::x86_xop_vfrcz_ss:
6296 case Intrinsic::x86_xop_vfrcz_sd:
6297 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6298 break;
6299
6300 case Intrinsic::x86_xop_vpermil2pd:
6301 case Intrinsic::x86_xop_vpermil2ps:
6302 case Intrinsic::x86_xop_vpermil2pd_256:
6303 case Intrinsic::x86_xop_vpermil2ps_256: {
6304 SmallVector<Value *, 4> Args(CI->args());
6305 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6306 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6307 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6308 NewCall = Builder.CreateCall(NewFn, Args);
6309 break;
6310 }
6311
6312 case Intrinsic::x86_sse41_ptestc:
6313 case Intrinsic::x86_sse41_ptestz:
6314 case Intrinsic::x86_sse41_ptestnzc: {
6315 // The arguments for these intrinsics used to be v4f32, and changed
6316 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6317 // So, the only thing required is a bitcast for both arguments.
6318 // First, check the arguments have the old type.
6319 Value *Arg0 = CI->getArgOperand(0);
6320 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6321 return;
6322
6323 // Old intrinsic, add bitcasts
6324 Value *Arg1 = CI->getArgOperand(1);
6325
6326 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6327
6328 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6329 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6330
6331 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6332 break;
6333 }
6334
6335 case Intrinsic::x86_rdtscp: {
6336 // This used to take 1 arguments. If we have no arguments, it is already
6337 // upgraded.
6338 if (CI->getNumOperands() == 0)
6339 return;
6340
6341 NewCall = Builder.CreateCall(NewFn);
6342 // Extract the second result and store it.
6343 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6344 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6345 // Replace the original call result with the first result of the new call.
6346 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6347
6348 NewCall->takeName(CI);
6349 CI->replaceAllUsesWith(TSC);
6350 CI->eraseFromParent();
6351 return;
6352 }
6353
6354 case Intrinsic::x86_sse41_insertps:
6355 case Intrinsic::x86_sse41_dppd:
6356 case Intrinsic::x86_sse41_dpps:
6357 case Intrinsic::x86_sse41_mpsadbw:
6358 case Intrinsic::x86_avx_dp_ps_256:
6359 case Intrinsic::x86_avx2_mpsadbw: {
6360 // Need to truncate the last argument from i32 to i8 -- this argument models
6361 // an inherently 8-bit immediate operand to these x86 instructions.
6362 SmallVector<Value *, 4> Args(CI->args());
6363
6364 // Replace the last argument with a trunc.
6365 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6366 NewCall = Builder.CreateCall(NewFn, Args);
6367 break;
6368 }
6369
6370 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6371 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6372 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6373 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6374 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6375 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6376 SmallVector<Value *, 4> Args(CI->args());
6377 unsigned NumElts =
6378 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6379 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6380
6381 NewCall = Builder.CreateCall(NewFn, Args);
6382 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6383
6384 NewCall->takeName(CI);
6385 CI->replaceAllUsesWith(Res);
6386 CI->eraseFromParent();
6387 return;
6388 }
6389
6390 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6391 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6392 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6393 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6394 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6395 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6396 SmallVector<Value *, 4> Args(CI->args());
6397 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6398 if (NewFn->getIntrinsicID() ==
6399 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6400 Args[1] = Builder.CreateBitCast(
6401 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6402
6403 NewCall = Builder.CreateCall(NewFn, Args);
6404 Value *Res = Builder.CreateBitCast(
6405 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6406
6407 NewCall->takeName(CI);
6408 CI->replaceAllUsesWith(Res);
6409 CI->eraseFromParent();
6410 return;
6411 }
6412 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6413 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6414 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6415 SmallVector<Value *, 4> Args(CI->args());
6416 unsigned NumElts =
6417 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6418 Args[1] = Builder.CreateBitCast(
6419 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6420 Args[2] = Builder.CreateBitCast(
6421 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6422
6423 NewCall = Builder.CreateCall(NewFn, Args);
6424 break;
6425 }
6426
6427 case Intrinsic::thread_pointer: {
6428 NewCall = Builder.CreateCall(NewFn, {});
6429 break;
6430 }
6431
6432 case Intrinsic::memcpy:
6433 case Intrinsic::memmove:
6434 case Intrinsic::memset: {
6435 // We have to make sure that the call signature is what we're expecting.
6436 // We only want to change the old signatures by removing the alignment arg:
6437 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6438 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6439 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6440 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6441 // Note: i8*'s in the above can be any pointer type
6442 if (CI->arg_size() != 5) {
6443 DefaultCase();
6444 return;
6445 }
6446 // Remove alignment argument (3), and add alignment attributes to the
6447 // dest/src pointers.
6448 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6449 CI->getArgOperand(2), CI->getArgOperand(4)};
6450 NewCall = Builder.CreateCall(NewFn, Args);
6451 AttributeList OldAttrs = CI->getAttributes();
6452 AttributeList NewAttrs = AttributeList::get(
6453 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6454 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6455 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6456 NewCall->setAttributes(NewAttrs);
6457 auto *MemCI = cast<MemIntrinsic>(NewCall);
6458 // All mem intrinsics support dest alignment.
6459 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6460 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6461 // Memcpy/Memmove also support source alignment.
6462 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6463 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6464 break;
6465 }
6466
6467 case Intrinsic::masked_load:
6468 case Intrinsic::masked_gather:
6469 case Intrinsic::masked_store:
6470 case Intrinsic::masked_scatter: {
6471 if (CI->arg_size() != 4) {
6472 DefaultCase();
6473 return;
6474 }
6475
6476 auto GetMaybeAlign = [](Value *Op) {
6477 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6478 uint64_t Val = CI->getZExtValue();
6479 if (Val == 0)
6480 return MaybeAlign();
6481 if (isPowerOf2_64(Val))
6482 return MaybeAlign(Val);
6483 }
6484 reportFatalUsageError("Invalid alignment argument");
6485 };
6486 auto GetAlign = [&](Value *Op) {
6487 MaybeAlign Align = GetMaybeAlign(Op);
6488 if (Align)
6489 return *Align;
6490 reportFatalUsageError("Invalid zero alignment argument");
6491 };
6492
6493 const DataLayout &DL = CI->getDataLayout();
6494 switch (NewFn->getIntrinsicID()) {
6495 case Intrinsic::masked_load:
6496 NewCall = Builder.CreateMaskedLoad(
6497 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6498 CI->getArgOperand(2), CI->getArgOperand(3));
6499 break;
6500 case Intrinsic::masked_gather:
6501 NewCall = Builder.CreateMaskedGather(
6502 CI->getType(), CI->getArgOperand(0),
6503 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6504 CI->getType()->getScalarType()),
6505 CI->getArgOperand(2), CI->getArgOperand(3));
6506 break;
6507 case Intrinsic::masked_store:
6508 NewCall = Builder.CreateMaskedStore(
6509 CI->getArgOperand(0), CI->getArgOperand(1),
6510 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6511 break;
6512 case Intrinsic::masked_scatter:
6513 NewCall = Builder.CreateMaskedScatter(
6514 CI->getArgOperand(0), CI->getArgOperand(1),
6515 DL.getValueOrABITypeAlignment(
6516 GetMaybeAlign(CI->getArgOperand(2)),
6517 CI->getArgOperand(0)->getType()->getScalarType()),
6518 CI->getArgOperand(3));
6519 break;
6520 default:
6521 llvm_unreachable("Unexpected intrinsic ID");
6522 }
6523 // Previous metadata is still valid.
6524 NewCall->copyMetadata(*CI);
6525 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6526 break;
6527 }
6528
6529 case Intrinsic::lifetime_start:
6530 case Intrinsic::lifetime_end: {
6531 if (CI->arg_size() != 2) {
6532 DefaultCase();
6533 return;
6534 }
6535
6536 Value *Ptr = CI->getArgOperand(1);
6537 // Try to strip pointer casts, such that the lifetime works on an alloca.
6538 Ptr = Ptr->stripPointerCasts();
6539 if (isa<AllocaInst>(Ptr)) {
6540 // Don't use NewFn, as we might have looked through an addrspacecast.
6541 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6542 NewCall = Builder.CreateLifetimeStart(Ptr);
6543 else
6544 NewCall = Builder.CreateLifetimeEnd(Ptr);
6545 break;
6546 }
6547
6548 // Otherwise remove the lifetime marker.
6549 CI->eraseFromParent();
6550 return;
6551 }
6552
6553 case Intrinsic::x86_avx512_vpdpbusd_128:
6554 case Intrinsic::x86_avx512_vpdpbusd_256:
6555 case Intrinsic::x86_avx512_vpdpbusd_512:
6556 case Intrinsic::x86_avx512_vpdpbusds_128:
6557 case Intrinsic::x86_avx512_vpdpbusds_256:
6558 case Intrinsic::x86_avx512_vpdpbusds_512:
6559 case Intrinsic::x86_avx2_vpdpbssd_128:
6560 case Intrinsic::x86_avx2_vpdpbssd_256:
6561 case Intrinsic::x86_avx10_vpdpbssd_512:
6562 case Intrinsic::x86_avx2_vpdpbssds_128:
6563 case Intrinsic::x86_avx2_vpdpbssds_256:
6564 case Intrinsic::x86_avx10_vpdpbssds_512:
6565 case Intrinsic::x86_avx2_vpdpbsud_128:
6566 case Intrinsic::x86_avx2_vpdpbsud_256:
6567 case Intrinsic::x86_avx10_vpdpbsud_512:
6568 case Intrinsic::x86_avx2_vpdpbsuds_128:
6569 case Intrinsic::x86_avx2_vpdpbsuds_256:
6570 case Intrinsic::x86_avx10_vpdpbsuds_512:
6571 case Intrinsic::x86_avx2_vpdpbuud_128:
6572 case Intrinsic::x86_avx2_vpdpbuud_256:
6573 case Intrinsic::x86_avx10_vpdpbuud_512:
6574 case Intrinsic::x86_avx2_vpdpbuuds_128:
6575 case Intrinsic::x86_avx2_vpdpbuuds_256:
6576 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6577 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6578 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6579 CI->getArgOperand(2)};
6580 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6581 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6582 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6583
6584 NewCall = Builder.CreateCall(NewFn, Args);
6585 break;
6586 }
6587 case Intrinsic::x86_avx512_vpdpwssd_128:
6588 case Intrinsic::x86_avx512_vpdpwssd_256:
6589 case Intrinsic::x86_avx512_vpdpwssd_512:
6590 case Intrinsic::x86_avx512_vpdpwssds_128:
6591 case Intrinsic::x86_avx512_vpdpwssds_256:
6592 case Intrinsic::x86_avx512_vpdpwssds_512:
6593 case Intrinsic::x86_avx2_vpdpwsud_128:
6594 case Intrinsic::x86_avx2_vpdpwsud_256:
6595 case Intrinsic::x86_avx10_vpdpwsud_512:
6596 case Intrinsic::x86_avx2_vpdpwsuds_128:
6597 case Intrinsic::x86_avx2_vpdpwsuds_256:
6598 case Intrinsic::x86_avx10_vpdpwsuds_512:
6599 case Intrinsic::x86_avx2_vpdpwusd_128:
6600 case Intrinsic::x86_avx2_vpdpwusd_256:
6601 case Intrinsic::x86_avx10_vpdpwusd_512:
6602 case Intrinsic::x86_avx2_vpdpwusds_128:
6603 case Intrinsic::x86_avx2_vpdpwusds_256:
6604 case Intrinsic::x86_avx10_vpdpwusds_512:
6605 case Intrinsic::x86_avx2_vpdpwuud_128:
6606 case Intrinsic::x86_avx2_vpdpwuud_256:
6607 case Intrinsic::x86_avx10_vpdpwuud_512:
6608 case Intrinsic::x86_avx2_vpdpwuuds_128:
6609 case Intrinsic::x86_avx2_vpdpwuuds_256:
6610 case Intrinsic::x86_avx10_vpdpwuuds_512:
6611 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6612 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6613 CI->getArgOperand(2)};
6614 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6615 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6616 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6617
6618 NewCall = Builder.CreateCall(NewFn, Args);
6619 break;
6620 }
6621 assert(NewCall && "Should have either set this variable or returned through "
6622 "the default case");
6623 NewCall->takeName(CI);
6624 CI->replaceAllUsesWith(NewCall);
6625 CI->eraseFromParent();
6626}
6627
6628void llvm::UpgradeCallsToIntrinsic(Function *F) {
6629 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6630
6631 // Check if this function should be upgraded and get the replacement function
6632 // if there is one.
6633 Function *NewFn;
6634 if (UpgradeIntrinsicFunction(F, NewFn)) {
6635 // Replace all users of the old function with the new function or new
6636 // instructions. This is not a range loop because the call is deleted.
6637 for (User *U : make_early_inc_range(F->users()))
6638 if (CallBase *CB = dyn_cast<CallBase>(U))
6639 UpgradeIntrinsicCall(CB, NewFn);
6640
6641 // Remove old function, no longer used, from the module.
6642 if (F != NewFn)
6643 F->eraseFromParent();
6644 }
6645}
6646
6647MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6648 const unsigned NumOperands = MD.getNumOperands();
6649 if (NumOperands == 0)
6650 return &MD; // Invalid, punt to a verifier error.
6651
6652 // Check if the tag uses struct-path aware TBAA format.
6653 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6654 return &MD;
6655
6656 auto &Context = MD.getContext();
6657 if (NumOperands == 3) {
6658 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6659 MDNode *ScalarType = MDNode::get(Context, Elts);
6660 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6661 Metadata *Elts2[] = {ScalarType, ScalarType,
6662 ConstantAsMetadata::get(
6663 Constant::getNullValue(Type::getInt64Ty(Context))),
6664 MD.getOperand(2)};
6665 return MDNode::get(Context, Elts2);
6666 }
6667 // Create a MDNode <MD, MD, offset 0>
6668 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6669 Type::getInt64Ty(Context)))};
6670 return MDNode::get(Context, Elts);
6671}
6672
6673Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6674 Instruction *&Temp) {
6675 if (Opc != Instruction::BitCast)
6676 return nullptr;
6677
6678 Temp = nullptr;
6679 Type *SrcTy = V->getType();
6680 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6681 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6682 LLVMContext &Context = V->getContext();
6683
6684 // We have no information about target data layout, so we assume that
6685 // the maximum pointer size is 64bit.
6686 Type *MidTy = Type::getInt64Ty(Context);
6687 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6688
6689 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6690 }
6691
6692 return nullptr;
6693}
6694
6695Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6696 if (Opc != Instruction::BitCast)
6697 return nullptr;
6698
6699 Type *SrcTy = C->getType();
6700 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6701 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6702 LLVMContext &Context = C->getContext();
6703
6704 // We have no information about target data layout, so we assume that
6705 // the maximum pointer size is 64bit.
6706 Type *MidTy = Type::getInt64Ty(Context);
6707
6708 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6709 DestTy);
6710 }
6711
6712 return nullptr;
6713}
6714
6715static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6716 if (Flag.getNumOperands() < 3)
6717 return std::nullopt;
6718 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6719 return Name->getString();
6720 return std::nullopt;
6721}
6722
6723/// Check the debug info version number, if it is out-dated, drop the debug
6724/// info. Return true if module is modified.
6725bool llvm::UpgradeDebugInfo(Module &M) {
6726 if (DisableAutoUpgradeDebugInfo)
6727 return false;
6728
6729 llvm::TimeTraceScope timeScope("Upgrade debug info");
6730 // We need to get metadata before the module is verified (i.e., getModuleFlag
6731 // makes assumptions that we haven't verified yet). Carefully extract the flag
6732 // from the metadata.
6733 unsigned Version = 0;
6734 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6735 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6736 if (auto Name = getModuleFlagNameSafely(*Flag))
6737 return *Name == "Debug Info Version";
6738 return false;
6739 });
6740 if (OpIt != ModFlags->op_end()) {
6741 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6742 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6743 Version = CI->getZExtValue();
6744 }
6745 }
6746
6747 if (Version == DEBUG_METADATA_VERSION) {
6748 bool BrokenDebugInfo = false;
6749 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6750 report_fatal_error("Broken module found, compilation aborted!");
6751 if (!BrokenDebugInfo)
6752 // Everything is ok.
6753 return false;
6754 else {
6755 // Diagnose malformed debug info.
6756 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6757 M.getContext().diagnose(Diag);
6758 }
6759 }
6760 bool Modified = StripDebugInfo(M);
6761 if (Modified && Version != DEBUG_METADATA_VERSION) {
6762 // Diagnose a version mismatch.
6763 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6764 M.getContext().diagnose(DiagVersion);
6765 }
6766 return Modified;
6767}
6768
6769static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6770 GlobalValue *GV, const Metadata *V) {
6771 Function *F = cast<Function>(GV);
6772
6773 constexpr StringLiteral DefaultValue = "1";
6774 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6775 unsigned Length = 0;
6776
6777 if (F->hasFnAttribute(Attr)) {
6778 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6779 // parse these elements placing them into Vect3
6780 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6781 for (; Length < 3 && !S.empty(); Length++) {
6782 auto [Part, Rest] = S.split(',');
6783 Vect3[Length] = Part.trim();
6784 S = Rest;
6785 }
6786 }
6787
6788 const unsigned Dim = DimC - 'x';
6789 assert(Dim < 3 && "Unexpected dim char");
6790
6791 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6792
6793 // local variable required for StringRef in Vect3 to point to.
6794 const std::string VStr = llvm::utostr(VInt);
6795 Vect3[Dim] = VStr;
6796 Length = std::max(Length, Dim + 1);
6797
6798 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6799 F->addFnAttr(Attr, NewAttr);
6800}
6801
6802static inline bool isXYZ(StringRef S) {
6803 return S == "x" || S == "y" || S == "z";
6804}
6805
6806bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6807 const Metadata *V) {
6808 if (K == "kernel") {
6809 if (!mdconst::extract<ConstantInt>(V)->isZero())
6810 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6811 return true;
6812 }
6813 if (K == "align") {
6814 // V is a bitfeild specifying two 16-bit values. The alignment value is
6815 // specfied in low 16-bits, The index is specified in the high bits. For the
6816 // index, 0 indicates the return value while higher values correspond to
6817 // each parameter (idx = param + 1).
6818 const uint64_t AlignIdxValuePair =
6819 mdconst::extract<ConstantInt>(V)->getZExtValue();
6820 const unsigned Idx = (AlignIdxValuePair >> 16);
6821 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6822 cast<Function>(GV)->addAttributeAtIndex(
6823 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6824 return true;
6825 }
6826 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6827 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6828 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6829 return true;
6830 }
6831 if (K == "minctasm") {
6832 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6833 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6834 return true;
6835 }
6836 if (K == "maxnreg") {
6837 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6838 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6839 return true;
6840 }
6841 if (K.consume_front("maxntid") && isXYZ(K)) {
6842 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6843 return true;
6844 }
6845 if (K.consume_front("reqntid") && isXYZ(K)) {
6846 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6847 return true;
6848 }
6849 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6850 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6851 return true;
6852 }
6853 if (K == "grid_constant") {
6854 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6855 for (const auto &Op : cast<MDNode>(V)->operands()) {
6856 // For some reason, the index is 1-based in the metadata. Good thing we're
6857 // able to auto-upgrade it!
6858 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6859 cast<Function>(GV)->addParamAttr(Index, Attr);
6860 }
6861 return true;
6862 }
6863
6864 return false;
6865}
6866
6867void llvm::UpgradeNVVMAnnotations(Module &M) {
6868 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6869 if (!NamedMD)
6870 return;
6871
6872 SmallVector<MDNode *, 8> NewNodes;
6873 SmallPtrSet<const MDNode *, 8> SeenNodes;
6874 for (MDNode *MD : NamedMD->operands()) {
6875 if (!SeenNodes.insert(MD).second)
6876 continue;
6877
6878 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6879 if (!GV)
6880 continue;
6881
6882 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6883
6884 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6885 // Each nvvm.annotations metadata entry will be of the following form:
6886 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6887 // start index = 1, to skip the global variable key
6888 // increment = 2, to skip the value for each property-value pairs
6889 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6890 MDString *K = cast<MDString>(MD->getOperand(j));
6891 const MDOperand &V = MD->getOperand(j + 1);
6892 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6893 if (!Upgraded)
6894 NewOperands.append({K, V});
6895 }
6896
6897 if (NewOperands.size() > 1)
6898 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6899 }
6900
6901 NamedMD->clearOperands();
6902 for (MDNode *N : NewNodes)
6903 NamedMD->addOperand(N);
6904}
6905
6906/// This checks for objc retain release marker which should be upgraded. It
6907/// returns true if module is modified.
6908static bool upgradeRetainReleaseMarker(Module &M) {
6909 bool Changed = false;
6910 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6911 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6912 if (ModRetainReleaseMarker) {
6913 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6914 if (Op) {
6915 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6916 if (ID) {
6917 SmallVector<StringRef, 4> ValueComp;
6918 ID->getString().split(ValueComp, "#");
6919 if (ValueComp.size() == 2) {
6920 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6921 ID = MDString::get(M.getContext(), NewValue);
6922 }
6923 M.addModuleFlag(Module::Error, MarkerKey, ID);
6924 M.eraseNamedMetadata(ModRetainReleaseMarker);
6925 Changed = true;
6926 }
6927 }
6928 }
6929 return Changed;
6930}
6931
6932void llvm::UpgradeARCRuntime(Module &M) {
6933 // This lambda converts normal function calls to ARC runtime functions to
6934 // intrinsic calls.
6935 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6936 llvm::Intrinsic::ID IntrinsicFunc) {
6937 Function *Fn = M.getFunction(OldFunc);
6938
6939 if (!Fn)
6940 return;
6941
6942 Function *NewFn =
6943 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6944
6945 for (User *U : make_early_inc_range(Fn->users())) {
6946 CallInst *CI = dyn_cast<CallInst>(U);
6947 if (!CI || CI->getCalledFunction() != Fn)
6948 continue;
6949
6950 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6951 FunctionType *NewFuncTy = NewFn->getFunctionType();
6952 SmallVector<Value *, 2> Args;
6953
6954 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6955 // value to the return type of the old function.
6956 if (NewFuncTy->getReturnType() != CI->getType() &&
6957 !CastInst::castIsValid(Instruction::BitCast, CI,
6958 NewFuncTy->getReturnType()))
6959 continue;
6960
6961 bool InvalidCast = false;
6962
6963 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6964 Value *Arg = CI->getArgOperand(I);
6965
6966 // Bitcast argument to the parameter type of the new function if it's
6967 // not a variadic argument.
6968 if (I < NewFuncTy->getNumParams()) {
6969 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6970 // to the parameter type of the new function.
6971 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6972 NewFuncTy->getParamType(I))) {
6973 InvalidCast = true;
6974 break;
6975 }
6976 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6977 }
6978 Args.push_back(Arg);
6979 }
6980
6981 if (InvalidCast)
6982 continue;
6983
6984 // Create a call instruction that calls the new function.
6985 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6986 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6987 NewCall->takeName(CI);
6988
6989 // Bitcast the return value back to the type of the old call.
6990 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6991
6992 if (!CI->use_empty())
6993 CI->replaceAllUsesWith(NewRetVal);
6994 CI->eraseFromParent();
6995 }
6996
6997 if (Fn->use_empty())
6998 Fn->eraseFromParent();
6999 };
7000
7001 // Unconditionally convert a call to "clang.arc.use" to a call to
7002 // "llvm.objc.clang.arc.use".
7003 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
7004
7005 // Upgrade the retain release marker. If there is no need to upgrade
7006 // the marker, that means either the module is already new enough to contain
7007 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
7008 if (!upgradeRetainReleaseMarker(M))
7009 return;
7010
7011 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
7012 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
7013 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
7014 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
7015 {"objc_autoreleaseReturnValue",
7016 llvm::Intrinsic::objc_autoreleaseReturnValue},
7017 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
7018 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
7019 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
7020 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
7021 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
7022 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
7023 {"objc_release", llvm::Intrinsic::objc_release},
7024 {"objc_retain", llvm::Intrinsic::objc_retain},
7025 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
7026 {"objc_retainAutoreleaseReturnValue",
7027 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
7028 {"objc_retainAutoreleasedReturnValue",
7029 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
7030 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
7031 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
7032 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
7033 {"objc_unsafeClaimAutoreleasedReturnValue",
7034 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
7035 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
7036 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
7037 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
7038 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
7039 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
7040 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
7041 {"objc_arc_annotation_topdown_bbstart",
7042 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
7043 {"objc_arc_annotation_topdown_bbend",
7044 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
7045 {"objc_arc_annotation_bottomup_bbstart",
7046 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
7047 {"objc_arc_annotation_bottomup_bbend",
7048 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
7049
7050 for (auto &I : RuntimeFuncs)
7051 UpgradeToIntrinsic(I.first, I.second);
7052}
7053
7054// Upgrade the way signing of pointers to init/fini functions is described.
7055//
7056// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
7057// constants, if signing was requested. After the upgrade, these arrays contain
7058// plain function pointers and the desired signing schema is described via a
7059// pair of module flags.
7060//
7061// Note that the upgrade is only performed if all elements of *both* arrays
7062// agree on a common signing schema.
7063static bool upgradePtrauthInitFiniArrays(Module &M) {
7064 // As we cannot always decide whether the particular module should have
7065 // ptrauth-init-fini flags, we have to treat absent flags as having zero
7066 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
7067 // returns as soon as it spots any non-signed init/fini pointer: either we
7068 // should request non-signed pointers (safe to omit both flags) or there is
7069 // no common schema (and thus we do not modify anything).
7070 //
7071 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
7072 // or whether we should request address diversity in addition to the basic
7073 // constant diversity. There is no value representing "decided not to sign"
7074 // for the reasons explained above.
7075 std::optional<bool> UseAddressDisc;
7076
7077 // Do not attempt upgrading if the new module flags already exist.
7078 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
7079 for (const MDNode *Flag : ModFlags->operands()) {
7080 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
7081 if (Name && (*Name == "ptrauth-init-fini" ||
7082 *Name == "ptrauth-init-fini-address-discrimination"))
7083 return false;
7084 }
7085 }
7086
7087 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
7088 constexpr unsigned ExpectedConstDisc = 0xD9D4;
7089 constexpr unsigned ExpectedAddressMarker = 1;
7090
7091 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
7092 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
7093 return nullptr; // Nothing to upgrade or unknown pattern found.
7094
7095 bool HasAddressDisc;
7096 if (!CPA->hasAddressDiscriminator())
7097 HasAddressDisc = false;
7098 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
7099 HasAddressDisc = true;
7100 else
7101 return nullptr; // Unknown pattern.
7102
7103 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
7104 return nullptr; // Disagreement with the decided mode.
7105
7106 UseAddressDisc = HasAddressDisc;
7107 return CPA->getPointer();
7108 };
7109
7110 // Do not apply any changes until we know the upgrade is non-ambiguous.
7111 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
7112 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
7113
7114 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
7115 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
7116 if (!GV || !GV->hasInitializer())
7117 continue; // Skip, but it is okay to upgrade the other variable.
7118
7119 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
7120 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
7121 return false;
7122
7123 std::vector<Constant *> NewStructors;
7124 NewStructors.reserve(OldStructorsArray->getNumOperands());
7125
7126 for (Use &U : OldStructorsArray->operands()) {
7127 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
7128 if (!Structor || Structor->getNumOperands() != 3)
7129 return false;
7130
7131 Constant *Prio = Structor->getOperand(0);
7132 Constant *Func = Structor->getOperand(1);
7133 Constant *Arg = Structor->getOperand(2);
7134
7135 Func = UpgradeSinglePointer(Func);
7136 if (!Func)
7137 return false;
7138
7139 NewStructors.push_back(
7140 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
7141 }
7142
7143 Constant *NewInit =
7144 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
7145 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
7146 }
7147
7148 if (GlobalArraysToUpgrade.empty())
7149 return false;
7150 assert(UseAddressDisc.has_value());
7151
7152 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
7153 GV->setInitializer(NewInit);
7154
7155 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
7156 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
7157 *UseAddressDisc);
7158
7159 return true;
7160}
7161
7162bool llvm::UpgradeModuleFlags(Module &M) {
7163 bool Changed = false;
7164 Changed |= upgradePtrauthInitFiniArrays(M);
7165
7166 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7167 if (!ModFlags)
7168 return Changed;
7169
7170 bool HasObjCFlag = false, HasClassProperties = false;
7171 bool HasSwiftVersionFlag = false;
7172 uint8_t SwiftMajorVersion, SwiftMinorVersion;
7173 uint32_t SwiftABIVersion;
7174 auto Int8Ty = Type::getInt8Ty(M.getContext());
7175 auto Int32Ty = Type::getInt32Ty(M.getContext());
7176
7177 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7178 MDNode *Op = ModFlags->getOperand(I);
7179 if (Op->getNumOperands() != 3)
7180 continue;
7181 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7182 if (!ID)
7183 continue;
7184 auto SetBehavior = [&](Module::ModFlagBehavior B) {
7185 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
7186 Type::getInt32Ty(M.getContext()), B)),
7187 MDString::get(M.getContext(), ID->getString()),
7188 Op->getOperand(2)};
7189 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7190 Changed = true;
7191 };
7192
7193 if (ID->getString() == "Objective-C Image Info Version")
7194 HasObjCFlag = true;
7195 if (ID->getString() == "Objective-C Class Properties")
7196 HasClassProperties = true;
7197 // Upgrade PIC from Error/Max to Min.
7198 if (ID->getString() == "PIC Level") {
7199 if (auto *Behavior =
7200 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7201 uint64_t V = Behavior->getLimitedValue();
7202 if (V == Module::Error || V == Module::Max)
7203 SetBehavior(Module::Min);
7204 }
7205 }
7206 // Upgrade "PIE Level" from Error to Max.
7207 if (ID->getString() == "PIE Level")
7208 if (auto *Behavior =
7209 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7210 if (Behavior->getLimitedValue() == Module::Error)
7211 SetBehavior(Module::Max);
7212
7213 // Upgrade branch protection and return address signing module flags. The
7214 // module flag behavior for these fields were Error and now they are Min.
7215 if (ID->getString() == "branch-target-enforcement" ||
7216 ID->getString().starts_with("sign-return-address")) {
7217 if (auto *Behavior =
7218 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7219 if (Behavior->getLimitedValue() == Module::Error) {
7220 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7221 Metadata *Ops[3] = {
7222 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7223 Op->getOperand(1), Op->getOperand(2)};
7224 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7225 Changed = true;
7226 }
7227 }
7228 }
7229
7230 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7231 // section name so that llvm-lto will not complain about mismatching
7232 // module flags that is functionally the same.
7233 if (ID->getString() == "Objective-C Image Info Section") {
7234 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7235 SmallVector<StringRef, 4> ValueComp;
7236 Value->getString().split(ValueComp, " ");
7237 if (ValueComp.size() != 1) {
7238 std::string NewValue;
7239 for (auto &S : ValueComp)
7240 NewValue += S.str();
7241 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7242 MDString::get(M.getContext(), NewValue)};
7243 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7244 Changed = true;
7245 }
7246 }
7247 }
7248
7249 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7250 // If the higher bits are set, it adds new module flag for swift info.
7251 if (ID->getString() == "Objective-C Garbage Collection") {
7252 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7253 if (Md) {
7254 assert(Md->getValue() && "Expected non-empty metadata");
7255 auto Type = Md->getValue()->getType();
7256 if (Type == Int8Ty)
7257 continue;
7258 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7259 if ((Val & 0xff) != Val) {
7260 HasSwiftVersionFlag = true;
7261 SwiftABIVersion = (Val & 0xff00) >> 8;
7262 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7263 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7264 }
7265 Metadata *Ops[3] = {
7266 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7267 Op->getOperand(1),
7268 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7269 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7270 Changed = true;
7271 }
7272 }
7273
7274 if (ID->getString() == "amdgpu_code_object_version") {
7275 Metadata *Ops[3] = {
7276 Op->getOperand(0),
7277 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7278 Op->getOperand(2)};
7279 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7280 Changed = true;
7281 }
7282
7283 // clang/PowerPC used to use "float-abi" to describe the long double format;
7284 // it has been renamed to "long-double-type", with its values changed to the
7285 // corresponding IR floating-point type names.
7286 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7287 StringRef Format;
7288 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7289 Format = S->getString();
7290
7291 // The "float-abi" key is now reserved for the target-independent
7292 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7293 // (including unrecognized ones, which were never valid) to the default.
7294 if (!FloatABI::parseABIType(Format)) {
7295 LongDoubleFormat NewFormat =
7296 StringSwitch<LongDoubleFormat>(Format)
7297 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7298 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7299 .Default(LongDoubleFormat::PPCDoubleDouble);
7300 Metadata *Ops[3] = {
7301 Op->getOperand(0),
7302 MDString::get(M.getContext(), "long-double-type"),
7303 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7304 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7305 Changed = true;
7306 }
7307 }
7308 }
7309
7310 // "Objective-C Class Properties" is recently added for Objective-C. We
7311 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7312 // flag of value 0, so we can correclty downgrade this flag when trying to
7313 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7314 // this module flag.
7315 if (HasObjCFlag && !HasClassProperties) {
7316 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7317 (uint32_t)0);
7318 Changed = true;
7319 }
7320
7321 if (HasSwiftVersionFlag) {
7322 M.addModuleFlag(Module::Error, "Swift ABI Version",
7323 SwiftABIVersion);
7324 M.addModuleFlag(Module::Error, "Swift Major Version",
7325 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7326 M.addModuleFlag(Module::Error, "Swift Minor Version",
7327 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7328 Changed = true;
7329 }
7330
7331 return Changed;
7332}
7333
7334bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7335 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7336 // If this metadata has operands, we expect all of them to be either from
7337 // before or from after the format change handled here, so we can bail out
7338 // fast if the first (if any) operands is of the new format.
7339 auto MatchesVersion = [](const MDNode *Op) {
7340 return Op->getNumOperands() >= 3 &&
7341 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7342 cast<ConstantAsMetadata>(Op->getOperand(2))
7343 ->getType()
7344 ->isIntegerTy(64);
7345 };
7346
7347 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7348 MatchesVersion(CFIConsts->getOperand(0)))
7349 return false;
7350
7351 bool Changed = false;
7352 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7353 MDNode *Op = CFIConsts->getOperand(I);
7354 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7355 assert(Op->getNumOperands() >= 2 &&
7356 "Expected at least 2 operands - name and linkage type");
7357 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7358 StringRef Name = NameMD->getString();
7359 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7360 GlobalValue::dropLLVMManglingEscape(Name));
7361
7362 SmallVector<Metadata *, 4> Elts;
7363 Elts.push_back(Op->getOperand(0));
7364 Elts.push_back(Op->getOperand(1));
7365 Elts.push_back(ConstantAsMetadata::get(
7366 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7367
7368 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7369 Elts.push_back(Op->getOperand(J));
7370
7371 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7372 Changed = true;
7373 }
7374
7375 return Changed;
7376}
7377
7378void llvm::UpgradeSectionAttributes(Module &M) {
7379 auto TrimSpaces = [](StringRef Section) -> std::string {
7380 SmallVector<StringRef, 5> Components;
7381 Section.split(Components, ',');
7382
7383 SmallString<32> Buffer;
7384 raw_svector_ostream OS(Buffer);
7385
7386 for (auto Component : Components)
7387 OS << ',' << Component.trim();
7388
7389 return std::string(OS.str().substr(1));
7390 };
7391
7392 for (auto &GV : M.globals()) {
7393 if (!GV.hasSection())
7394 continue;
7395
7396 StringRef Section = GV.getSection();
7397
7398 if (!Section.starts_with("__DATA, __objc_catlist"))
7399 continue;
7400
7401 // __DATA, __objc_catlist, regular, no_dead_strip
7402 // __DATA,__objc_catlist,regular,no_dead_strip
7403 GV.setSection(TrimSpaces(Section));
7404 }
7405}
7406
7407namespace {
7408// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7409// callsites within a function that did not also have the strictfp attribute.
7410// Since 10.0, if strict FP semantics are needed within a function, the
7411// function must have the strictfp attribute and all calls within the function
7412// must also have the strictfp attribute. This latter restriction is
7413// necessary to prevent unwanted libcall simplification when a function is
7414// being cloned (such as for inlining).
7415//
7416// The "dangling" strictfp attribute usage was only used to prevent constant
7417// folding and other libcall simplification. The nobuiltin attribute on the
7418// callsite has the same effect.
7419struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7420 StrictFPUpgradeVisitor() = default;
7421
7422 void visitCallBase(CallBase &Call) {
7423 if (!Call.isStrictFP())
7424 return;
7425 if (isa<ConstrainedFPIntrinsic>(&Call))
7426 return;
7427 // If we get here, the caller doesn't have the strictfp attribute
7428 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7429 Call.removeFnAttr(Attribute::StrictFP);
7430 Call.addFnAttr(Attribute::NoBuiltin);
7431 }
7432};
7433
7434/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7435struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7436 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7437 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7438
7439 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7440 if (!RMW.isFloatingPointOperation())
7441 return;
7442
7443 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7444 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7445 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7446 RMW.setMetadata(LLVMContext::MD_atomic_ignore_denormal_mode, Empty);
7447 }
7448};
7449} // namespace
7450
7451void llvm::UpgradeFunctionAttributes(Function &F) {
7452 // If a function definition doesn't have the strictfp attribute,
7453 // convert any callsite strictfp attributes to nobuiltin.
7454 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7455 StrictFPUpgradeVisitor SFPV;
7456 SFPV.visit(F);
7457 }
7458
7459 // Remove all incompatibile attributes from function.
7460 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7461 F.getReturnType(), F.getAttributes().getRetAttrs()));
7462 for (auto &Arg : F.args())
7463 Arg.removeAttrs(
7464 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7465
7466 bool AddingAttrs = false, RemovingAttrs = false;
7467 AttrBuilder AttrsToAdd(F.getContext());
7468 AttributeMask AttrsToRemove;
7469
7470 // Older versions of LLVM treated an "implicit-section-name" attribute
7471 // similarly to directly setting the section on a Function.
7472 if (Attribute A = F.getFnAttribute("implicit-section-name");
7473 A.isValid() && A.isStringAttribute()) {
7474 F.setSection(A.getValueAsString());
7475 AttrsToRemove.addAttribute("implicit-section-name");
7476 RemovingAttrs = true;
7477 }
7478
7479 if (Attribute A = F.getFnAttribute("nooutline");
7480 A.isValid() && A.isStringAttribute()) {
7481 AttrsToRemove.addAttribute("nooutline");
7482 AttrsToAdd.addAttribute(Attribute::NoOutline);
7483 AddingAttrs = RemovingAttrs = true;
7484 }
7485
7486 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7487 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7488 AttrsToRemove.addAttribute("uniform-work-group-size");
7489 RemovingAttrs = true;
7490 if (A.getValueAsString() == "true") {
7491 AttrsToAdd.addAttribute("uniform-work-group-size");
7492 AddingAttrs = true;
7493 }
7494 }
7495
7496 if (!F.empty()) {
7497 // For some reason this is called twice, and the first time is before any
7498 // instructions are loaded into the body.
7499
7500 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7501 A.isValid()) {
7502
7503 if (A.getValueAsBool()) {
7504 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7505 Visitor.visit(F);
7506 }
7507
7508 // We will leave behind dead attribute uses on external declarations, but
7509 // clang never added these to declarations anyway.
7510 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7511 RemovingAttrs = true;
7512 }
7513 }
7514
7515 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7516 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7517
7518 bool HandleDenormalMode = false;
7519
7520 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7521 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7522 if (ParsedMode.isValid()) {
7523 DenormalFPMath = ParsedMode;
7524 AttrsToRemove.addAttribute("denormal-fp-math");
7525 AddingAttrs = RemovingAttrs = true;
7526 HandleDenormalMode = true;
7527 }
7528 }
7529
7530 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7531 Attr.isValid()) {
7532 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7533 if (ParsedMode.isValid()) {
7534 DenormalFPMathF32 = ParsedMode;
7535 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7536 AddingAttrs = RemovingAttrs = true;
7537 HandleDenormalMode = true;
7538 }
7539 }
7540
7541 if (HandleDenormalMode)
7542 AttrsToAdd.addDenormalFPEnvAttr(
7543 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7544
7545 if (RemovingAttrs)
7546 F.removeFnAttrs(AttrsToRemove);
7547
7548 if (AddingAttrs)
7549 F.addFnAttrs(AttrsToAdd);
7550}
7551
7552// Check if the function attribute is not present and set it.
7553static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7554 StringRef Value) {
7555 if (!F.hasFnAttribute(FnAttrName))
7556 F.addFnAttr(FnAttrName, Value);
7557}
7558
7559// Check if the function attribute is not present and set it if needed.
7560// If the attribute is "false" then removes it.
7561// If the attribute is "true" resets it to a valueless attribute.
7562static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7563 if (!F.hasFnAttribute(FnAttrName)) {
7564 if (Set)
7565 F.addFnAttr(FnAttrName);
7566 } else {
7567 auto A = F.getFnAttribute(FnAttrName);
7568 if ("false" == A.getValueAsString())
7569 F.removeFnAttr(FnAttrName);
7570 else if ("true" == A.getValueAsString()) {
7571 F.removeFnAttr(FnAttrName);
7572 F.addFnAttr(FnAttrName);
7573 }
7574 }
7575}
7576
7577void llvm::copyModuleAttrToFunctions(Module &M) {
7578 Triple T(M.getTargetTriple());
7579 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7580 return;
7581
7582 uint64_t BTEValue = 0;
7583 uint64_t BPPLRValue = 0;
7584 uint64_t GCSValue = 0;
7585 uint64_t SRAValue = 0;
7586 uint64_t SRAALLValue = 0;
7587 uint64_t SRABKeyValue = 0;
7588
7589 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7590 if (ModFlags) {
7591 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7592 MDNode *Op = ModFlags->getOperand(I);
7593 if (Op->getNumOperands() != 3)
7594 continue;
7595
7596 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7597 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7598 if (!ID || !CI)
7599 continue;
7600
7601 StringRef IDStr = ID->getString();
7602 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7603 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7604 : IDStr == "guarded-control-stack" ? &GCSValue
7605 : IDStr == "sign-return-address" ? &SRAValue
7606 : IDStr == "sign-return-address-all" ? &SRAALLValue
7607 : IDStr == "sign-return-address-with-bkey"
7608 ? &SRABKeyValue
7609 : nullptr;
7610 if (!ValPtr)
7611 continue;
7612
7613 *ValPtr = CI->getZExtValue();
7614 if (*ValPtr == 2)
7615 return;
7616 }
7617 }
7618
7619 bool BTE = BTEValue == 1;
7620 bool BPPLR = BPPLRValue == 1;
7621 bool GCS = GCSValue == 1;
7622 bool SRA = SRAValue == 1;
7623
7624 StringRef SignTypeValue = "non-leaf";
7625 if (SRA && SRAALLValue == 1)
7626 SignTypeValue = "all";
7627
7628 StringRef SignKeyValue = "a_key";
7629 if (SRA && SRABKeyValue == 1)
7630 SignKeyValue = "b_key";
7631
7632 for (Function &F : M.getFunctionList()) {
7633 if (F.isDeclaration())
7634 continue;
7635
7636 if (SRA) {
7637 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7638 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7639 } else {
7640 if (auto A = F.getFnAttribute("sign-return-address");
7641 A.isValid() && "none" == A.getValueAsString()) {
7642 F.removeFnAttr("sign-return-address");
7643 F.removeFnAttr("sign-return-address-key");
7644 }
7645 }
7646 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7647 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7648 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7649 }
7650
7651 if (BTE)
7652 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7653 if (BPPLR)
7654 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7655 if (GCS)
7656 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7657 if (SRA) {
7658 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7659 if (SRAALLValue == 1)
7660 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7661 if (SRABKeyValue == 1)
7662 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7663 }
7664}
7665
7666/// Return the replacement tags if \p T still uses a removed two-operand form.
7667static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7668 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7669 return nullptr;
7670 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7671 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7672}
7673
7674/// Build the single-operand node that replaces a boolean operand: nonzero
7675/// selects the enable tag, zero the disable tag.
7676static Metadata *makeBooleanLoopNode(LLVMContext &C,
7677 const BooleanLoopTags &Tags,
7678 const MDOperand &Op) {
7679 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7680 return MDTuple::get(C,
7681 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7682}
7683
7684static bool isOldLoopArgument(Metadata *MD) {
7685 auto *T = dyn_cast_or_null<MDTuple>(MD);
7686 if (!T)
7687 return false;
7688 if (T->getNumOperands() < 1)
7689 return false;
7690 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7691 if (!S)
7692 return false;
7693 if (S->getString().starts_with("llvm.vectorizer."))
7694 return true;
7695 return getOldBooleanLoopTags(T) != nullptr;
7696}
7697
7698static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7699 StringRef OldPrefix = "llvm.vectorizer.";
7700 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7701
7702 if (OldTag == "llvm.vectorizer.unroll")
7703 return MDString::get(C, "llvm.loop.interleave.count");
7704
7705 return MDString::get(
7706 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7707 .str());
7708}
7709
7710static Metadata *upgradeLoopArgument(Metadata *MD) {
7711 auto *T = dyn_cast_or_null<MDTuple>(MD);
7712 if (!T)
7713 return MD;
7714 if (T->getNumOperands() < 1)
7715 return MD;
7716 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7717 if (!OldTag)
7718 return MD;
7719
7720 LLVMContext &C = T->getContext();
7721
7722 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7723 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7724 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7725
7726 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7727 return MD;
7728
7729 // This has an old tag. Upgrade it.
7730 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7731
7732 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7733 // vectorize.enable/disable pair, not a two-operand enable node.
7734 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7735 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7736 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7737
7738 SmallVector<Metadata *, 8> Ops;
7739 Ops.reserve(T->getNumOperands());
7740 Ops.push_back(NewTag);
7741 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7742 Ops.push_back(T->getOperand(I));
7743
7744 return MDTuple::get(C, Ops);
7745}
7746
7747MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7748 auto *T = dyn_cast<MDTuple>(&N);
7749 if (!T)
7750 return &N;
7751
7752 if (none_of(T->operands(), isOldLoopArgument))
7753 return &N;
7754
7755 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7756 // any MDNode carrying those tags with more than one operand, so a leftover
7757 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7758 // In-place mutation is safe on distinct MDNodes.
7759 if (T->isDistinct()) {
7760 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7761 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7762 if (OpT && getOldBooleanLoopTags(OpT))
7763 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7764 }
7765 if (none_of(T->operands(), isOldLoopArgument))
7766 return &N;
7767 }
7768
7769 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7770 // attachment; the original distinct loop-ID is kept as the first operand.
7771 SmallVector<Metadata *, 8> Ops;
7772 Ops.reserve(T->getNumOperands());
7773 for (Metadata *MD : T->operands())
7774 Ops.push_back(upgradeLoopArgument(MD));
7775
7776 return MDTuple::get(T->getContext(), Ops);
7777}
7778
7779std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7780 Triple T(TT);
7781 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7782 // the address space of globals to 1. This does not apply to SPIRV Logical.
7783 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7784 !DL.contains("-G") && !DL.starts_with("G")) {
7785 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7786 }
7787
7788 if (T.isLoongArch64() || T.isRISCV64()) {
7789 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7790 auto I = DL.find("-n64-");
7791 if (I != StringRef::npos)
7792 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7793 return DL.str();
7794 }
7795
7796 // AMDGPU data layout upgrades.
7797 std::string Res = DL.str();
7798 if (T.isAMDGPU()) {
7799 // Define address spaces for constants.
7800 if (!DL.contains("-G") && !DL.starts_with("G"))
7801 Res.append(Res.empty() ? "G1" : "-G1");
7802
7803 // AMDGCN data layout upgrades.
7804 if (T.isAMDGCN()) {
7805
7806 // Add missing non-integral declarations.
7807 // This goes before adding new address spaces to prevent incoherent string
7808 // values.
7809 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7810 Res.append("-ni:7:8:9");
7811 // Update ni:7 to ni:7:8:9.
7812 if (DL.ends_with("ni:7"))
7813 Res.append(":8:9");
7814 if (DL.ends_with("ni:7:8"))
7815 Res.append(":9");
7816
7817 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7818 // resources) An empty data layout has already been upgraded to G1 by now.
7819 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7820 Res.append("-p7:160:256:256:32");
7821 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7822 Res.append("-p8:128:128:128:48");
7823 constexpr StringRef OldP8("-p8:128:128-");
7824 if (DL.contains(OldP8))
7825 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7826 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7827 Res.append("-p9:192:256:256:32");
7828
7829 // Add sizing for address space 10 through 15.
7830 // AS 10-14 are reserved and defaulted to 32:32
7831 // AS 15 is in use and is 32:32.
7832 for (StringRef AS : {"p10", "p11", "p12", "p13", "p14", "p15"}) {
7833 if (!DL.contains(("-" + AS).str()) && !DL.starts_with(AS))
7834 Res.append(("-" + AS + ":32:32").str());
7835 }
7836 }
7837
7838 // Upgrade the ELF mangling mode.
7839 if (!DL.contains("m:e"))
7840 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7841
7842 return Res;
7843 }
7844
7845 if (T.isSystemZ() && !DL.empty()) {
7846 // Make sure the stack alignment is present.
7847 if (!DL.contains("-S64"))
7848 return "E-S64" + DL.drop_front(1).str();
7849 return DL.str();
7850 }
7851
7852 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7853 // If the datalayout matches the expected format, add pointer size address
7854 // spaces to the datalayout.
7855 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7856 if (!DL.contains(AddrSpaces)) {
7857 SmallVector<StringRef, 4> Groups;
7858 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7859 if (R.match(Res, &Groups))
7860 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7861 }
7862 };
7863
7864 // AArch64 data layout upgrades.
7865 if (T.isAArch64()) {
7866 // Add "-Fn32"
7867 if (!DL.empty() && !DL.contains("-Fn32"))
7868 Res.append("-Fn32");
7869 AddPtr32Ptr64AddrSpaces();
7870 return Res;
7871 }
7872
7873 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7874 T.isWasm()) {
7875 // Mips64 with o32 ABI did not add "-i128:128".
7876 // Add "-i128:128"
7877 std::string I64 = "-i64:64";
7878 std::string I128 = "-i128:128";
7879 if (!StringRef(Res).contains(I128)) {
7880 size_t Pos = Res.find(I64);
7881 if (Pos != size_t(-1))
7882 Res.insert(Pos + I64.size(), I128);
7883 }
7884 }
7885
7886 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7887 size_t Pos = Res.find("-S128");
7888 if (Pos == StringRef::npos)
7889 Pos = Res.size();
7890 Res.insert(Pos, "-f64:32:64");
7891 }
7892
7893 if (!T.isX86())
7894 return Res;
7895
7896 AddPtr32Ptr64AddrSpaces();
7897
7898 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7899 // for i128 operations prior to this being reflected in the data layout, and
7900 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7901 // boundaries, so although this is a breaking change, the upgrade is expected
7902 // to fix more IR than it breaks.
7903 // Intel MCU is an exception and uses 4-byte-alignment.
7904 if (!T.isOSIAMCU()) {
7905 std::string I128 = "-i128:128";
7906 if (StringRef Ref = Res; !Ref.contains(I128)) {
7907 SmallVector<StringRef, 4> Groups;
7908 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7909 if (R.match(Res, &Groups))
7910 Res = (Groups[1] + I128 + Groups[3]).str();
7911 }
7912 }
7913
7914 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7915 // Raising the alignment is safe because Clang did not produce f80 values in
7916 // the MSVC environment before this upgrade was added.
7917 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7918 StringRef Ref = Res;
7919 auto I = Ref.find("-f80:32-");
7920 if (I != StringRef::npos)
7921 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7922 }
7923
7924 return Res;
7925}
7926
7927void llvm::UpgradeAttributes(AttrBuilder &B) {
7928 StringRef FramePointer;
7929 Attribute A = B.getAttribute("no-frame-pointer-elim");
7930 if (A.isValid()) {
7931 // The value can be "true" or "false".
7932 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7933 B.removeAttribute("no-frame-pointer-elim");
7934 }
7935 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7936 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7937 if (FramePointer != "all")
7938 FramePointer = "non-leaf";
7939 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7940 }
7941 if (!FramePointer.empty())
7942 B.addAttribute("frame-pointer", FramePointer);
7943
7944 A = B.getAttribute("null-pointer-is-valid");
7945 if (A.isValid()) {
7946 // The value can be "true" or "false".
7947 bool NullPointerIsValid = A.getValueAsString() == "true";
7948 B.removeAttribute("null-pointer-is-valid");
7949 if (NullPointerIsValid)
7950 B.addAttribute(Attribute::NullPointerIsValid);
7951 }
7952
7953 A = B.getAttribute("uniform-work-group-size");
7954 if (A.isValid()) {
7955 StringRef Val = A.getValueAsString();
7956 if (!Val.empty()) {
7957 bool IsTrue = Val == "true";
7958 B.removeAttribute("uniform-work-group-size");
7959 if (IsTrue)
7960 B.addAttribute("uniform-work-group-size");
7961 }
7962 }
7963}
7964
7965void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7966 // clang.arc.attachedcall bundles are now required to have an operand.
7967 // If they don't, it's okay to drop them entirely: when there is an operand,
7968 // the "attachedcall" is meaningful and required, but without an operand,
7969 // it's just a marker NOP. Dropping it merely prevents an optimization.
7970 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7971 return OBD.getTag() == "clang.arc.attachedcall" &&
7972 OBD.inputs().empty();
7973 });
7974}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:534
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:535
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:70
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2675
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1502
getFullArgCountForDefaultArgUpgrade
static unsigned getFullArgCountForDefaultArgUpgrade(Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
Definition AutoUpgrade.cpp:1669
G2S_ID
#define G2S_ID(ID_SUFFIX, NAME)
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7710
shouldUpgradeNVPTXMBarrierInitIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1551
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6802
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1705
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2515
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1367
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5662
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1334
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1347
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6908
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2776
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:3002
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1648
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2750
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1601
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7553
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1436
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6806
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5522
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7698
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5622
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6769
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2630
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2887
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1644
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2601
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5159
G2S_CTA_ID
#define G2S_CTA_ID(ID_SUFFIX, NAME)
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2988
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7667
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2927
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1416
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6715
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1558
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7676
G2S_CLUSTER_CASE
#define G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2549
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1564
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1214
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1391
getNVVMFAddUpgrade
static std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFAddUpgrade(StringRef Name)
Definition AutoUpgrade.cpp:1524
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7684
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5229
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5592
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5319
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2866
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5534
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5693
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2581
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2613
upgradeIntrinsicWithDefaultArgs
static bool upgradeIntrinsicWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1689
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2815
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
NVVM_TMA_G2S_MODES
#define NVVM_TMA_G2S_MODES(M)
Definition AutoUpgrade.cpp:1189
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2949
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2735
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3259
shouldUpgradeNVPTXBulkG2SClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic(Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
Definition AutoUpgrade.cpp:1290
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2981
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:3011
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5544
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7562
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2897
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2848
shouldUpgradeNVPTXBulkG2SCTAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1318
shouldUpgradeNVPTXTMAG2SCTAIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1257
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5605
G2S_CTA_CASE
#define G2S_CTA_CASE(ID_SUFFIX, NAME)
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:7063
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3467
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5640
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:540
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:453
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:466
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:774
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:934
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:786
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:790
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:811
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:850
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:794
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:800
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:854
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:842
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:802
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:822
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:818
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:846
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:981
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:106
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:226
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:511
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:359
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1595
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1715
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3109
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3392
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:548
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2463
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2395
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2449
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3481
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:140
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:218
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition DerivedTypes.h:204
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:602
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:594
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2903
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:357
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:555
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:353
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1783
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:703
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:352
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1516
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1081
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1245
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
llvm::MDString
A single uniqued string.
Definition Metadata.h:733
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:615
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:597
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1496
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1525
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:107
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1767
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1587
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1579
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1575
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1594
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1863
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1585
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:84
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:865
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2095
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:592
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:993
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:718
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1252
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:423
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:424
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:300
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:299
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:818
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:298
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:222
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:276
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:275
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:274
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5328
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:428
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:712
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:348
llvm::Value::hasName
bool hasName() const
Definition Value.h:263
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:485
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:536
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
Changed
Changed
Definition ObjCARCOpts.cpp:2468
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:35
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:167
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1451
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:802
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:650
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:637
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1423
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1416
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:720
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:662
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5754
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7378
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3250
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:192
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:126
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:117
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2463
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7747
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7927
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6628
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6867
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:7162
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7334
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7577
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7965
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6695
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7779
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2482
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:711
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6673
llvm::RecurKind::FAdd
@ FAdd
Sum of floats.
Definition IVDescriptors.h:50
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition FloatingPointMode.h:38
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
Definition FloatingPointMode.h:40
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
Definition FloatingPointMode.h:41
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
Definition FloatingPointMode.h:42
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
Definition FloatingPointMode.h:43
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6725
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7451
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6647
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6932
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7943
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0