LLVM 24.0.0git
AutoUpgrade.cpp
Go to the documentation of this file.
1//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the auto-upgrade helper functions.
10// This is where deprecated IR intrinsics and other IR features are updated to
11// current specifications.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/IR/AutoUpgrade.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/BinaryFormat/Dwarf.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Constants.h"
25#include "llvm/IR/DebugInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/Function.h"
29#include "llvm/IR/GlobalValue.h"
30#include "llvm/IR/IRBuilder.h"
31#include "llvm/IR/InstVisitor.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsAArch64.h"
36#include "llvm/IR/IntrinsicsAMDGPU.h"
37#include "llvm/IR/IntrinsicsARM.h"
38#include "llvm/IR/IntrinsicsNVPTX.h"
39#include "llvm/IR/IntrinsicsRISCV.h"
40#include "llvm/IR/IntrinsicsWebAssembly.h"
41#include "llvm/IR/IntrinsicsX86.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Metadata.h"
45#include "llvm/IR/Module.h"
46#include "llvm/IR/NVVMIntrinsicUtils.h"
47#include "llvm/IR/Value.h"
48#include "llvm/IR/Verifier.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CodeGen.h"
51#include "llvm/Support/CommandLine.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/NVPTXAddrSpace.h"
54#include "llvm/Support/NVVMAttributes.h"
55#include "llvm/Support/Regex.h"
56#include "llvm/Support/TimeProfiler.h"
57#include "llvm/TargetParser/Triple.h"
58#include <cstdint>
59#include <cstring>
60#include <numeric>
61
62using namespace llvm;
63
64static cl::opt<bool>
65 DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info",
66 cl::desc("Disable autoupgrade of debug info"));
67
68static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
69
70// Report a fatal error along with the
71// Call Instruction which caused the error
72[[noreturn]] static void reportFatalUsageErrorWithCI(StringRef reason,
73 CallBase *CI) {
74 CI->print(llvm::errs());
75 llvm::errs() << "\n";
76 reportFatalUsageError(reason);
77}
78
79// Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
80// changed their type from v4f32 to v2i64.
81static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID,
82 Function *&NewFn) {
83 // Check whether this is an old version of the function, which received
84 // v4f32 arguments.
85 Type *Arg0Type = F->getFunctionType()->getParamType(0);
86 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
87 return false;
88
89 // Yes, it's old, replace it with new version.
90 rename(F);
91 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
92 return true;
93}
94
95// Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
96// arguments have changed their type from i32 to i8.
97static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
98 Function *&NewFn) {
99 // Check that the last argument is an i32.
100 Type *LastArgType = F->getFunctionType()->getParamType(
101 F->getFunctionType()->getNumParams() - 1);
102 if (!LastArgType->isIntegerTy(32))
103 return false;
104
105 // Move this function aside and map down.
106 rename(F);
107 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
108 return true;
109}
110
111// Upgrade the declaration of fp compare intrinsics that change return type
112// from scalar to vXi1 mask.
113static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
114 Function *&NewFn) {
115 // Check if the return type is a vector.
116 if (F->getReturnType()->isVectorTy())
117 return false;
118
119 rename(F);
120 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
121 return true;
122}
123
124// Upgrade the declaration of multiply and add bytes intrinsics whose input
125// arguments' types have changed from vectors of i32 to vectors of i8
126static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID,
127 Function *&NewFn) {
128 // check if input argument type is a vector of i8
129 Type *Arg1Type = F->getFunctionType()->getParamType(1);
130 Type *Arg2Type = F->getFunctionType()->getParamType(2);
131 if (Arg1Type->isVectorTy() &&
132 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(8) &&
133 Arg2Type->isVectorTy() &&
134 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(8))
135 return false;
136
137 rename(F);
138 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
139 return true;
140}
141
142// Upgrade the declaration of multipy and add words intrinsics whose input
143// arguments' types have changed to vectors of i32 to vectors of i16
144static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID,
145 Function *&NewFn) {
146 // check if input argument type is a vector of i16
147 Type *Arg1Type = F->getFunctionType()->getParamType(1);
148 Type *Arg2Type = F->getFunctionType()->getParamType(2);
149 if (Arg1Type->isVectorTy() &&
150 cast<VectorType>(Arg1Type)->getElementType()->isIntegerTy(16) &&
151 Arg2Type->isVectorTy() &&
152 cast<VectorType>(Arg2Type)->getElementType()->isIntegerTy(16))
153 return false;
154
155 rename(F);
156 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
157 return true;
158}
159
160static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID,
161 Function *&NewFn) {
162 if (F->getReturnType()->getScalarType()->isBFloatTy())
163 return false;
164
165 rename(F);
166 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
167 return true;
168}
169
170static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID,
171 Function *&NewFn) {
172 if (F->getFunctionType()->getParamType(1)->getScalarType()->isBFloatTy())
173 return false;
174
175 rename(F);
176 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
177 return true;
178}
179
180static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
181 // All of the intrinsics matches below should be marked with which llvm
182 // version started autoupgrading them. At some point in the future we would
183 // like to use this information to remove upgrade code for some older
184 // intrinsics. It is currently undecided how we will determine that future
185 // point.
186 if (Name.consume_front("avx."))
187 return (Name.starts_with("blend.p") || // Added in 3.7
188 Name == "cvt.ps2.pd.256" || // Added in 3.9
189 Name == "cvtdq2.pd.256" || // Added in 3.9
190 Name == "cvtdq2.ps.256" || // Added in 7.0
191 Name.starts_with("movnt.") || // Added in 3.2
192 Name.starts_with("sqrt.p") || // Added in 7.0
193 Name.starts_with("storeu.") || // Added in 3.9
194 Name.starts_with("vbroadcast.s") || // Added in 3.5
195 Name.starts_with("vbroadcastf128") || // Added in 4.0
196 Name.starts_with("vextractf128.") || // Added in 3.7
197 Name.starts_with("vinsertf128.") || // Added in 3.7
198 Name.starts_with("vperm2f128.") || // Added in 6.0
199 Name.starts_with("vpermil.")); // Added in 3.1
200
201 if (Name.consume_front("avx2."))
202 return (Name == "movntdqa" || // Added in 5.0
203 Name.starts_with("pabs.") || // Added in 6.0
204 Name.starts_with("padds.") || // Added in 8.0
205 Name.starts_with("paddus.") || // Added in 8.0
206 Name.starts_with("pblendd.") || // Added in 3.7
207 Name == "pblendw" || // Added in 3.7
208 Name.starts_with("pbroadcast") || // Added in 3.8
209 Name.starts_with("pcmpeq.") || // Added in 3.1
210 Name.starts_with("pcmpgt.") || // Added in 3.1
211 Name.starts_with("pmax") || // Added in 3.9
212 Name.starts_with("pmin") || // Added in 3.9
213 Name.starts_with("pmovsx") || // Added in 3.9
214 Name.starts_with("pmovzx") || // Added in 3.9
215 Name == "pmul.dq" || // Added in 7.0
216 Name == "pmulu.dq" || // Added in 7.0
217 Name.starts_with("psll.dq") || // Added in 3.7
218 Name.starts_with("psrl.dq") || // Added in 3.7
219 Name.starts_with("psubs.") || // Added in 8.0
220 Name.starts_with("psubus.") || // Added in 8.0
221 Name.starts_with("vbroadcast") || // Added in 3.8
222 Name == "vbroadcasti128" || // Added in 3.7
223 Name == "vextracti128" || // Added in 3.7
224 Name == "vinserti128" || // Added in 3.7
225 Name == "vperm2i128"); // Added in 6.0
226
227 if (Name.consume_front("avx512.")) {
228 if (Name.consume_front("mask."))
229 // 'avx512.mask.*'
230 return (Name.starts_with("add.p") || // Added in 7.0. 128/256 in 4.0
231 Name.starts_with("and.") || // Added in 3.9
232 Name.starts_with("andn.") || // Added in 3.9
233 Name.starts_with("broadcast.s") || // Added in 3.9
234 Name.starts_with("broadcastf32x4.") || // Added in 6.0
235 Name.starts_with("broadcastf32x8.") || // Added in 6.0
236 Name.starts_with("broadcastf64x2.") || // Added in 6.0
237 Name.starts_with("broadcastf64x4.") || // Added in 6.0
238 Name.starts_with("broadcasti32x4.") || // Added in 6.0
239 Name.starts_with("broadcasti32x8.") || // Added in 6.0
240 Name.starts_with("broadcasti64x2.") || // Added in 6.0
241 Name.starts_with("broadcasti64x4.") || // Added in 6.0
242 Name.starts_with("cmp.b") || // Added in 5.0
243 Name.starts_with("cmp.d") || // Added in 5.0
244 Name.starts_with("cmp.q") || // Added in 5.0
245 Name.starts_with("cmp.w") || // Added in 5.0
246 Name.starts_with("compress.b") || // Added in 9.0
247 Name.starts_with("compress.d") || // Added in 9.0
248 Name.starts_with("compress.p") || // Added in 9.0
249 Name.starts_with("compress.q") || // Added in 9.0
250 Name.starts_with("compress.store.") || // Added in 7.0
251 Name.starts_with("compress.w") || // Added in 9.0
252 Name.starts_with("conflict.") || // Added in 9.0
253 Name.starts_with("cvtdq2pd.") || // Added in 4.0
254 Name.starts_with("cvtdq2ps.") || // Added in 7.0 updated 9.0
255 Name == "cvtpd2dq.256" || // Added in 7.0
256 Name == "cvtpd2ps.256" || // Added in 7.0
257 Name == "cvtps2pd.128" || // Added in 7.0
258 Name == "cvtps2pd.256" || // Added in 7.0
259 Name.starts_with("cvtqq2pd.") || // Added in 7.0 updated 9.0
260 Name == "cvtqq2ps.256" || // Added in 9.0
261 Name == "cvtqq2ps.512" || // Added in 9.0
262 Name == "cvttpd2dq.256" || // Added in 7.0
263 Name == "cvttps2dq.128" || // Added in 7.0
264 Name == "cvttps2dq.256" || // Added in 7.0
265 Name.starts_with("cvtudq2pd.") || // Added in 4.0
266 Name.starts_with("cvtudq2ps.") || // Added in 7.0 updated 9.0
267 Name.starts_with("cvtuqq2pd.") || // Added in 7.0 updated 9.0
268 Name == "cvtuqq2ps.256" || // Added in 9.0
269 Name == "cvtuqq2ps.512" || // Added in 9.0
270 Name.starts_with("dbpsadbw.") || // Added in 7.0
271 Name.starts_with("div.p") || // Added in 7.0. 128/256 in 4.0
272 Name.starts_with("expand.b") || // Added in 9.0
273 Name.starts_with("expand.d") || // Added in 9.0
274 Name.starts_with("expand.load.") || // Added in 7.0
275 Name.starts_with("expand.p") || // Added in 9.0
276 Name.starts_with("expand.q") || // Added in 9.0
277 Name.starts_with("expand.w") || // Added in 9.0
278 Name.starts_with("fpclass.p") || // Added in 7.0
279 Name.starts_with("insert") || // Added in 4.0
280 Name.starts_with("load.") || // Added in 3.9
281 Name.starts_with("loadu.") || // Added in 3.9
282 Name.starts_with("lzcnt.") || // Added in 5.0
283 Name.starts_with("max.p") || // Added in 7.0. 128/256 in 5.0
284 Name.starts_with("min.p") || // Added in 7.0. 128/256 in 5.0
285 Name.starts_with("movddup") || // Added in 3.9
286 Name.starts_with("move.s") || // Added in 4.0
287 Name.starts_with("movshdup") || // Added in 3.9
288 Name.starts_with("movsldup") || // Added in 3.9
289 Name.starts_with("mul.p") || // Added in 7.0. 128/256 in 4.0
290 Name.starts_with("or.") || // Added in 3.9
291 Name.starts_with("pabs.") || // Added in 6.0
292 Name.starts_with("packssdw.") || // Added in 5.0
293 Name.starts_with("packsswb.") || // Added in 5.0
294 Name.starts_with("packusdw.") || // Added in 5.0
295 Name.starts_with("packuswb.") || // Added in 5.0
296 Name.starts_with("padd.") || // Added in 4.0
297 Name.starts_with("padds.") || // Added in 8.0
298 Name.starts_with("paddus.") || // Added in 8.0
299 Name.starts_with("palignr.") || // Added in 3.9
300 Name.starts_with("pand.") || // Added in 3.9
301 Name.starts_with("pandn.") || // Added in 3.9
302 Name.starts_with("pavg") || // Added in 6.0
303 Name.starts_with("pbroadcast") || // Added in 6.0
304 Name.starts_with("pcmpeq.") || // Added in 3.9
305 Name.starts_with("pcmpgt.") || // Added in 3.9
306 Name.starts_with("perm.df.") || // Added in 3.9
307 Name.starts_with("perm.di.") || // Added in 3.9
308 Name.starts_with("permvar.") || // Added in 7.0
309 Name.starts_with("pmaddubs.w.") || // Added in 7.0
310 Name.starts_with("pmaddw.d.") || // Added in 7.0
311 Name.starts_with("pmax") || // Added in 4.0
312 Name.starts_with("pmin") || // Added in 4.0
313 Name == "pmov.qd.256" || // Added in 9.0
314 Name == "pmov.qd.512" || // Added in 9.0
315 Name == "pmov.wb.256" || // Added in 9.0
316 Name == "pmov.wb.512" || // Added in 9.0
317 Name.starts_with("pmovsx") || // Added in 4.0
318 Name.starts_with("pmovzx") || // Added in 4.0
319 Name.starts_with("pmul.dq.") || // Added in 4.0
320 Name.starts_with("pmul.hr.sw.") || // Added in 7.0
321 Name.starts_with("pmulh.w.") || // Added in 7.0
322 Name.starts_with("pmulhu.w.") || // Added in 7.0
323 Name.starts_with("pmull.") || // Added in 4.0
324 Name.starts_with("pmultishift.qb.") || // Added in 8.0
325 Name.starts_with("pmulu.dq.") || // Added in 4.0
326 Name.starts_with("por.") || // Added in 3.9
327 Name.starts_with("prol.") || // Added in 8.0
328 Name.starts_with("prolv.") || // Added in 8.0
329 Name.starts_with("pror.") || // Added in 8.0
330 Name.starts_with("prorv.") || // Added in 8.0
331 Name.starts_with("pshuf.b.") || // Added in 4.0
332 Name.starts_with("pshuf.d.") || // Added in 3.9
333 Name.starts_with("pshufh.w.") || // Added in 3.9
334 Name.starts_with("pshufl.w.") || // Added in 3.9
335 Name.starts_with("psll.d") || // Added in 4.0
336 Name.starts_with("psll.q") || // Added in 4.0
337 Name.starts_with("psll.w") || // Added in 4.0
338 Name.starts_with("pslli") || // Added in 4.0
339 Name.starts_with("psllv") || // Added in 4.0
340 Name.starts_with("psra.d") || // Added in 4.0
341 Name.starts_with("psra.q") || // Added in 4.0
342 Name.starts_with("psra.w") || // Added in 4.0
343 Name.starts_with("psrai") || // Added in 4.0
344 Name.starts_with("psrav") || // Added in 4.0
345 Name.starts_with("psrl.d") || // Added in 4.0
346 Name.starts_with("psrl.q") || // Added in 4.0
347 Name.starts_with("psrl.w") || // Added in 4.0
348 Name.starts_with("psrli") || // Added in 4.0
349 Name.starts_with("psrlv") || // Added in 4.0
350 Name.starts_with("psub.") || // Added in 4.0
351 Name.starts_with("psubs.") || // Added in 8.0
352 Name.starts_with("psubus.") || // Added in 8.0
353 Name.starts_with("pternlog.") || // Added in 7.0
354 Name.starts_with("punpckh") || // Added in 3.9
355 Name.starts_with("punpckl") || // Added in 3.9
356 Name.starts_with("pxor.") || // Added in 3.9
357 Name.starts_with("shuf.f") || // Added in 6.0
358 Name.starts_with("shuf.i") || // Added in 6.0
359 Name.starts_with("shuf.p") || // Added in 4.0
360 Name.starts_with("sqrt.p") || // Added in 7.0
361 Name.starts_with("store.b.") || // Added in 3.9
362 Name.starts_with("store.d.") || // Added in 3.9
363 Name.starts_with("store.p") || // Added in 3.9
364 Name.starts_with("store.q.") || // Added in 3.9
365 Name.starts_with("store.w.") || // Added in 3.9
366 Name == "store.ss" || // Added in 7.0
367 Name.starts_with("storeu.") || // Added in 3.9
368 Name.starts_with("sub.p") || // Added in 7.0. 128/256 in 4.0
369 Name.starts_with("ucmp.") || // Added in 5.0
370 Name.starts_with("unpckh.") || // Added in 3.9
371 Name.starts_with("unpckl.") || // Added in 3.9
372 Name.starts_with("valign.") || // Added in 4.0
373 Name == "vcvtph2ps.128" || // Added in 11.0
374 Name == "vcvtph2ps.256" || // Added in 11.0
375 Name.starts_with("vextract") || // Added in 4.0
376 Name.starts_with("vfmadd.") || // Added in 7.0
377 Name.starts_with("vfmaddsub.") || // Added in 7.0
378 Name.starts_with("vfnmadd.") || // Added in 7.0
379 Name.starts_with("vfnmsub.") || // Added in 7.0
380 Name.starts_with("vpdpbusd.") || // Added in 7.0
381 Name.starts_with("vpdpbusds.") || // Added in 7.0
382 Name.starts_with("vpdpwssd.") || // Added in 7.0
383 Name.starts_with("vpdpwssds.") || // Added in 7.0
384 Name.starts_with("vpermi2var.") || // Added in 7.0
385 Name.starts_with("vpermil.p") || // Added in 3.9
386 Name.starts_with("vpermilvar.") || // Added in 4.0
387 Name.starts_with("vpermt2var.") || // Added in 7.0
388 Name.starts_with("vpmadd52") || // Added in 7.0
389 Name.starts_with("vpshld.") || // Added in 7.0
390 Name.starts_with("vpshldv.") || // Added in 8.0
391 Name.starts_with("vpshrd.") || // Added in 7.0
392 Name.starts_with("vpshrdv.") || // Added in 8.0
393 Name.starts_with("vpshufbitqmb.") || // Added in 8.0
394 Name.starts_with("xor.")); // Added in 3.9
395
396 if (Name.consume_front("mask3."))
397 // 'avx512.mask3.*'
398 return (Name.starts_with("vfmadd.") || // Added in 7.0
399 Name.starts_with("vfmaddsub.") || // Added in 7.0
400 Name.starts_with("vfmsub.") || // Added in 7.0
401 Name.starts_with("vfmsubadd.") || // Added in 7.0
402 Name.starts_with("vfnmsub.")); // Added in 7.0
403
404 if (Name.consume_front("maskz."))
405 // 'avx512.maskz.*'
406 return (Name.starts_with("pternlog.") || // Added in 7.0
407 Name.starts_with("vfmadd.") || // Added in 7.0
408 Name.starts_with("vfmaddsub.") || // Added in 7.0
409 Name.starts_with("vpdpbusd.") || // Added in 7.0
410 Name.starts_with("vpdpbusds.") || // Added in 7.0
411 Name.starts_with("vpdpwssd.") || // Added in 7.0
412 Name.starts_with("vpdpwssds.") || // Added in 7.0
413 Name.starts_with("vpermt2var.") || // Added in 7.0
414 Name.starts_with("vpmadd52") || // Added in 7.0
415 Name.starts_with("vpshldv.") || // Added in 8.0
416 Name.starts_with("vpshrdv.")); // Added in 8.0
417
418 // 'avx512.*'
419 return (Name == "movntdqa" || // Added in 5.0
420 Name == "pmul.dq.512" || // Added in 7.0
421 Name == "pmulu.dq.512" || // Added in 7.0
422 Name.starts_with("broadcastm") || // Added in 6.0
423 Name.starts_with("cmp.p") || // Added in 12.0
424 Name.starts_with("cvtb2mask.") || // Added in 7.0
425 Name.starts_with("cvtd2mask.") || // Added in 7.0
426 Name.starts_with("cvtmask2") || // Added in 5.0
427 Name.starts_with("cvtq2mask.") || // Added in 7.0
428 Name == "cvtusi2sd" || // Added in 7.0
429 Name.starts_with("cvtw2mask.") || // Added in 7.0
430 Name == "kand.w" || // Added in 7.0
431 Name == "kandn.w" || // Added in 7.0
432 Name == "knot.w" || // Added in 7.0
433 Name == "kor.w" || // Added in 7.0
434 Name == "kortestc.w" || // Added in 7.0
435 Name == "kortestz.w" || // Added in 7.0
436 Name.starts_with("kunpck") || // added in 6.0
437 Name == "kxnor.w" || // Added in 7.0
438 Name == "kxor.w" || // Added in 7.0
439 Name.starts_with("padds.") || // Added in 8.0
440 Name.starts_with("pbroadcast") || // Added in 3.9
441 Name.starts_with("prol") || // Added in 8.0
442 Name.starts_with("pror") || // Added in 8.0
443 Name.starts_with("psll.dq") || // Added in 3.9
444 Name.starts_with("psrl.dq") || // Added in 3.9
445 Name.starts_with("psubs.") || // Added in 8.0
446 Name.starts_with("ptestm") || // Added in 6.0
447 Name.starts_with("ptestnm") || // Added in 6.0
448 Name.starts_with("storent.") || // Added in 3.9
449 Name.starts_with("vbroadcast.s") || // Added in 7.0
450 Name.starts_with("vpshld.") || // Added in 8.0
451 Name.starts_with("vpshrd.")); // Added in 8.0
452 }
453
454 if (Name.consume_front("fma."))
455 return (Name.starts_with("vfmadd.") || // Added in 7.0
456 Name.starts_with("vfmsub.") || // Added in 7.0
457 Name.starts_with("vfmsubadd.") || // Added in 7.0
458 Name.starts_with("vfnmadd.") || // Added in 7.0
459 Name.starts_with("vfnmsub.")); // Added in 7.0
460
461 if (Name.consume_front("fma4."))
462 return Name.starts_with("vfmadd.s"); // Added in 7.0
463
464 if (Name.consume_front("sse."))
465 return (Name == "add.ss" || // Added in 4.0
466 Name == "cvtsi2ss" || // Added in 7.0
467 Name == "cvtsi642ss" || // Added in 7.0
468 Name == "div.ss" || // Added in 4.0
469 Name == "mul.ss" || // Added in 4.0
470 Name.starts_with("sqrt.p") || // Added in 7.0
471 Name == "sqrt.ss" || // Added in 7.0
472 Name.starts_with("storeu.") || // Added in 3.9
473 Name == "sub.ss"); // Added in 4.0
474
475 if (Name.consume_front("sse2."))
476 return (Name == "add.sd" || // Added in 4.0
477 Name == "cvtdq2pd" || // Added in 3.9
478 Name == "cvtdq2ps" || // Added in 7.0
479 Name == "cvtps2pd" || // Added in 3.9
480 Name == "cvtsi2sd" || // Added in 7.0
481 Name == "cvtsi642sd" || // Added in 7.0
482 Name == "cvtss2sd" || // Added in 7.0
483 Name == "div.sd" || // Added in 4.0
484 Name == "mul.sd" || // Added in 4.0
485 Name.starts_with("padds.") || // Added in 8.0
486 Name.starts_with("paddus.") || // Added in 8.0
487 Name.starts_with("pcmpeq.") || // Added in 3.1
488 Name.starts_with("pcmpgt.") || // Added in 3.1
489 Name == "pmaxs.w" || // Added in 3.9
490 Name == "pmaxu.b" || // Added in 3.9
491 Name == "pmins.w" || // Added in 3.9
492 Name == "pminu.b" || // Added in 3.9
493 Name == "pmulu.dq" || // Added in 7.0
494 Name.starts_with("pshuf") || // Added in 3.9
495 Name.starts_with("psll.dq") || // Added in 3.7
496 Name.starts_with("psrl.dq") || // Added in 3.7
497 Name.starts_with("psubs.") || // Added in 8.0
498 Name.starts_with("psubus.") || // Added in 8.0
499 Name.starts_with("sqrt.p") || // Added in 7.0
500 Name == "sqrt.sd" || // Added in 7.0
501 Name == "storel.dq" || // Added in 3.9
502 Name.starts_with("storeu.") || // Added in 3.9
503 Name == "sub.sd"); // Added in 4.0
504
505 if (Name.consume_front("sse41."))
506 return (Name.starts_with("blendp") || // Added in 3.7
507 Name == "movntdqa" || // Added in 5.0
508 Name == "pblendw" || // Added in 3.7
509 Name == "pmaxsb" || // Added in 3.9
510 Name == "pmaxsd" || // Added in 3.9
511 Name == "pmaxud" || // Added in 3.9
512 Name == "pmaxuw" || // Added in 3.9
513 Name == "pminsb" || // Added in 3.9
514 Name == "pminsd" || // Added in 3.9
515 Name == "pminud" || // Added in 3.9
516 Name == "pminuw" || // Added in 3.9
517 Name.starts_with("pmovsx") || // Added in 3.8
518 Name.starts_with("pmovzx") || // Added in 3.9
519 Name == "pmuldq"); // Added in 7.0
520
521 if (Name.consume_front("sse42."))
522 return Name == "crc32.64.8"; // Added in 3.4
523
524 if (Name.consume_front("sse4a."))
525 return Name.starts_with("movnt."); // Added in 3.9
526
527 if (Name.consume_front("ssse3."))
528 return (Name == "pabs.b.128" || // Added in 6.0
529 Name == "pabs.d.128" || // Added in 6.0
530 Name == "pabs.w.128"); // Added in 6.0
531
532 if (Name.consume_front("xop."))
533 return (Name == "vpcmov" || // Added in 3.8
534 Name == "vpcmov.256" || // Added in 5.0
535 Name.starts_with("vpcom") || // Added in 3.2, Updated in 9.0
536 Name.starts_with("vprot")); // Added in 8.0
537
538 if (Name.consume_front("bmi."))
539 return (Name.starts_with("pdep.") || // Added in 23.0
540 Name.starts_with("pext.")); // Added in 23.0
541
542 return (Name == "addcarry.u32" || // Added in 8.0
543 Name == "addcarry.u64" || // Added in 8.0
544 Name == "addcarryx.u32" || // Added in 8.0
545 Name == "addcarryx.u64" || // Added in 8.0
546 Name == "subborrow.u32" || // Added in 8.0
547 Name == "subborrow.u64" || // Added in 8.0
548 Name.starts_with("vcvtph2ps.")); // Added in 11.0
549}
550
551static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name,
552 Function *&NewFn) {
553 // Only handle intrinsics that start with "x86.".
554 if (!Name.consume_front("x86."))
555 return false;
556
557 if (shouldUpgradeX86Intrinsic(F, Name)) {
558 NewFn = nullptr;
559 return true;
560 }
561
562 if (Name == "rdtscp") { // Added in 8.0
563 // If this intrinsic has 0 operands, it's the new version.
564 if (F->getFunctionType()->getNumParams() == 0)
565 return false;
566
567 rename(F);
568 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
569 Intrinsic::x86_rdtscp);
570 return true;
571 }
572
573 Intrinsic::ID ID;
574
575 // SSE4.1 ptest functions may have an old signature.
576 if (Name.consume_front("sse41.ptest")) { // Added in 3.2
577 ID = StringSwitch<Intrinsic::ID>(Name)
578 .Case("c", Intrinsic::x86_sse41_ptestc)
579 .Case("z", Intrinsic::x86_sse41_ptestz)
580 .Case("nzc", Intrinsic::x86_sse41_ptestnzc)
581 .Default(Intrinsic::not_intrinsic);
582 if (ID != Intrinsic::not_intrinsic)
583 return upgradePTESTIntrinsic(F, ID, NewFn);
584
585 return false;
586 }
587
588 // Several blend and other instructions with masks used the wrong number of
589 // bits.
590
591 // Added in 3.6
592 ID = StringSwitch<Intrinsic::ID>(Name)
593 .Case("sse41.insertps", Intrinsic::x86_sse41_insertps)
594 .Case("sse41.dppd", Intrinsic::x86_sse41_dppd)
595 .Case("sse41.dpps", Intrinsic::x86_sse41_dpps)
596 .Case("sse41.mpsadbw", Intrinsic::x86_sse41_mpsadbw)
597 .Case("avx.dp.ps.256", Intrinsic::x86_avx_dp_ps_256)
598 .Case("avx2.mpsadbw", Intrinsic::x86_avx2_mpsadbw)
599 .Default(Intrinsic::not_intrinsic);
600 if (ID != Intrinsic::not_intrinsic)
601 return upgradeX86IntrinsicsWith8BitMask(F, ID, NewFn);
602
603 if (Name.consume_front("avx512.")) {
604 if (Name.consume_front("mask.cmp.")) {
605 // Added in 7.0
606 ID = StringSwitch<Intrinsic::ID>(Name)
607 .Case("pd.128", Intrinsic::x86_avx512_mask_cmp_pd_128)
608 .Case("pd.256", Intrinsic::x86_avx512_mask_cmp_pd_256)
609 .Case("pd.512", Intrinsic::x86_avx512_mask_cmp_pd_512)
610 .Case("ps.128", Intrinsic::x86_avx512_mask_cmp_ps_128)
611 .Case("ps.256", Intrinsic::x86_avx512_mask_cmp_ps_256)
612 .Case("ps.512", Intrinsic::x86_avx512_mask_cmp_ps_512)
613 .Default(Intrinsic::not_intrinsic);
614 if (ID != Intrinsic::not_intrinsic)
615 return upgradeX86MaskedFPCompare(F, ID, NewFn);
616 } else if (Name.starts_with("vpdpbusd.") ||
617 Name.starts_with("vpdpbusds.")) {
618 // Added in 21.1
619 ID = StringSwitch<Intrinsic::ID>(Name)
620 .Case("vpdpbusd.128", Intrinsic::x86_avx512_vpdpbusd_128)
621 .Case("vpdpbusd.256", Intrinsic::x86_avx512_vpdpbusd_256)
622 .Case("vpdpbusd.512", Intrinsic::x86_avx512_vpdpbusd_512)
623 .Case("vpdpbusds.128", Intrinsic::x86_avx512_vpdpbusds_128)
624 .Case("vpdpbusds.256", Intrinsic::x86_avx512_vpdpbusds_256)
625 .Case("vpdpbusds.512", Intrinsic::x86_avx512_vpdpbusds_512)
626 .Default(Intrinsic::not_intrinsic);
627 if (ID != Intrinsic::not_intrinsic)
628 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
629 } else if (Name.starts_with("vpdpwssd.") ||
630 Name.starts_with("vpdpwssds.")) {
631 // Added in 21.1
632 ID = StringSwitch<Intrinsic::ID>(Name)
633 .Case("vpdpwssd.128", Intrinsic::x86_avx512_vpdpwssd_128)
634 .Case("vpdpwssd.256", Intrinsic::x86_avx512_vpdpwssd_256)
635 .Case("vpdpwssd.512", Intrinsic::x86_avx512_vpdpwssd_512)
636 .Case("vpdpwssds.128", Intrinsic::x86_avx512_vpdpwssds_128)
637 .Case("vpdpwssds.256", Intrinsic::x86_avx512_vpdpwssds_256)
638 .Case("vpdpwssds.512", Intrinsic::x86_avx512_vpdpwssds_512)
639 .Default(Intrinsic::not_intrinsic);
640 if (ID != Intrinsic::not_intrinsic)
641 return upgradeX86MultiplyAddWords(F, ID, NewFn);
642 }
643 return false; // No other 'x86.avx512.*'.
644 }
645
646 if (Name.consume_front("avx2.")) {
647 if (Name.consume_front("vpdpb")) {
648 // Added in 21.1
649 ID = StringSwitch<Intrinsic::ID>(Name)
650 .Case("ssd.128", Intrinsic::x86_avx2_vpdpbssd_128)
651 .Case("ssd.256", Intrinsic::x86_avx2_vpdpbssd_256)
652 .Case("ssds.128", Intrinsic::x86_avx2_vpdpbssds_128)
653 .Case("ssds.256", Intrinsic::x86_avx2_vpdpbssds_256)
654 .Case("sud.128", Intrinsic::x86_avx2_vpdpbsud_128)
655 .Case("sud.256", Intrinsic::x86_avx2_vpdpbsud_256)
656 .Case("suds.128", Intrinsic::x86_avx2_vpdpbsuds_128)
657 .Case("suds.256", Intrinsic::x86_avx2_vpdpbsuds_256)
658 .Case("uud.128", Intrinsic::x86_avx2_vpdpbuud_128)
659 .Case("uud.256", Intrinsic::x86_avx2_vpdpbuud_256)
660 .Case("uuds.128", Intrinsic::x86_avx2_vpdpbuuds_128)
661 .Case("uuds.256", Intrinsic::x86_avx2_vpdpbuuds_256)
662 .Default(Intrinsic::not_intrinsic);
663 if (ID != Intrinsic::not_intrinsic)
664 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
665 } else if (Name.consume_front("vpdpw")) {
666 // Added in 21.1
667 ID = StringSwitch<Intrinsic::ID>(Name)
668 .Case("sud.128", Intrinsic::x86_avx2_vpdpwsud_128)
669 .Case("sud.256", Intrinsic::x86_avx2_vpdpwsud_256)
670 .Case("suds.128", Intrinsic::x86_avx2_vpdpwsuds_128)
671 .Case("suds.256", Intrinsic::x86_avx2_vpdpwsuds_256)
672 .Case("usd.128", Intrinsic::x86_avx2_vpdpwusd_128)
673 .Case("usd.256", Intrinsic::x86_avx2_vpdpwusd_256)
674 .Case("usds.128", Intrinsic::x86_avx2_vpdpwusds_128)
675 .Case("usds.256", Intrinsic::x86_avx2_vpdpwusds_256)
676 .Case("uud.128", Intrinsic::x86_avx2_vpdpwuud_128)
677 .Case("uud.256", Intrinsic::x86_avx2_vpdpwuud_256)
678 .Case("uuds.128", Intrinsic::x86_avx2_vpdpwuuds_128)
679 .Case("uuds.256", Intrinsic::x86_avx2_vpdpwuuds_256)
680 .Default(Intrinsic::not_intrinsic);
681 if (ID != Intrinsic::not_intrinsic)
682 return upgradeX86MultiplyAddWords(F, ID, NewFn);
683 }
684 return false; // No other 'x86.avx2.*'
685 }
686
687 if (Name.consume_front("avx10.")) {
688 if (Name.consume_front("vpdpb")) {
689 // Added in 21.1
690 ID = StringSwitch<Intrinsic::ID>(Name)
691 .Case("ssd.512", Intrinsic::x86_avx10_vpdpbssd_512)
692 .Case("ssds.512", Intrinsic::x86_avx10_vpdpbssds_512)
693 .Case("sud.512", Intrinsic::x86_avx10_vpdpbsud_512)
694 .Case("suds.512", Intrinsic::x86_avx10_vpdpbsuds_512)
695 .Case("uud.512", Intrinsic::x86_avx10_vpdpbuud_512)
696 .Case("uuds.512", Intrinsic::x86_avx10_vpdpbuuds_512)
697 .Default(Intrinsic::not_intrinsic);
698 if (ID != Intrinsic::not_intrinsic)
699 return upgradeX86MultiplyAddBytes(F, ID, NewFn);
700 } else if (Name.consume_front("vpdpw")) {
701 ID = StringSwitch<Intrinsic::ID>(Name)
702 .Case("sud.512", Intrinsic::x86_avx10_vpdpwsud_512)
703 .Case("suds.512", Intrinsic::x86_avx10_vpdpwsuds_512)
704 .Case("usd.512", Intrinsic::x86_avx10_vpdpwusd_512)
705 .Case("usds.512", Intrinsic::x86_avx10_vpdpwusds_512)
706 .Case("uud.512", Intrinsic::x86_avx10_vpdpwuud_512)
707 .Case("uuds.512", Intrinsic::x86_avx10_vpdpwuuds_512)
708 .Default(Intrinsic::not_intrinsic);
709 if (ID != Intrinsic::not_intrinsic)
710 return upgradeX86MultiplyAddWords(F, ID, NewFn);
711 }
712 return false; // No other 'x86.avx10.*'
713 }
714
715 if (Name.consume_front("avx512bf16.")) {
716 // Added in 9.0
717 ID = StringSwitch<Intrinsic::ID>(Name)
718 .Case("cvtne2ps2bf16.128",
719 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128)
720 .Case("cvtne2ps2bf16.256",
721 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256)
722 .Case("cvtne2ps2bf16.512",
723 Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512)
724 .Case("mask.cvtneps2bf16.128",
725 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
726 .Case("cvtneps2bf16.256",
727 Intrinsic::x86_avx512bf16_cvtneps2bf16_256)
728 .Case("cvtneps2bf16.512",
729 Intrinsic::x86_avx512bf16_cvtneps2bf16_512)
730 .Default(Intrinsic::not_intrinsic);
731 if (ID != Intrinsic::not_intrinsic)
732 return upgradeX86BF16Intrinsic(F, ID, NewFn);
733
734 // Added in 9.0
735 ID = StringSwitch<Intrinsic::ID>(Name)
736 .Case("dpbf16ps.128", Intrinsic::x86_avx512bf16_dpbf16ps_128)
737 .Case("dpbf16ps.256", Intrinsic::x86_avx512bf16_dpbf16ps_256)
738 .Case("dpbf16ps.512", Intrinsic::x86_avx512bf16_dpbf16ps_512)
739 .Default(Intrinsic::not_intrinsic);
740 if (ID != Intrinsic::not_intrinsic)
741 return upgradeX86BF16DPIntrinsic(F, ID, NewFn);
742 return false; // No other 'x86.avx512bf16.*'.
743 }
744
745 if (Name.consume_front("xop.")) {
746 Intrinsic::ID ID = Intrinsic::not_intrinsic;
747 if (Name.starts_with("vpermil2")) { // Added in 3.9
748 // Upgrade any XOP PERMIL2 index operand still using a float/double
749 // vector.
750 auto Idx = F->getFunctionType()->getParamType(2);
751 if (Idx->isFPOrFPVectorTy()) {
752 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
753 unsigned EltSize = Idx->getScalarSizeInBits();
754 if (EltSize == 64 && IdxSize == 128)
755 ID = Intrinsic::x86_xop_vpermil2pd;
756 else if (EltSize == 32 && IdxSize == 128)
757 ID = Intrinsic::x86_xop_vpermil2ps;
758 else if (EltSize == 64 && IdxSize == 256)
759 ID = Intrinsic::x86_xop_vpermil2pd_256;
760 else
761 ID = Intrinsic::x86_xop_vpermil2ps_256;
762 }
763 } else if (F->arg_size() == 2)
764 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
765 ID = StringSwitch<Intrinsic::ID>(Name)
766 .Case("vfrcz.ss", Intrinsic::x86_xop_vfrcz_ss)
767 .Case("vfrcz.sd", Intrinsic::x86_xop_vfrcz_sd)
768 .Default(Intrinsic::not_intrinsic);
769
770 if (ID != Intrinsic::not_intrinsic) {
771 rename(F);
772 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
773 return true;
774 }
775 return false; // No other 'x86.xop.*'
776 }
777
778 if (Name == "seh.recoverfp") {
779 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
780 Intrinsic::eh_recoverfp);
781 return true;
782 }
783
784 return false;
785}
786
787// Upgrade ARM (IsArm) or Aarch64 (!IsArm) intrinsic fns. Return true iff so.
788// IsArm: 'arm.*', !IsArm: 'aarch64.*'.
789static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
790 StringRef Name,
791 Function *&NewFn) {
792 if (Name.starts_with("rbit")) {
793 // '(arm|aarch64).rbit'.
794 NewFn = Intrinsic::getOrInsertDeclaration(
795 F->getParent(), Intrinsic::bitreverse, F->arg_begin()->getType());
796 return true;
797 }
798
799 if (Name == "thread.pointer") {
800 // '(arm|aarch64).thread.pointer'.
801 NewFn = Intrinsic::getOrInsertDeclaration(
802 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
803 return true;
804 }
805
806 bool Neon = Name.consume_front("neon.");
807 if (Neon) {
808 // '(arm|aarch64).neon.*'.
809 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and
810 // v16i8 respectively.
811 if (Name.consume_front("bfdot.")) {
812 // (arm|aarch64).neon.bfdot.*'.
813 Intrinsic::ID ID =
814 StringSwitch<Intrinsic::ID>(Name)
815 .Cases({"v2f32.v8i8", "v4f32.v16i8"},
816 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfdot
817 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfdot)
818 .Default(Intrinsic::not_intrinsic);
819 if (ID != Intrinsic::not_intrinsic) {
820 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
821 assert((OperandWidth == 64 || OperandWidth == 128) &&
822 "Unexpected operand width");
823 LLVMContext &Ctx = F->getParent()->getContext();
824 std::array<Type *, 2> Tys{
825 {F->getReturnType(),
826 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)}};
827 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
828 return true;
829 }
830 return false; // No other '(arm|aarch64).neon.bfdot.*'.
831 }
832
833 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic
834 // anymore and accept v8bf16 instead of v16i8.
835 if (Name.consume_front("bfm")) {
836 // (arm|aarch64).neon.bfm*'.
837 if (Name.consume_back(".v4f32.v16i8")) {
838 // (arm|aarch64).neon.bfm*.v4f32.v16i8'.
839 Intrinsic::ID ID =
840 StringSwitch<Intrinsic::ID>(Name)
841 .Case("mla",
842 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmmla
843 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmmla)
844 .Case("lalb",
845 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalb
846 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalb)
847 .Case("lalt",
848 IsArm ? (Intrinsic::ID)Intrinsic::arm_neon_bfmlalt
849 : (Intrinsic::ID)Intrinsic::aarch64_neon_bfmlalt)
850 .Default(Intrinsic::not_intrinsic);
851 if (ID != Intrinsic::not_intrinsic) {
852 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
853 return true;
854 }
855 return false; // No other '(arm|aarch64).neon.bfm*.v16i8'.
856 }
857 return false; // No other '(arm|aarch64).neon.bfm*.
858 }
859 // Continue on to Aarch64 Neon or Arm Neon.
860 }
861 // Continue on to Arm or Aarch64.
862
863 if (IsArm) {
864 // 'arm.*'.
865 if (Neon) {
866 // 'arm.neon.*'.
867 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
868 .StartsWith("vclz.", Intrinsic::ctlz)
869 .StartsWith("vcnt.", Intrinsic::ctpop)
870 .StartsWith("vqadds.", Intrinsic::sadd_sat)
871 .StartsWith("vqaddu.", Intrinsic::uadd_sat)
872 .StartsWith("vqsubs.", Intrinsic::ssub_sat)
873 .StartsWith("vqsubu.", Intrinsic::usub_sat)
874 .StartsWith("vrinta.", Intrinsic::round)
875 .StartsWith("vrintn.", Intrinsic::roundeven)
876 .StartsWith("vrintm.", Intrinsic::floor)
877 .StartsWith("vrintp.", Intrinsic::ceil)
878 .StartsWith("vrintx.", Intrinsic::rint)
879 .StartsWith("vrintz.", Intrinsic::trunc)
880 .Default(Intrinsic::not_intrinsic);
881 if (ID != Intrinsic::not_intrinsic) {
882 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
883 F->arg_begin()->getType());
884 return true;
885 }
886
887 if (Name.consume_front("vst")) {
888 // 'arm.neon.vst*'.
889 static const Regex vstRegex("^([1234]|[234]lane)\\.v[a-z0-9]*$");
890 SmallVector<StringRef, 2> Groups;
891 if (vstRegex.match(Name, &Groups)) {
892 static const Intrinsic::ID StoreInts[] = {
893 Intrinsic::arm_neon_vst1, Intrinsic::arm_neon_vst2,
894 Intrinsic::arm_neon_vst3, Intrinsic::arm_neon_vst4};
895
896 static const Intrinsic::ID StoreLaneInts[] = {
897 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
898 Intrinsic::arm_neon_vst4lane};
899
900 auto fArgs = F->getFunctionType()->params();
901 Type *Tys[] = {fArgs[0], fArgs[1]};
902 if (Groups[1].size() == 1)
903 NewFn = Intrinsic::getOrInsertDeclaration(
904 F->getParent(), StoreInts[fArgs.size() - 3], Tys);
905 else
906 NewFn = Intrinsic::getOrInsertDeclaration(
907 F->getParent(), StoreLaneInts[fArgs.size() - 5], Tys);
908 return true;
909 }
910 return false; // No other 'arm.neon.vst*'.
911 }
912
913 return false; // No other 'arm.neon.*'.
914 }
915
916 if (Name.consume_front("mve.")) {
917 // 'arm.mve.*'.
918 if (Name == "vctp64") {
919 if (cast<FixedVectorType>(F->getReturnType())->getNumElements() == 4) {
920 // A vctp64 returning a v4i1 is converted to return a v2i1. Rename
921 // the function and deal with it below in UpgradeIntrinsicCall.
922 rename(F);
923 return true;
924 }
925 return false; // Not 'arm.mve.vctp64'.
926 }
927
928 if (Name.starts_with("vrintn.v")) {
929 NewFn = Intrinsic::getOrInsertDeclaration(
930 F->getParent(), Intrinsic::roundeven, F->arg_begin()->getType());
931 return true;
932 }
933
934 // These too are changed to accept a v2i1 instead of the old v4i1.
935 if (Name.consume_back(".v4i1")) {
936 // 'arm.mve.*.v4i1'.
937 if (Name.consume_back(".predicated.v2i64.v4i32"))
938 // 'arm.mve.*.predicated.v2i64.v4i32.v4i1'
939 return Name == "mull.int" || Name == "vqdmull";
940
941 if (Name.consume_back(".v2i64")) {
942 // 'arm.mve.*.v2i64.v4i1'
943 bool IsGather = Name.consume_front("vldr.gather.");
944 if (IsGather || Name.consume_front("vstr.scatter.")) {
945 if (Name.consume_front("base.")) {
946 // Optional 'wb.' prefix.
947 Name.consume_front("wb.");
948 // 'arm.mve.(vldr.gather|vstr.scatter).base.(wb.)?
949 // predicated.v2i64.v2i64.v4i1'.
950 return Name == "predicated.v2i64";
951 }
952
953 if (Name.consume_front("offset.predicated."))
954 return Name == (IsGather ? "v2i64.p0i64" : "p0i64.v2i64") ||
955 Name == (IsGather ? "v2i64.p0" : "p0.v2i64");
956
957 // No other 'arm.mve.(vldr.gather|vstr.scatter).*.v2i64.v4i1'.
958 return false;
959 }
960
961 return false; // No other 'arm.mve.*.v2i64.v4i1'.
962 }
963 return false; // No other 'arm.mve.*.v4i1'.
964 }
965 return false; // No other 'arm.mve.*'.
966 }
967
968 if (Name.consume_front("cde.vcx")) {
969 // 'arm.cde.vcx*'.
970 if (Name.consume_back(".predicated.v2i64.v4i1"))
971 // 'arm.cde.vcx*.predicated.v2i64.v4i1'.
972 return Name == "1q" || Name == "1qa" || Name == "2q" || Name == "2qa" ||
973 Name == "3q" || Name == "3qa";
974
975 return false; // No other 'arm.cde.vcx*'.
976 }
977 } else {
978 // 'aarch64.*'.
979 if (Neon) {
980 // 'aarch64.neon.*'.
981 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
982 .StartsWith("frintn", Intrinsic::roundeven)
983 .StartsWith("rbit", Intrinsic::bitreverse)
984 .Default(Intrinsic::not_intrinsic);
985 if (ID != Intrinsic::not_intrinsic) {
986 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
987 F->arg_begin()->getType());
988 return true;
989 }
990
991 if (Name.starts_with("addp")) {
992 // 'aarch64.neon.addp*'.
993 if (F->arg_size() != 2)
994 return false; // Invalid IR.
995 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
996 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
997 NewFn = Intrinsic::getOrInsertDeclaration(
998 F->getParent(), Intrinsic::aarch64_neon_faddp, Ty);
999 return true;
1000 }
1001 }
1002
1003 // Changed in 20.0: bfcvt/bfcvtn/bcvtn2 have been replaced with fptrunc.
1004 if (Name.starts_with("bfcvt")) {
1005 NewFn = nullptr;
1006 return true;
1007 }
1008
1009 // vcvtfp2hf and vcvthf2fp -> fpext and fptrunc
1010 if (Name == "vcvtfp2hf" || Name == "vcvthf2fp") {
1011 NewFn = nullptr;
1012 return true;
1013 }
1014
1015 return false; // No other 'aarch64.neon.*'.
1016 }
1017 if (Name.consume_front("sve.")) {
1018 // 'aarch64.sve.*'.
1019 if (Name.consume_front("bf")) {
1020 if (Name == "mmla") {
1021 Type *Tys[] = {F->getReturnType(),
1022 std::next(F->arg_begin())->getType()};
1023 NewFn = Intrinsic::getOrInsertDeclaration(
1024 F->getParent(), Intrinsic::aarch64_sve_fmmla, Tys);
1025 return true;
1026 }
1027 if (Name.consume_back(".lane")) {
1028 // 'aarch64.sve.bf*.lane'.
1029 Intrinsic::ID ID =
1030 StringSwitch<Intrinsic::ID>(Name)
1031 .Case("dot", Intrinsic::aarch64_sve_bfdot_lane_v2)
1032 .Case("mlalb", Intrinsic::aarch64_sve_bfmlalb_lane_v2)
1033 .Case("mlalt", Intrinsic::aarch64_sve_bfmlalt_lane_v2)
1034 .Default(Intrinsic::not_intrinsic);
1035 if (ID != Intrinsic::not_intrinsic) {
1036 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1037 return true;
1038 }
1039 return false; // No other 'aarch64.sve.bf*.lane'.
1040 }
1041 return false; // No other 'aarch64.sve.bf*'.
1042 }
1043
1044 // 'aarch64.sve.fcvt.bf16f32' || 'aarch64.sve.fcvtnt.bf16f32'
1045 if (Name == "fcvt.bf16f32" || Name == "fcvtnt.bf16f32") {
1046 NewFn = nullptr;
1047 return true;
1048 }
1049
1050 if (Name.consume_front("convert.from.svbool")) {
1051 // 'aarch64.sve.convert.from.svbool'
1052 auto *TTy = dyn_cast<TargetExtType>(F->getReturnType());
1053 if (!TTy || TTy->getName() != "aarch64.svcount")
1054 return false;
1055
1056 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount;
1057 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1058 return true;
1059 }
1060
1061 if (Name.consume_front("convert.to.svbool")) {
1062 // 'aarch64.sve.convert.to.svbool'
1063 auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType());
1064 if (!TTy || TTy->getName() != "aarch64.svcount")
1065 return false;
1066
1067 Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount;
1068 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1069 return true;
1070 }
1071
1072 if (Name.consume_front("addqv")) {
1073 // 'aarch64.sve.addqv'.
1074 if (!F->getReturnType()->isFPOrFPVectorTy())
1075 return false;
1076
1077 auto Args = F->getFunctionType()->params();
1078 Type *Tys[] = {F->getReturnType(), Args[1]};
1079 NewFn = Intrinsic::getOrInsertDeclaration(
1080 F->getParent(), Intrinsic::aarch64_sve_faddqv, Tys);
1081 return true;
1082 }
1083
1084 if (Name.consume_front("ld")) {
1085 // 'aarch64.sve.ld*'.
1086 static const Regex LdRegex("^[234](.nxv[a-z0-9]+|$)");
1087 if (LdRegex.match(Name)) {
1088 Type *ScalarTy =
1089 cast<VectorType>(F->getReturnType())->getElementType();
1090 ElementCount EC =
1091 cast<VectorType>(F->arg_begin()->getType())->getElementCount();
1092 assert(F->arg_size() == 2 &&
1093 "Expected 2 arguments for ld* intrinsic.");
1094 Type *PtrTy = F->getArg(1)->getType();
1095 Type *Ty = VectorType::get(ScalarTy, EC);
1096 static const Intrinsic::ID LoadIDs[] = {
1097 Intrinsic::aarch64_sve_ld2_sret,
1098 Intrinsic::aarch64_sve_ld3_sret,
1099 Intrinsic::aarch64_sve_ld4_sret,
1100 };
1101 NewFn = Intrinsic::getOrInsertDeclaration(
1102 F->getParent(), LoadIDs[Name[0] - '2'], {Ty, PtrTy});
1103 return true;
1104 }
1105 return false; // No other 'aarch64.sve.ld*'.
1106 }
1107
1108 if (Name.consume_front("tuple.")) {
1109 // 'aarch64.sve.tuple.*'.
1110 if (Name.starts_with("get")) {
1111 // 'aarch64.sve.tuple.get*'.
1112 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
1113 NewFn = Intrinsic::getOrInsertDeclaration(
1114 F->getParent(), Intrinsic::vector_extract, Tys);
1115 return true;
1116 }
1117
1118 if (Name.starts_with("set")) {
1119 // 'aarch64.sve.tuple.set*'.
1120 auto Args = F->getFunctionType()->params();
1121 Type *Tys[] = {Args[0], Args[2], Args[1]};
1122 NewFn = Intrinsic::getOrInsertDeclaration(
1123 F->getParent(), Intrinsic::vector_insert, Tys);
1124 return true;
1125 }
1126
1127 static const Regex CreateTupleRegex("^create[234](.nxv[a-z0-9]+|$)");
1128 if (CreateTupleRegex.match(Name)) {
1129 // 'aarch64.sve.tuple.create*'.
1130 auto Args = F->getFunctionType()->params();
1131 Type *Tys[] = {F->getReturnType(), Args[1]};
1132 NewFn = Intrinsic::getOrInsertDeclaration(
1133 F->getParent(), Intrinsic::vector_insert, Tys);
1134 return true;
1135 }
1136 return false; // No other 'aarch64.sve.tuple.*'.
1137 }
1138
1139 if (Name.starts_with("rev.nxv")) {
1140 // 'aarch64.sve.rev.<Ty>'
1141 NewFn = Intrinsic::getOrInsertDeclaration(
1142 F->getParent(), Intrinsic::vector_reverse, F->getReturnType());
1143 return true;
1144 }
1145
1146 return false; // No other 'aarch64.sve.*'.
1147 }
1148 if (Name.consume_front("sme.")) {
1149 // 'aarch64.sme.*'.
1150 if (Name.consume_front("ftmopa.")) {
1151 // The FP8 FTMOPA intrinsics were split out from the non-FP8 FTMOPA
1152 // intrinsics to model their FPMR dependency.
1153 Intrinsic::ID ID =
1154 StringSwitch<Intrinsic::ID>(Name)
1155 .Case("za16.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za16)
1156 .Case("za32.nxv16i8", Intrinsic::aarch64_sme_fp8_ftmopa_za32)
1157 .Default(Intrinsic::not_intrinsic);
1158 if (ID != Intrinsic::not_intrinsic) {
1159 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
1160 return true;
1161 }
1162 return false; // No other 'aarch64.sme.ftmopa.*'.
1163 }
1164
1165 return false; // No other 'aarch64.sme.*'.
1166 }
1167 }
1168 return false; // No other 'arm.*', 'aarch64.*'.
1169}
1170
1171static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F,
1172 StringRef Name) {
1173 if (Name.consume_front("cp.async.bulk.tensor.g2s.")) {
1174 Intrinsic::ID ID =
1175 StringSwitch<Intrinsic::ID>(Name)
1176 .Case("im2col.3d",
1177 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d)
1178 .Case("im2col.4d",
1179 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d)
1180 .Case("im2col.5d",
1181 Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d)
1182 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d)
1183 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d)
1184 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d)
1185 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d)
1186 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d)
1187 .Default(Intrinsic::not_intrinsic);
1188
1189 if (ID == Intrinsic::not_intrinsic)
1190 return ID;
1191
1192 // These intrinsics may need upgrade for two reasons:
1193 // (1) When the address-space of the first argument is shared[AS=3]
1194 // (and we upgrade it to use shared_cluster address-space[AS=7])
1195 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1196 NVPTXAS::ADDRESS_SPACE_SHARED)
1197 return ID;
1198
1199 // (2) When there are only two boolean flag arguments at the end:
1200 //
1201 // The last three parameters of the older version of these
1202 // intrinsics are: arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag
1203 //
1204 // The newer version reads as:
1205 // arg1, arg2, .. i64 ch, i1 mc_flag, i1 ch_flag, i32 cta_group_flag
1206 //
1207 // So, when the type of the [N-3]rd argument is "not i1", then
1208 // it is the older version and we need to upgrade.
1209 size_t FlagStartIndex = F->getFunctionType()->getNumParams() - 3;
1210 Type *ArgType = F->getFunctionType()->getParamType(FlagStartIndex);
1211 if (!ArgType->isIntegerTy(1))
1212 return ID;
1213 }
1214
1215 return Intrinsic::not_intrinsic;
1216}
1217
1218// The legacy TMA reduction intrinsics encode the reduction operator in their
1219// name, while the current ones take it as an immediate argument. Map the
1220// operator part of a legacy name to the corresponding immediate value.
1221static std::optional<unsigned> getNVPTXTMAReductionOp(StringRef Name) {
1222 return StringSwitch<std::optional<unsigned>>(Name)
1223 .Case("add", static_cast<unsigned>(nvvm::TMAReductionOp::ADD))
1224 .Case("min", static_cast<unsigned>(nvvm::TMAReductionOp::MIN))
1225 .Case("max", static_cast<unsigned>(nvvm::TMAReductionOp::MAX))
1226 .Case("inc", static_cast<unsigned>(nvvm::TMAReductionOp::INC))
1227 .Case("dec", static_cast<unsigned>(nvvm::TMAReductionOp::DEC))
1228 .Case("and", static_cast<unsigned>(nvvm::TMAReductionOp::AND))
1229 .Case("or", static_cast<unsigned>(nvvm::TMAReductionOp::OR))
1230 .Case("xor", static_cast<unsigned>(nvvm::TMAReductionOp::XOR))
1231 .Default(std::nullopt);
1232}
1233
1234static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name) {
1235 if (!Name.consume_front("cp.async.bulk.tensor.reduce."))
1236 return Intrinsic::not_intrinsic;
1237
1238 auto [RedOpName, ShapeName] = Name.split('.');
1239 if (!getNVPTXTMAReductionOp(RedOpName))
1240 return Intrinsic::not_intrinsic;
1241
1242 return StringSwitch<Intrinsic::ID>(ShapeName)
1243 .Case("tile.1d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d)
1244 .Case("tile.2d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d)
1245 .Case("tile.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d)
1246 .Case("tile.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d)
1247 .Case("tile.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d)
1248 .Case("im2col.3d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d)
1249 .Case("im2col.4d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d)
1250 .Case("im2col.5d", Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d)
1251 .Default(Intrinsic::not_intrinsic);
1252}
1253
1254static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
1255 StringRef Name) {
1256 if (Name.consume_front("mapa.shared.cluster"))
1257 if (F->getReturnType()->getPointerAddressSpace() ==
1258 NVPTXAS::ADDRESS_SPACE_SHARED)
1259 return Intrinsic::nvvm_mapa_shared_cluster;
1260
1261 if (Name.consume_front("cp.async.bulk.")) {
1262 Intrinsic::ID ID =
1263 StringSwitch<Intrinsic::ID>(Name)
1264 .Case("global.to.shared.cluster",
1265 Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster)
1266 .Case("shared.cta.to.cluster",
1267 Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster)
1268 .Default(Intrinsic::not_intrinsic);
1269
1270 if (ID != Intrinsic::not_intrinsic)
1271 if (F->getArg(0)->getType()->getPointerAddressSpace() ==
1272 NVPTXAS::ADDRESS_SPACE_SHARED)
1273 return ID;
1274 }
1275
1276 return Intrinsic::not_intrinsic;
1277}
1278
1279static Intrinsic::ID
1280shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name) {
1281 if (!Name.consume_front("tcgen05.commit."))
1282 return Intrinsic::not_intrinsic;
1283
1284 if (Name.consume_front("shared."))
1285 return StringSwitch<Intrinsic::ID>(Name)
1286 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_cg1)
1287 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_cg2)
1288 .Default(Intrinsic::not_intrinsic);
1289
1290 if (Name.consume_front("mc.shared.")) {
1291 // Only upgrade older i16 mc variants.
1292 if (!F->getArg(1)->getType()->isIntegerTy(16))
1293 return Intrinsic::not_intrinsic;
1294
1295 return StringSwitch<Intrinsic::ID>(Name)
1296 .Case("cg1", Intrinsic::nvvm_tcgen05_commit_mc_cg1)
1297 .Case("cg2", Intrinsic::nvvm_tcgen05_commit_mc_cg2)
1298 .Default(Intrinsic::not_intrinsic);
1299 }
1300
1301 return Intrinsic::not_intrinsic;
1302}
1303
1304static Intrinsic::ID
1305shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name) {
1306 if (F->arg_size() != 2)
1307 return Intrinsic::not_intrinsic;
1308
1309 if (Name.consume_front("tcgen05.alloc.shared.") ||
1310 Name.consume_front("tcgen05.alloc."))
1311 return StringSwitch<Intrinsic::ID>(Name)
1312 .Case("cg1", Intrinsic::nvvm_tcgen05_alloc_cg1)
1313 .Case("cg2", Intrinsic::nvvm_tcgen05_alloc_cg2)
1314 .Default(Intrinsic::not_intrinsic);
1315
1316 if (Name.consume_front("tcgen05.dealloc."))
1317 return StringSwitch<Intrinsic::ID>(Name)
1318 .Case("cg1", Intrinsic::nvvm_tcgen05_dealloc_cg1)
1319 .Case("cg2", Intrinsic::nvvm_tcgen05_dealloc_cg2)
1320 .Default(Intrinsic::not_intrinsic);
1321
1322 return Intrinsic::not_intrinsic;
1323}
1324
1325static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name) {
1326 if (Name.consume_front("fma.rn."))
1327 return StringSwitch<Intrinsic::ID>(Name)
1328 .Case("bf16", Intrinsic::nvvm_fma_rn_bf16)
1329 .Case("bf16x2", Intrinsic::nvvm_fma_rn_bf16x2)
1330 .Case("relu.bf16", Intrinsic::nvvm_fma_rn_relu_bf16)
1331 .Case("relu.bf16x2", Intrinsic::nvvm_fma_rn_relu_bf16x2)
1332 .Default(Intrinsic::not_intrinsic);
1333
1334 if (Name.consume_front("fmax."))
1335 return StringSwitch<Intrinsic::ID>(Name)
1336 .Case("bf16", Intrinsic::nvvm_fmax_bf16)
1337 .Case("bf16x2", Intrinsic::nvvm_fmax_bf16x2)
1338 .Case("ftz.bf16", Intrinsic::nvvm_fmax_ftz_bf16)
1339 .Case("ftz.bf16x2", Intrinsic::nvvm_fmax_ftz_bf16x2)
1340 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmax_ftz_nan_bf16)
1341 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmax_ftz_nan_bf16x2)
1342 .Case("ftz.nan.xorsign.abs.bf16",
1343 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16)
1344 .Case("ftz.nan.xorsign.abs.bf16x2",
1345 Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_bf16x2)
1346 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16)
1347 .Case("ftz.xorsign.abs.bf16x2",
1348 Intrinsic::nvvm_fmax_ftz_xorsign_abs_bf16x2)
1349 .Case("nan.bf16", Intrinsic::nvvm_fmax_nan_bf16)
1350 .Case("nan.bf16x2", Intrinsic::nvvm_fmax_nan_bf16x2)
1351 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16)
1352 .Case("nan.xorsign.abs.bf16x2",
1353 Intrinsic::nvvm_fmax_nan_xorsign_abs_bf16x2)
1354 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmax_xorsign_abs_bf16)
1355 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmax_xorsign_abs_bf16x2)
1356 .Default(Intrinsic::not_intrinsic);
1357
1358 if (Name.consume_front("fmin."))
1359 return StringSwitch<Intrinsic::ID>(Name)
1360 .Case("bf16", Intrinsic::nvvm_fmin_bf16)
1361 .Case("bf16x2", Intrinsic::nvvm_fmin_bf16x2)
1362 .Case("ftz.bf16", Intrinsic::nvvm_fmin_ftz_bf16)
1363 .Case("ftz.bf16x2", Intrinsic::nvvm_fmin_ftz_bf16x2)
1364 .Case("ftz.nan.bf16", Intrinsic::nvvm_fmin_ftz_nan_bf16)
1365 .Case("ftz.nan.bf16x2", Intrinsic::nvvm_fmin_ftz_nan_bf16x2)
1366 .Case("ftz.nan.xorsign.abs.bf16",
1367 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16)
1368 .Case("ftz.nan.xorsign.abs.bf16x2",
1369 Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_bf16x2)
1370 .Case("ftz.xorsign.abs.bf16", Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16)
1371 .Case("ftz.xorsign.abs.bf16x2",
1372 Intrinsic::nvvm_fmin_ftz_xorsign_abs_bf16x2)
1373 .Case("nan.bf16", Intrinsic::nvvm_fmin_nan_bf16)
1374 .Case("nan.bf16x2", Intrinsic::nvvm_fmin_nan_bf16x2)
1375 .Case("nan.xorsign.abs.bf16", Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16)
1376 .Case("nan.xorsign.abs.bf16x2",
1377 Intrinsic::nvvm_fmin_nan_xorsign_abs_bf16x2)
1378 .Case("xorsign.abs.bf16", Intrinsic::nvvm_fmin_xorsign_abs_bf16)
1379 .Case("xorsign.abs.bf16x2", Intrinsic::nvvm_fmin_xorsign_abs_bf16x2)
1380 .Default(Intrinsic::not_intrinsic);
1381
1382 if (Name.consume_front("neg."))
1383 return StringSwitch<Intrinsic::ID>(Name)
1384 .Case("bf16", Intrinsic::nvvm_neg_bf16)
1385 .Case("bf16x2", Intrinsic::nvvm_neg_bf16x2)
1386 .Default(Intrinsic::not_intrinsic);
1387
1388 return Intrinsic::not_intrinsic;
1389}
1390
1391static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID) {
1392 FunctionType *NewFnTy = Intrinsic::getType(F->getContext(), IID);
1393 FunctionType *OldFnTy = F->getFunctionType();
1394 auto IsOldBF16StorageTy = [](Type *OldTy, Type *NewTy) {
1395 return OldTy->getScalarType()->isIntegerTy() &&
1396 OldTy->getPrimitiveSizeInBits() == NewTy->getPrimitiveSizeInBits();
1397 };
1398
1399 if (!IsOldBF16StorageTy(OldFnTy->getReturnType(), NewFnTy->getReturnType()))
1400 return false;
1401
1402 if (OldFnTy->getNumParams() != NewFnTy->getNumParams())
1403 return false;
1404
1405 for (unsigned I = 0, E = OldFnTy->getNumParams(); I != E; ++I)
1406 if (!IsOldBF16StorageTy(OldFnTy->getParamType(I), NewFnTy->getParamType(I)))
1407 return false;
1408
1409 return true;
1410}
1411
1412static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F,
1413 StringRef Name) {
1414 if (!Name.consume_front("tcgen05.mma."))
1415 return Intrinsic::not_intrinsic;
1416
1417 // tcgen05.mma.ws.* variants do not need collector-b appended.
1418 if (Name.starts_with("ws"))
1419 return Intrinsic::not_intrinsic;
1420
1421 return F->getIntrinsicID();
1422}
1423
1424static bool consumeNVVMPtrAddrSpace(StringRef &Name) {
1425 return Name.consume_front("local") || Name.consume_front("shared") ||
1426 Name.consume_front("global") || Name.consume_front("constant") ||
1427 Name.consume_front("param");
1428}
1429
1430static unsigned getFunctionalOpcodeForVP(StringRef Name) {
1431 if (!Name.consume_front("vp."))
1432 return 0;
1433 return StringSwitch<unsigned>(Name)
1434 .StartsWith("select", Instruction::Select)
1435 .StartsWith("add", Instruction::Add)
1436 .StartsWith("sub", Instruction::Sub)
1437 .StartsWith("mul", Instruction::Mul)
1438 .StartsWith("ashr", Instruction::AShr)
1439 .StartsWith("lshr", Instruction::LShr)
1440 .StartsWith("shl", Instruction::Shl)
1441 .StartsWith("or", Instruction::Or)
1442 .StartsWith("and", Instruction::And)
1443 .StartsWith("xor", Instruction::Xor)
1444 .StartsWith("fadd", Instruction::FAdd)
1445 .StartsWith("fsub", Instruction::FSub)
1446 .StartsWith("fmuladd", 0)
1447 .StartsWith("fmul", Instruction::FMul)
1448 .StartsWith("fdiv", Instruction::FDiv)
1449 .StartsWith("frem", Instruction::FRem)
1450 .StartsWith("fneg", Instruction::FNeg)
1451 .StartsWith("trunc", Instruction::Trunc)
1452 .StartsWith("zext", Instruction::ZExt)
1453 .StartsWith("sext", Instruction::SExt)
1454 .StartsWith("fptrunc", Instruction::FPTrunc)
1455 .StartsWith("fpext", Instruction::FPExt)
1456 .StartsWith("fptoui", Instruction::FPToUI)
1457 .StartsWith("fptosi", Instruction::FPToSI)
1458 .StartsWith("uitofp", Instruction::UIToFP)
1459 .StartsWith("sitofp", Instruction::SIToFP)
1460 .StartsWith("ptrtoint", Instruction::PtrToInt)
1461 .StartsWith("inttoptr", Instruction::IntToPtr)
1462 .StartsWith("icmp", Instruction::ICmp)
1463 .StartsWith("fcmp", Instruction::FCmp)
1464 .Default(0);
1465}
1466
1467static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name) {
1468 if (!Name.consume_front("vp."))
1469 return 0;
1470 return StringSwitch<Intrinsic::ID>(Name)
1471 .StartsWith("abs", Intrinsic::abs)
1472 .StartsWith("smax", Intrinsic::smax)
1473 .StartsWith("smin", Intrinsic::smin)
1474 .StartsWith("umax", Intrinsic::umax)
1475 .StartsWith("umin", Intrinsic::umin)
1476 .StartsWith("copysign", Intrinsic::copysign)
1477 .StartsWith("minnum", Intrinsic::minnum)
1478 .StartsWith("maxnum", Intrinsic::maxnum)
1479 .StartsWith("minimum", Intrinsic::minimum)
1480 .StartsWith("maximum", Intrinsic::maximum)
1481 .StartsWith("fabs", Intrinsic::fabs)
1482 .StartsWith("sqrt", Intrinsic::sqrt)
1483 .StartsWith("fma", Intrinsic::fma)
1484 .StartsWith("fmuladd", Intrinsic::fmuladd)
1485 .StartsWith("ceil", Intrinsic::ceil)
1486 .StartsWith("floor", Intrinsic::floor)
1487 .StartsWith("rint", Intrinsic::rint)
1488 .StartsWith("nearbyint", Intrinsic::nearbyint)
1489 .StartsWith("roundeven", Intrinsic::roundeven)
1490 .StartsWith("roundtozero", Intrinsic::trunc)
1491 .StartsWith("round", Intrinsic::round)
1492 .StartsWith("lrint", Intrinsic::lrint)
1493 .StartsWith("llrint", Intrinsic::llrint)
1494 .StartsWith("bitreverse", Intrinsic::bitreverse)
1495 .StartsWith("bswap", Intrinsic::bswap)
1496 .StartsWith("ctpop", Intrinsic::ctpop)
1497 .StartsWith("ctlz", Intrinsic::ctlz)
1498 .StartsWith("cttz.elts", 0)
1499 .StartsWith("cttz", Intrinsic::cttz)
1500 .StartsWith("sadd.sat", Intrinsic::sadd_sat)
1501 .StartsWith("uadd.sat", Intrinsic::uadd_sat)
1502 .StartsWith("ssub.sat", Intrinsic::ssub_sat)
1503 .StartsWith("usub.sat", Intrinsic::usub_sat)
1504 .StartsWith("fshl", Intrinsic::fshl)
1505 .StartsWith("fshr", Intrinsic::fshr)
1506 .StartsWith("is.fpclass", Intrinsic::is_fpclass)
1507 .Default(0);
1508}
1509
1513
1514static bool convertIntrinsicValidType(StringRef Name,
1515 const FunctionType *FuncTy) {
1516 Type *HalfTy = Type::getHalfTy(FuncTy->getContext());
1517 if (Name.starts_with("to.fp16")) {
1518 return CastInst::castIsValid(Instruction::FPTrunc, FuncTy->getParamType(0),
1519 HalfTy) &&
1520 CastInst::castIsValid(Instruction::BitCast, HalfTy,
1521 FuncTy->getReturnType());
1522 }
1523
1524 if (Name.starts_with("from.fp16")) {
1525 return CastInst::castIsValid(Instruction::BitCast, FuncTy->getParamType(0),
1526 HalfTy) &&
1527 CastInst::castIsValid(Instruction::FPExt, HalfTy,
1528 FuncTy->getReturnType());
1529 }
1530
1531 return false;
1532}
1533
1534static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn) {
1535 Intrinsic::ID IID = Intrinsic::lookupIntrinsicID(F->getName());
1536 if (IID == Intrinsic::not_intrinsic)
1537 return false;
1538
1539 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
1540 if (Defaults.empty())
1541 return false;
1542
1543 // Overloaded intrinsics are out of scope for the default-arg feature
1544 // and will be supported in a follow-up.
1545 if (Intrinsic::isOverloaded(IID))
1546 return false;
1547
1548 // Get the canonical full declaration for this intrinsic.
1549 Function *FullDecl = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1550
1551 // If the existing declaration already has all args, nothing to upgrade
1552 if (F->arg_size() >= FullDecl->arg_size())
1553 return false;
1554
1555 // Defaults are a contiguous trailing block, so checking the first missing
1556 // argument is enough.
1557 if (F->arg_size() < FirstDefault)
1558 return false;
1559
1560 NewFn = FullDecl;
1561 return true;
1562}
1563
1564static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
1565 bool CanUpgradeDebugIntrinsicsToRecords) {
1566 assert(F && "Illegal to upgrade a non-existent Function.");
1567
1568 StringRef Name = F->getName();
1569
1570 // Quickly eliminate it, if it's not a candidate.
1571 if (!Name.consume_front("llvm.") || Name.empty())
1572 return false;
1573
1574 switch (Name[0]) {
1575 default: break;
1576 case 'a': {
1577 bool IsArm = Name.consume_front("arm.");
1578 if (IsArm || Name.consume_front("aarch64.")) {
1579 if (upgradeArmOrAarch64IntrinsicFunction(IsArm, F, Name, NewFn))
1580 return true;
1581 break;
1582 }
1583
1584 if (Name.consume_front("amdgcn.")) {
1585 if (Name == "alignbit") {
1586 // Target specific intrinsic became redundant
1587 NewFn = Intrinsic::getOrInsertDeclaration(
1588 F->getParent(), Intrinsic::fshr, {F->getReturnType()});
1589 return true;
1590 }
1591
1592 if (Name.consume_front("atomic.")) {
1593 if (Name.starts_with("inc") || Name.starts_with("dec") ||
1594 Name.starts_with("cond.sub") || Name.starts_with("csub")) {
1595 // These were replaced with atomicrmw uinc_wrap, udec_wrap, usub_cond
1596 // and usub_sat so there's no new declaration.
1597 NewFn = nullptr;
1598 return true;
1599 }
1600 break; // No other 'amdgcn.atomic.*'
1601 }
1602
1603 switch (F->getIntrinsicID()) {
1604 default:
1605 break;
1606 // Legacy wmma iu intrinsics without the optional clamp operand.
1607 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
1608 if (F->arg_size() == 7) {
1609 NewFn = nullptr;
1610 return true;
1611 }
1612 break;
1613 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
1614 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
1615 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
1616 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
1617 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
1618 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
1619 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
1620 if (F->arg_size() == 8) {
1621 NewFn = nullptr;
1622 return true;
1623 }
1624 break;
1625 }
1626
1627 if (Name.consume_front("ds.") || Name.consume_front("global.atomic.") ||
1628 Name.consume_front("flat.atomic.")) {
1629 if (Name.starts_with("fadd") ||
1630 // FIXME: We should also remove fmin.num and fmax.num intrinsics.
1631 (Name.starts_with("fmin") && !Name.starts_with("fmin.num")) ||
1632 (Name.starts_with("fmax") && !Name.starts_with("fmax.num"))) {
1633 // Replaced with atomicrmw fadd/fmin/fmax, so there's no new
1634 // declaration.
1635 NewFn = nullptr;
1636 return true;
1637 }
1638 }
1639
1640 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
1641 NewFn = nullptr;
1642 return true;
1643 }
1644
1645 if (Name.starts_with("ldexp.")) {
1646 // Target specific intrinsic became redundant
1647 NewFn = Intrinsic::getOrInsertDeclaration(
1648 F->getParent(), Intrinsic::ldexp,
1649 {F->getReturnType(), F->getArg(1)->getType()});
1650 return true;
1651 }
1652 break; // No other 'amdgcn.*'
1653 }
1654
1655 break;
1656 }
1657 case 'c': {
1658 if (F->arg_size() == 1) {
1659 if (Name.consume_front("convert.")) {
1660 if (convertIntrinsicValidType(Name, F->getFunctionType())) {
1661 NewFn = nullptr;
1662 return true;
1663 }
1664 }
1665
1666 Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
1667 .StartsWith("ctlz.", Intrinsic::ctlz)
1668 .StartsWith("cttz.", Intrinsic::cttz)
1669 .Default(Intrinsic::not_intrinsic);
1670 if (ID != Intrinsic::not_intrinsic) {
1671 rename(F);
1672 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1673 F->arg_begin()->getType());
1674 return true;
1675 }
1676 }
1677
1678 Intrinsic::ID CoroEndID = Intrinsic::not_intrinsic;
1679 if (Name == "coro.end" &&
1680 (F->arg_size() == 2 || F->getReturnType()->isIntegerTy(1)))
1681 CoroEndID = Intrinsic::coro_end;
1682 else if (Name == "coro.end.async" && F->getReturnType()->isIntegerTy(1))
1683 CoroEndID = Intrinsic::coro_end_async;
1684
1685 if (CoroEndID != Intrinsic::not_intrinsic) {
1686 rename(F);
1687 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), CoroEndID);
1688 return true;
1689 }
1690
1691 break;
1692 }
1693 case 'd':
1694 if (Name.consume_front("dbg.")) {
1695 // Mark debug intrinsics for upgrade to new debug format.
1696 if (CanUpgradeDebugIntrinsicsToRecords) {
1697 if (Name == "addr" || Name == "value" || Name == "assign" ||
1698 Name == "declare" || Name == "label") {
1699 // There's no function to replace these with.
1700 NewFn = nullptr;
1701 // But we do want these to get upgraded.
1702 return true;
1703 }
1704 }
1705 // Update llvm.dbg.addr intrinsics even in "new debug mode"; they'll get
1706 // converted to DbgVariableRecords later.
1707 if (Name == "addr" || (Name == "value" && F->arg_size() == 4)) {
1708 rename(F);
1709 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1710 Intrinsic::dbg_value);
1711 return true;
1712 }
1713 break; // No other 'dbg.*'.
1714 }
1715 break;
1716 case 'e':
1717 if (Name.consume_front("experimental.vector.")) {
1718 Intrinsic::ID ID =
1719 StringSwitch<Intrinsic::ID>(Name)
1720 // Skip over extract.last.active, otherwise it will be 'upgraded'
1721 // to a regular vector extract which is a different operation.
1722 .StartsWith("extract.last.active.", Intrinsic::not_intrinsic)
1723 .StartsWith("extract.", Intrinsic::vector_extract)
1724 .StartsWith("insert.", Intrinsic::vector_insert)
1725 .StartsWith("reverse.", Intrinsic::vector_reverse)
1726 .StartsWith("interleave2.", Intrinsic::vector_interleave2)
1727 .StartsWith("deinterleave2.", Intrinsic::vector_deinterleave2)
1728 .StartsWith("partial.reduce.add",
1729 Intrinsic::vector_partial_reduce_add)
1730 .Default(Intrinsic::not_intrinsic);
1731 if (ID != Intrinsic::not_intrinsic) {
1732 const auto *FT = F->getFunctionType();
1733 SmallVector<Type *, 2> Tys;
1734 if (ID == Intrinsic::vector_extract ||
1735 ID == Intrinsic::vector_interleave2)
1736 // Extracting overloads the return type.
1737 Tys.push_back(FT->getReturnType());
1738 if (ID != Intrinsic::vector_interleave2)
1739 Tys.push_back(FT->getParamType(0));
1740 if (ID == Intrinsic::vector_insert ||
1741 ID == Intrinsic::vector_partial_reduce_add)
1742 // Inserting overloads the inserted type.
1743 Tys.push_back(FT->getParamType(1));
1744 rename(F);
1745 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, Tys);
1746 return true;
1747 }
1748
1749 if (Name.consume_front("reduce.")) {
1750 SmallVector<StringRef, 2> Groups;
1751 static const Regex R("^([a-z]+)\\.[a-z][0-9]+");
1752 if (R.match(Name, &Groups))
1753 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1754 .Case("add", Intrinsic::vector_reduce_add)
1755 .Case("mul", Intrinsic::vector_reduce_mul)
1756 .Case("and", Intrinsic::vector_reduce_and)
1757 .Case("or", Intrinsic::vector_reduce_or)
1758 .Case("xor", Intrinsic::vector_reduce_xor)
1759 .Case("smax", Intrinsic::vector_reduce_smax)
1760 .Case("smin", Intrinsic::vector_reduce_smin)
1761 .Case("umax", Intrinsic::vector_reduce_umax)
1762 .Case("umin", Intrinsic::vector_reduce_umin)
1763 .Case("fmax", Intrinsic::vector_reduce_fmax)
1764 .Case("fmin", Intrinsic::vector_reduce_fmin)
1765 .Default(Intrinsic::not_intrinsic);
1766
1767 bool V2 = false;
1768 if (ID == Intrinsic::not_intrinsic) {
1769 static const Regex R2("^v2\\.([a-z]+)\\.[fi][0-9]+");
1770 Groups.clear();
1771 V2 = true;
1772 if (R2.match(Name, &Groups))
1773 ID = StringSwitch<Intrinsic::ID>(Groups[1])
1774 .Case("fadd", Intrinsic::vector_reduce_fadd)
1775 .Case("fmul", Intrinsic::vector_reduce_fmul)
1776 .Default(Intrinsic::not_intrinsic);
1777 }
1778 if (ID != Intrinsic::not_intrinsic) {
1779 rename(F);
1780 auto Args = F->getFunctionType()->params();
1781 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
1782 {Args[V2 ? 1 : 0]});
1783 return true;
1784 }
1785 break; // No other 'expermental.vector.reduce.*'.
1786 }
1787
1788 if (Name.consume_front("splice"))
1789 return true;
1790 break; // No other 'experimental.vector.*'.
1791 }
1792 if (Name.consume_front("experimental.stepvector.")) {
1793 Intrinsic::ID ID = Intrinsic::stepvector;
1794 rename(F);
1795 NewFn = Intrinsic::getOrInsertDeclaration(
1796 F->getParent(), ID, F->getFunctionType()->getReturnType());
1797 return true;
1798 }
1799 break; // No other 'e*'.
1800 case 'f':
1801 if (Name.starts_with("flt.rounds")) {
1802 rename(F);
1803 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1804 Intrinsic::get_rounding);
1805 return true;
1806 }
1807 break;
1808 case 'i':
1809 if (Name.starts_with("invariant.group.barrier")) {
1810 // Rename invariant.group.barrier to launder.invariant.group
1811 auto Args = F->getFunctionType()->params();
1812 Type* ObjectPtr[1] = {Args[0]};
1813 rename(F);
1814 NewFn = Intrinsic::getOrInsertDeclaration(
1815 F->getParent(), Intrinsic::launder_invariant_group, ObjectPtr);
1816 return true;
1817 }
1818 break;
1819 case 'l': {
1820 bool IsLifetimeStart = Name.consume_front("lifetime.start");
1821 bool IsLifetimeEnd = !IsLifetimeStart && Name.consume_front("lifetime.end");
1822 if (IsLifetimeStart || IsLifetimeEnd) {
1823 if (F->arg_size() == 2) {
1824 Intrinsic::ID IID = IsLifetimeStart ? Intrinsic::lifetime_start
1825 : Intrinsic::lifetime_end;
1826 rename(F);
1827 // Old 2 argument form of these intrinsics have [Size, Ptr] as
1828 // arguments. Use the Ptr argument to create new declaration.
1829 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1830 F->getArg(1)->getType());
1831 return true;
1832 } else if (F->arg_size() == 1 && Name == ".i64") {
1833 // Matches @llvm.lifetime.{start/end}.i64 which used to be created by
1834 // Autoupgrade prior to
1835 // https://github.com/llvm/llvm-project/pull/204601. This is an invalid
1836 // intrinsic with no expected calls. To allow auto-upgrade process to
1837 // delete such invalid intrinsic declaration, set NewFn = nullptr
1838 // and return true here. If there are actual calls to this intrinsic
1839 // (which is not expected), they will be deleted in
1840 // UpgradeIntrinsicCall.
1841 NewFn = nullptr;
1842 return true;
1843 }
1844 }
1845 break;
1846 }
1847 case 'm': {
1848 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
1849 // alignment parameter to embedding the alignment as an attribute of
1850 // the pointer args.
1851 if (unsigned ID = StringSwitch<unsigned>(Name)
1852 .StartsWith("memcpy.", Intrinsic::memcpy)
1853 .StartsWith("memmove.", Intrinsic::memmove)
1854 .Default(0)) {
1855 if (F->arg_size() == 5) {
1856 rename(F);
1857 // Get the types of dest, src, and len
1858 ArrayRef<Type *> ParamTypes =
1859 F->getFunctionType()->params().slice(0, 3);
1860 NewFn =
1861 Intrinsic::getOrInsertDeclaration(F->getParent(), ID, ParamTypes);
1862 return true;
1863 }
1864 }
1865 if (Name.starts_with("memset.") && F->arg_size() == 5) {
1866 rename(F);
1867 // Get the types of dest, and len
1868 const auto *FT = F->getFunctionType();
1869 Type *ParamTypes[2] = {
1870 FT->getParamType(0), // Dest
1871 FT->getParamType(2) // len
1872 };
1873 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
1874 Intrinsic::memset, ParamTypes);
1875 return true;
1876 }
1877
1878 unsigned MaskedID =
1879 StringSwitch<unsigned>(Name)
1880 .StartsWith("masked.load", Intrinsic::masked_load)
1881 .StartsWith("masked.gather", Intrinsic::masked_gather)
1882 .StartsWith("masked.store", Intrinsic::masked_store)
1883 .StartsWith("masked.scatter", Intrinsic::masked_scatter)
1884 .Default(0);
1885 if (MaskedID && F->arg_size() == 4) {
1886 rename(F);
1887 if (MaskedID == Intrinsic::masked_load ||
1888 MaskedID == Intrinsic::masked_gather) {
1889 NewFn = Intrinsic::getOrInsertDeclaration(
1890 F->getParent(), MaskedID,
1891 {F->getReturnType(), F->getArg(0)->getType()});
1892 return true;
1893 }
1894 NewFn = Intrinsic::getOrInsertDeclaration(
1895 F->getParent(), MaskedID,
1896 {F->getArg(0)->getType(), F->getArg(1)->getType()});
1897 return true;
1898 }
1899 break;
1900 }
1901 case 'n': {
1902 if (Name.consume_front("nvvm.")) {
1903 // Check for nvvm intrinsics corresponding exactly to an LLVM intrinsic.
1904 if (F->arg_size() == 1) {
1905 Intrinsic::ID IID =
1906 StringSwitch<Intrinsic::ID>(Name)
1907 .Cases({"brev32", "brev64"}, Intrinsic::bitreverse)
1908 .Case("clz.i", Intrinsic::ctlz)
1909 .Case("popc.i", Intrinsic::ctpop)
1910 .Default(Intrinsic::not_intrinsic);
1911 if (IID != Intrinsic::not_intrinsic) {
1912 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1913 {F->getReturnType()});
1914 return true;
1915 }
1916 } else if (F->arg_size() == 2) {
1917 Intrinsic::ID IID =
1918 StringSwitch<Intrinsic::ID>(Name)
1919 .Cases({"max.s", "max.i", "max.ll"}, Intrinsic::smax)
1920 .Cases({"min.s", "min.i", "min.ll"}, Intrinsic::smin)
1921 .Cases({"max.us", "max.ui", "max.ull"}, Intrinsic::umax)
1922 .Cases({"min.us", "min.ui", "min.ull"}, Intrinsic::umin)
1923 .Default(Intrinsic::not_intrinsic);
1924 if (IID != Intrinsic::not_intrinsic) {
1925 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1926 {F->getReturnType()});
1927 return true;
1928 }
1929 }
1930
1931 // Check for nvvm intrinsics that need a return type adjustment.
1932 {
1933 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
1934 if (IID != Intrinsic::not_intrinsic &&
1935 isLegacyNVPTXBF16IntSignature(F, IID)) {
1936 NewFn = nullptr;
1937 return true;
1938 }
1939 }
1940
1941 // Upgrade Distributed Shared Memory Intrinsics
1942 Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
1943 if (IID != Intrinsic::not_intrinsic) {
1944 rename(F);
1945 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1946 return true;
1947 }
1948
1949 // Upgrade TMA reduction intrinsics
1950 // llvm.nvvm.cp.async.bulk.tensor.reduce.<red_op>* =>
1951 // llvm.nvvm.cp.async.bulk.tensor.reduce.<shape>*
1952 IID = shouldUpgradeNVPTXTMAReductionIntrinsics(Name);
1953 if (IID != Intrinsic::not_intrinsic) {
1954 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1955 return true;
1956 }
1957
1958 // Upgrade tcgen05.commit shared variants to anyptr intrinsics.
1959 IID = shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(F, Name);
1960 if (IID != Intrinsic::not_intrinsic) {
1961 rename(F);
1962 NewFn = Intrinsic::getOrInsertDeclaration(
1963 F->getParent(), IID, F->getReturnType(),
1964 F->getFunctionType()->params());
1965 return true;
1966 }
1967
1968 // Upgrade tcgen05.alloc/dealloc with the is_exclusive argument and
1969 // tcgen05.alloc shared variants to anyptr intrinsics.
1970 IID = shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(F, Name);
1971 if (IID != Intrinsic::not_intrinsic) {
1972 rename(F);
1973 if (Intrinsic::isOverloaded(IID))
1974 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
1975 {F->getArg(0)->getType()});
1976 else
1977 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1978 return true;
1979 }
1980
1981 // Upgrade TMA copy G2S Intrinsics
1982 IID = shouldUpgradeNVPTXTMAG2SIntrinsics(F, Name);
1983 if (IID != Intrinsic::not_intrinsic) {
1984 rename(F);
1985 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1986 return true;
1987 }
1988
1989 // Upgrade tcgen05.mma intrinsics missing collector_usage_b.
1990 IID = shouldUpgradeNVPTXTcgen05MMAIntrinsic(F, Name);
1991 if (IID != Intrinsic::not_intrinsic) {
1992 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
1993 return NewFn != F;
1994 }
1995
1996 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
1997 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
1998 //
1999 // TODO: We could add lohi.i2d.
2000 bool Expand = false;
2001 if (Name.consume_front("abs."))
2002 // nvvm.abs.{i,ii}
2003 Expand =
2004 Name == "i" || Name == "ll" || Name == "bf16" || Name == "bf16x2";
2005 else if (Name.consume_front("fabs."))
2006 // nvvm.fabs.{f,ftz.f,d}
2007 Expand = Name == "f" || Name == "ftz.f" || Name == "d";
2008 else if (Name.consume_front("ex2.approx."))
2009 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
2010 Expand =
2011 Name == "f" || Name == "ftz.f" || Name == "d" || Name == "f16x2";
2012 else if (Name.consume_front("atomic.load."))
2013 // nvvm.atomic.load.add.{f32,f64}.p
2014 // nvvm.atomic.load.{inc,dec}.32.p
2015 Expand = StringSwitch<bool>(Name)
2016 .StartsWith("add.f32.p", true)
2017 .StartsWith("add.f64.p", true)
2018 .StartsWith("inc.32.p", true)
2019 .StartsWith("dec.32.p", true)
2020 .Default(false);
2021 else if (Name.consume_front("atomic."))
2022 // nvvm.atomic.{add,exch,max,min,inc,dec,and,or,xor}.gen.{i,f}.{cta,sys}
2023 // nvvm.atomic.cas.gen.i.{cta,sys}
2024 Expand = StringSwitch<bool>(Name)
2025 .StartsWith("add.gen.", true)
2026 .StartsWith("exch.gen.", true)
2027 .StartsWith("max.gen.", true)
2028 .StartsWith("min.gen.", true)
2029 .StartsWith("inc.gen.", true)
2030 .StartsWith("dec.gen.", true)
2031 .StartsWith("and.gen.", true)
2032 .StartsWith("or.gen.", true)
2033 .StartsWith("xor.gen.", true)
2034 .StartsWith("cas.gen.", true)
2035 .Default(false);
2036 else if (Name.consume_front("bitcast."))
2037 // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
2038 Expand =
2039 Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
2040 else if (Name.consume_front("rotate."))
2041 // nvvm.rotate.{b32,b64,right.b64}
2042 Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
2043 else if (Name.consume_front("ptr.gen.to."))
2044 // nvvm.ptr.gen.to.{local,shared,global,constant,param}
2045 Expand = consumeNVVMPtrAddrSpace(Name);
2046 else if (Name.consume_front("ptr."))
2047 // nvvm.ptr.{local,shared,global,constant,param}.to.gen
2048 Expand = consumeNVVMPtrAddrSpace(Name) && Name.starts_with(".to.gen");
2049 else if (Name.consume_front("ldg.global."))
2050 // nvvm.ldg.global.{i,p,f}
2051 Expand = (Name.starts_with("i.") || Name.starts_with("f.") ||
2052 Name.starts_with("p."));
2053 else
2054 Expand = StringSwitch<bool>(Name)
2055 .Case("barrier0", true)
2056 .Case("barrier.n", true)
2057 .Case("barrier.sync.cnt", true)
2058 .Case("barrier.sync", true)
2059 .Case("barrier", true)
2060 .Case("bar.sync", true)
2061 .Case("barrier0.popc", true)
2062 .Case("barrier0.and", true)
2063 .Case("barrier0.or", true)
2064 .Case("clz.ll", true)
2065 .Case("popc.ll", true)
2066 .Case("h2f", true)
2067 .Case("swap.lo.hi.b64", true)
2068 .Case("tanh.approx.f32", true)
2069 .Default(false);
2070
2071 if (Expand) {
2072 NewFn = nullptr;
2073 return true;
2074 }
2075 break; // No other 'nvvm.*'.
2076 }
2077 break;
2078 }
2079 case 'o':
2080 if (Name.starts_with("objectsize.")) {
2081 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
2082 if (F->arg_size() == 2 || F->arg_size() == 3) {
2083 rename(F);
2084 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(),
2085 Intrinsic::objectsize, Tys);
2086 return true;
2087 }
2088 }
2089 break;
2090
2091 case 'p':
2092 if (Name.starts_with("ptr.annotation.") && F->arg_size() == 4) {
2093 rename(F);
2094 NewFn = Intrinsic::getOrInsertDeclaration(
2095 F->getParent(), Intrinsic::ptr_annotation,
2096 {F->arg_begin()->getType(), F->getArg(1)->getType()});
2097 return true;
2098 }
2099 break;
2100
2101 case 'r': {
2102 if (Name.consume_front("riscv.")) {
2103 Intrinsic::ID ID;
2104 ID = StringSwitch<Intrinsic::ID>(Name)
2105 .Case("aes32dsi", Intrinsic::riscv_aes32dsi)
2106 .Case("aes32dsmi", Intrinsic::riscv_aes32dsmi)
2107 .Case("aes32esi", Intrinsic::riscv_aes32esi)
2108 .Case("aes32esmi", Intrinsic::riscv_aes32esmi)
2109 .Default(Intrinsic::not_intrinsic);
2110 if (ID != Intrinsic::not_intrinsic) {
2111 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32)) {
2112 rename(F);
2113 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2114 return true;
2115 }
2116 break; // No other applicable upgrades.
2117 }
2118
2119 ID = StringSwitch<Intrinsic::ID>(Name)
2120 .StartsWith("sm4ks", Intrinsic::riscv_sm4ks)
2121 .StartsWith("sm4ed", Intrinsic::riscv_sm4ed)
2122 .Default(Intrinsic::not_intrinsic);
2123 if (ID != Intrinsic::not_intrinsic) {
2124 if (!F->getFunctionType()->getParamType(2)->isIntegerTy(32) ||
2125 F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2126 rename(F);
2127 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2128 return true;
2129 }
2130 break; // No other applicable upgrades.
2131 }
2132
2133 ID = StringSwitch<Intrinsic::ID>(Name)
2134 .StartsWith("sha256sig0", Intrinsic::riscv_sha256sig0)
2135 .StartsWith("sha256sig1", Intrinsic::riscv_sha256sig1)
2136 .StartsWith("sha256sum0", Intrinsic::riscv_sha256sum0)
2137 .StartsWith("sha256sum1", Intrinsic::riscv_sha256sum1)
2138 .StartsWith("sm3p0", Intrinsic::riscv_sm3p0)
2139 .StartsWith("sm3p1", Intrinsic::riscv_sm3p1)
2140 .Default(Intrinsic::not_intrinsic);
2141 if (ID != Intrinsic::not_intrinsic) {
2142 if (F->getFunctionType()->getReturnType()->isIntegerTy(64)) {
2143 rename(F);
2144 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2145 return true;
2146 }
2147 break; // No other applicable upgrades.
2148 }
2149
2150 // Replace llvm.riscv.clmul with llvm.clmul.
2151 if (Name == "clmul.i32" || Name == "clmul.i64") {
2152 NewFn = Intrinsic::getOrInsertDeclaration(
2153 F->getParent(), Intrinsic::clmul, {F->getReturnType()});
2154 return true;
2155 }
2156
2157 break; // No other 'riscv.*' intrinsics
2158 }
2159 } break;
2160
2161 case 's':
2162 if (Name == "stackprotectorcheck") {
2163 NewFn = nullptr;
2164 return true;
2165 }
2166 break;
2167
2168 case 't':
2169 if (Name == "thread.pointer") {
2170 NewFn = Intrinsic::getOrInsertDeclaration(
2171 F->getParent(), Intrinsic::thread_pointer, F->getReturnType());
2172 return true;
2173 }
2174 break;
2175
2176 case 'v': {
2177 if (Name == "var.annotation" && F->arg_size() == 4) {
2178 rename(F);
2179 NewFn = Intrinsic::getOrInsertDeclaration(
2180 F->getParent(), Intrinsic::var_annotation,
2181 {{F->arg_begin()->getType(), F->getArg(1)->getType()}});
2182 return true;
2183 }
2184 if (Name.consume_front("vector.splice")) {
2185 if (Name.starts_with(".left") || Name.starts_with(".right"))
2186 break;
2187 return true;
2188 }
2189 if (shouldUpgradeVPIntrinsic(Name))
2190 return true;
2191 break;
2192 }
2193
2194 case 'w':
2195 if (Name.consume_front("wasm.")) {
2196 Intrinsic::ID ID =
2197 StringSwitch<Intrinsic::ID>(Name)
2198 .StartsWith("fma.", Intrinsic::wasm_relaxed_madd)
2199 .StartsWith("fms.", Intrinsic::wasm_relaxed_nmadd)
2200 .StartsWith("laneselect.", Intrinsic::wasm_relaxed_laneselect)
2201 .Default(Intrinsic::not_intrinsic);
2202 if (ID != Intrinsic::not_intrinsic) {
2203 rename(F);
2204 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
2205 F->getReturnType());
2206 return true;
2207 }
2208
2209 if (Name.consume_front("dot.i8x16.i7x16.")) {
2210 ID = StringSwitch<Intrinsic::ID>(Name)
2211 .Case("signed", Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed)
2212 .Case("add.signed",
2213 Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed)
2214 .Default(Intrinsic::not_intrinsic);
2215 if (ID != Intrinsic::not_intrinsic) {
2216 rename(F);
2217 NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID);
2218 return true;
2219 }
2220 break; // No other 'wasm.dot.i8x16.i7x16.*'.
2221 }
2222 break; // No other 'wasm.*'.
2223 }
2224 break;
2225
2226 case 'x':
2227 if (upgradeX86IntrinsicFunction(F, Name, NewFn))
2228 return true;
2229 }
2230
2231 auto *ST = dyn_cast<StructType>(F->getReturnType());
2232 if (ST && (!ST->isLiteral() || ST->isPacked()) &&
2233 F->getIntrinsicID() != Intrinsic::not_intrinsic) {
2234 // Replace return type with literal non-packed struct. Only do this for
2235 // intrinsics declared to return a struct, not for intrinsics with
2236 // overloaded return type, in which case the exact struct type will be
2237 // mangled into the name.
2238 if (Intrinsic::hasStructReturnType(F->getIntrinsicID())) {
2239 FunctionType *FT = F->getFunctionType();
2240 auto *NewST = StructType::get(ST->getContext(), ST->elements());
2241 auto *NewFT = FunctionType::get(NewST, FT->params(), FT->isVarArg());
2242 std::string Name = F->getName().str();
2243 rename(F);
2244 NewFn = Function::Create(NewFT, F->getLinkage(), F->getAddressSpace(),
2245 Name, F->getParent());
2246
2247 // The new function may also need remangling.
2248 if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(NewFn))
2249 NewFn = *Result;
2250 return true;
2251 }
2252 }
2253
2254 // Remangle our intrinsic since we upgrade the mangling
2255 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
2256 if (Result != std::nullopt) {
2257 NewFn = *Result;
2258 return true;
2259 }
2260
2261 // This may not belong here. This function is effectively being overloaded
2262 // to both detect an intrinsic which needs upgrading, and to provide the
2263 // upgraded form of the intrinsic. We should perhaps have two separate
2264 // functions for this.
2265 if (upgradeIntrinsicDeclWithDefaultArgs(F, NewFn))
2266 return true;
2267
2268 return false;
2269}
2270
2271bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn,
2272 bool CanUpgradeDebugIntrinsicsToRecords) {
2273 NewFn = nullptr;
2274 bool Upgraded =
2275 upgradeIntrinsicFunction1(F, NewFn, CanUpgradeDebugIntrinsicsToRecords);
2276
2277 // Upgrade intrinsic attributes. This does not change the function.
2278 if (NewFn)
2279 F = NewFn;
2280 if (Intrinsic::ID id = F->getIntrinsicID()) {
2281 // Only do this if the intrinsic signature is valid.
2282 SmallVector<Type *> OverloadTys;
2283 if (Intrinsic::isSignatureValid(id, F->getFunctionType(), OverloadTys))
2284 F->setAttributes(
2285 Intrinsic::getAttributes(F->getContext(), id, F->getFunctionType()));
2286 }
2287 return Upgraded;
2288}
2289
2290GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
2291 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
2292 GV->getName() == "llvm.global_dtors")) ||
2293 !GV->hasInitializer())
2294 return nullptr;
2295 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
2296 if (!ATy)
2297 return nullptr;
2298 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
2299 if (!STy || STy->getNumElements() != 2)
2300 return nullptr;
2301
2302 LLVMContext &C = GV->getContext();
2303 IRBuilder<> IRB(C);
2304 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
2305 IRB.getPtrTy());
2306 Constant *Init = GV->getInitializer();
2307 unsigned N = Init->getNumOperands();
2308 std::vector<Constant *> NewCtors(N);
2309 for (unsigned i = 0; i != N; ++i) {
2310 auto Ctor = cast<Constant>(Init->getOperand(i));
2311 NewCtors[i] = ConstantStruct::get(EltTy, Ctor->getAggregateElement(0u),
2312 Ctor->getAggregateElement(1),
2313 ConstantPointerNull::get(IRB.getPtrTy()));
2314 }
2315 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
2316
2317 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
2318 NewInit, GV->getName());
2319}
2320
2321// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
2322// to byte shuffles.
2323static Value *upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2324 unsigned Shift) {
2325 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2326 unsigned NumElts = ResultTy->getNumElements() * 8;
2327
2328 // Bitcast from a 64-bit element type to a byte element type.
2329 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2330 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2331
2332 // We'll be shuffling in zeroes.
2333 Value *Res = Constant::getNullValue(VecTy);
2334
2335 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2336 // we'll just return the zero vector.
2337 if (Shift < 16) {
2338 int Idxs[64];
2339 // 256/512-bit version is split into 2/4 16-byte lanes.
2340 for (unsigned l = 0; l != NumElts; l += 16)
2341 for (unsigned i = 0; i != 16; ++i) {
2342 unsigned Idx = NumElts + i - Shift;
2343 if (Idx < NumElts)
2344 Idx -= NumElts - 16; // end of lane, switch operand.
2345 Idxs[l + i] = Idx + l;
2346 }
2347
2348 Res = Builder.CreateShuffleVector(Res, Op, ArrayRef(Idxs, NumElts));
2349 }
2350
2351 // Bitcast back to a 64-bit element type.
2352 return Builder.CreateBitCast(Res, ResultTy, "cast");
2353}
2354
2355// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
2356// to byte shuffles.
2357static Value *upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
2358 unsigned Shift) {
2359 auto *ResultTy = cast<FixedVectorType>(Op->getType());
2360 unsigned NumElts = ResultTy->getNumElements() * 8;
2361
2362 // Bitcast from a 64-bit element type to a byte element type.
2363 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
2364 Op = Builder.CreateBitCast(Op, VecTy, "cast");
2365
2366 // We'll be shuffling in zeroes.
2367 Value *Res = Constant::getNullValue(VecTy);
2368
2369 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
2370 // we'll just return the zero vector.
2371 if (Shift < 16) {
2372 int Idxs[64];
2373 // 256/512-bit version is split into 2/4 16-byte lanes.
2374 for (unsigned l = 0; l != NumElts; l += 16)
2375 for (unsigned i = 0; i != 16; ++i) {
2376 unsigned Idx = i + Shift;
2377 if (Idx >= 16)
2378 Idx += NumElts - 16; // end of lane, switch operand.
2379 Idxs[l + i] = Idx + l;
2380 }
2381
2382 Res = Builder.CreateShuffleVector(Op, Res, ArrayRef(Idxs, NumElts));
2383 }
2384
2385 // Bitcast back to a 64-bit element type.
2386 return Builder.CreateBitCast(Res, ResultTy, "cast");
2387}
2388
2389static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
2390 unsigned NumElts) {
2391 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
2392 llvm::VectorType *MaskTy = FixedVectorType::get(
2393 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
2394 Mask = Builder.CreateBitCast(Mask, MaskTy);
2395
2396 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
2397 // i8 and we need to extract down to the right number of elements.
2398 if (NumElts <= 4) {
2399 int Indices[4];
2400 for (unsigned i = 0; i != NumElts; ++i)
2401 Indices[i] = i;
2402 Mask = Builder.CreateShuffleVector(Mask, Mask, ArrayRef(Indices, NumElts),
2403 "extract");
2404 }
2405
2406 return Mask;
2407}
2408
2409static Value *emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2410 Value *Op1) {
2411 // If the mask is all ones just emit the first operation.
2412 if (const auto *C = dyn_cast<Constant>(Mask))
2413 if (C->isAllOnesValue())
2414 return Op0;
2415
2416 Mask = getX86MaskVec(Builder, Mask,
2417 cast<FixedVectorType>(Op0->getType())->getNumElements());
2418 return Builder.CreateSelect(Mask, Op0, Op1);
2419}
2420
2421static Value *emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0,
2422 Value *Op1) {
2423 // If the mask is all ones just emit the first operation.
2424 if (const auto *C = dyn_cast<Constant>(Mask))
2425 if (C->isAllOnesValue())
2426 return Op0;
2427
2428 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
2429 Mask->getType()->getIntegerBitWidth());
2430 Mask = Builder.CreateBitCast(Mask, MaskTy);
2431 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
2432 return Builder.CreateSelect(Mask, Op0, Op1);
2433}
2434
2435// Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
2436// PALIGNR handles large immediates by shifting while VALIGN masks the immediate
2437// so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
2438static Value *upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
2439 Value *Op1, Value *Shift,
2440 Value *Passthru, Value *Mask,
2441 bool IsVALIGN) {
2442 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
2443
2444 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2445 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
2446 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
2447 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
2448
2449 // Mask the immediate for VALIGN.
2450 if (IsVALIGN)
2451 ShiftVal &= (NumElts - 1);
2452
2453 // If palignr is shifting the pair of vectors more than the size of two
2454 // lanes, emit zero.
2455 if (ShiftVal >= 32)
2456 return llvm::Constant::getNullValue(Op0->getType());
2457
2458 // If palignr is shifting the pair of input vectors more than one lane,
2459 // but less than two lanes, convert to shifting in zeroes.
2460 if (ShiftVal > 16) {
2461 ShiftVal -= 16;
2462 Op1 = Op0;
2463 Op0 = llvm::Constant::getNullValue(Op0->getType());
2464 }
2465
2466 int Indices[64];
2467 // 256-bit palignr operates on 128-bit lanes so we need to handle that
2468 for (unsigned l = 0; l < NumElts; l += 16) {
2469 for (unsigned i = 0; i != 16; ++i) {
2470 unsigned Idx = ShiftVal + i;
2471 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
2472 Idx += NumElts - 16; // End of lane, switch operand.
2473 Indices[l + i] = Idx + l;
2474 }
2475 }
2476
2477 Value *Align = Builder.CreateShuffleVector(
2478 Op1, Op0, ArrayRef(Indices, NumElts), "palignr");
2479
2480 return emitX86Select(Builder, Mask, Align, Passthru);
2481}
2482
2483static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
2484 bool ZeroMask, bool IndexForm) {
2485 Type *Ty = CI.getType();
2486 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
2487 unsigned EltWidth = Ty->getScalarSizeInBits();
2488 bool IsFloat = Ty->isFPOrFPVectorTy();
2489 Intrinsic::ID IID;
2490 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
2491 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
2492 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
2493 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
2494 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
2495 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
2496 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
2497 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
2498 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2499 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
2500 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2501 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
2502 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2503 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
2504 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2505 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
2506 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2507 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
2508 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2509 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
2510 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2511 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
2512 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2513 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
2514 else if (VecWidth == 128 && EltWidth == 16)
2515 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
2516 else if (VecWidth == 256 && EltWidth == 16)
2517 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
2518 else if (VecWidth == 512 && EltWidth == 16)
2519 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
2520 else if (VecWidth == 128 && EltWidth == 8)
2521 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
2522 else if (VecWidth == 256 && EltWidth == 8)
2523 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
2524 else if (VecWidth == 512 && EltWidth == 8)
2525 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
2526 else
2527 llvm_unreachable("Unexpected intrinsic");
2528
2529 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
2530 CI.getArgOperand(2) };
2531
2532 // If this isn't index form we need to swap operand 0 and 1.
2533 if (!IndexForm)
2534 std::swap(Args[0], Args[1]);
2535
2536 Value *V = Builder.CreateIntrinsic(IID, Args);
2537 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
2538 : Builder.CreateBitCast(CI.getArgOperand(1),
2539 Ty);
2540 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
2541}
2542
2543static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
2544 Intrinsic::ID IID) {
2545 Type *Ty = CI.getType();
2546 Value *Op0 = CI.getOperand(0);
2547 Value *Op1 = CI.getOperand(1);
2548 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1});
2549
2550 if (CI.arg_size() == 4) { // For masked intrinsics.
2551 Value *VecSrc = CI.getOperand(2);
2552 Value *Mask = CI.getOperand(3);
2553 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2554 }
2555 return Res;
2556}
2557
2558static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
2559 bool IsRotateRight) {
2560 Type *Ty = CI.getType();
2561 Value *Src = CI.getArgOperand(0);
2562 Value *Amt = CI.getArgOperand(1);
2563
2564 // Amount may be scalar immediate, in which case create a splat vector.
2565 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2566 // we only care about the lowest log2 bits anyway.
2567 if (Amt->getType() != Ty) {
2568 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2569 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2570 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2571 }
2572
2573 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2574 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Src, Src, Amt});
2575
2576 if (CI.arg_size() == 4) { // For masked intrinsics.
2577 Value *VecSrc = CI.getOperand(2);
2578 Value *Mask = CI.getOperand(3);
2579 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2580 }
2581 return Res;
2582}
2583
2584static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
2585 bool IsSigned) {
2586 Type *Ty = CI.getType();
2587 Value *LHS = CI.getArgOperand(0);
2588 Value *RHS = CI.getArgOperand(1);
2589
2590 CmpInst::Predicate Pred;
2591 switch (Imm) {
2592 case 0x0:
2593 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
2594 break;
2595 case 0x1:
2596 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2597 break;
2598 case 0x2:
2599 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
2600 break;
2601 case 0x3:
2602 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
2603 break;
2604 case 0x4:
2605 Pred = ICmpInst::ICMP_EQ;
2606 break;
2607 case 0x5:
2608 Pred = ICmpInst::ICMP_NE;
2609 break;
2610 case 0x6:
2611 return Constant::getNullValue(Ty); // FALSE
2612 case 0x7:
2613 return Constant::getAllOnesValue(Ty); // TRUE
2614 default:
2615 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
2616 }
2617
2618 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
2619 Value *Ext = Builder.CreateSExt(Cmp, Ty);
2620 return Ext;
2621}
2622
2623static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
2624 bool IsShiftRight, bool ZeroMask) {
2625 Type *Ty = CI.getType();
2626 Value *Op0 = CI.getArgOperand(0);
2627 Value *Op1 = CI.getArgOperand(1);
2628 Value *Amt = CI.getArgOperand(2);
2629
2630 if (IsShiftRight)
2631 std::swap(Op0, Op1);
2632
2633 // Amount may be scalar immediate, in which case create a splat vector.
2634 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
2635 // we only care about the lowest log2 bits anyway.
2636 if (Amt->getType() != Ty) {
2637 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
2638 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
2639 Amt = Builder.CreateVectorSplat(NumElts, Amt);
2640 }
2641
2642 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
2643 Value *Res = Builder.CreateIntrinsic(IID, Ty, {Op0, Op1, Amt});
2644
2645 unsigned NumArgs = CI.arg_size();
2646 if (NumArgs >= 4) { // For masked intrinsics.
2647 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
2648 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
2649 CI.getArgOperand(0);
2650 Value *Mask = CI.getOperand(NumArgs - 1);
2651 Res = emitX86Select(Builder, Mask, Res, VecSrc);
2652 }
2653 return Res;
2654}
2655
2656static Value *upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data,
2657 Value *Mask, bool Aligned) {
2658 const Align Alignment =
2659 Aligned
2660 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedValue() / 8)
2661 : Align(1);
2662
2663 // If the mask is all ones just emit a regular store.
2664 if (const auto *C = dyn_cast<Constant>(Mask))
2665 if (C->isAllOnesValue())
2666 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
2667
2668 // Convert the mask from an integer type to a vector of i1.
2669 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
2670 Mask = getX86MaskVec(Builder, Mask, NumElts);
2671 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
2672}
2673
2674static Value *upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr,
2675 Value *Passthru, Value *Mask, bool Aligned) {
2676 Type *ValTy = Passthru->getType();
2677 const Align Alignment =
2678 Aligned
2679 ? Align(
2680 Passthru->getType()->getPrimitiveSizeInBits().getFixedValue() /
2681 8)
2682 : Align(1);
2683
2684 // If the mask is all ones just emit a regular store.
2685 if (const auto *C = dyn_cast<Constant>(Mask))
2686 if (C->isAllOnesValue())
2687 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
2688
2689 // Convert the mask from an integer type to a vector of i1.
2690 unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
2691 Mask = getX86MaskVec(Builder, Mask, NumElts);
2692 return Builder.CreateMaskedLoad(ValTy, Ptr, Alignment, Mask, Passthru);
2693}
2694
2695static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
2696 Type *Ty = CI.getType();
2697 Value *Op0 = CI.getArgOperand(0);
2698 Value *Res = Builder.CreateIntrinsic(Intrinsic::abs, Ty,
2699 {Op0, Builder.getInt1(false)});
2700 if (CI.arg_size() == 3)
2701 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
2702 return Res;
2703}
2704
2705static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
2706 Type *Ty = CI.getType();
2707
2708 // Arguments have a vXi32 type so cast to vXi64.
2709 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
2710 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
2711
2712 if (IsSigned) {
2713 // Shift left then arithmetic shift right.
2714 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
2715 LHS = Builder.CreateShl(LHS, ShiftAmt);
2716 LHS = Builder.CreateAShr(LHS, ShiftAmt);
2717 RHS = Builder.CreateShl(RHS, ShiftAmt);
2718 RHS = Builder.CreateAShr(RHS, ShiftAmt);
2719 } else {
2720 // Clear the upper bits.
2721 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
2722 LHS = Builder.CreateAnd(LHS, Mask);
2723 RHS = Builder.CreateAnd(RHS, Mask);
2724 }
2725
2726 Value *Res = Builder.CreateMul(LHS, RHS);
2727
2728 if (CI.arg_size() == 4)
2729 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
2730
2731 return Res;
2732}
2733
2734// Applying mask on vector of i1's and make sure result is at least 8 bits wide.
2735static Value *applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
2736 Value *Mask) {
2737 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
2738 if (Mask) {
2739 const auto *C = dyn_cast<Constant>(Mask);
2740 if (!C || !C->isAllOnesValue())
2741 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
2742 }
2743
2744 if (NumElts < 8) {
2745 int Indices[8];
2746 for (unsigned i = 0; i != NumElts; ++i)
2747 Indices[i] = i;
2748 for (unsigned i = NumElts; i != 8; ++i)
2749 Indices[i] = NumElts + i % NumElts;
2750 Vec = Builder.CreateShuffleVector(Vec,
2751 Constant::getNullValue(Vec->getType()),
2752 Indices);
2753 }
2754 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
2755}
2756
2757static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
2758 unsigned CC, bool Signed) {
2759 Value *Op0 = CI.getArgOperand(0);
2760 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
2761
2762 Value *Cmp;
2763 if (CC == 3) {
2764 Cmp = Constant::getNullValue(
2765 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2766 } else if (CC == 7) {
2767 Cmp = Constant::getAllOnesValue(
2768 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
2769 } else {
2770 ICmpInst::Predicate Pred;
2771 switch (CC) {
2772 default: llvm_unreachable("Unknown condition code");
2773 case 0: Pred = ICmpInst::ICMP_EQ; break;
2774 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
2775 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
2776 case 4: Pred = ICmpInst::ICMP_NE; break;
2777 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
2778 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
2779 }
2780 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
2781 }
2782
2783 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
2784
2785 return applyX86MaskOn1BitsVec(Builder, Cmp, Mask);
2786}
2787
2788// Replace a masked intrinsic with an older unmasked intrinsic.
2789static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
2790 Intrinsic::ID IID) {
2791 Value *Rep =
2792 Builder.CreateIntrinsic(IID, {CI.getArgOperand(0), CI.getArgOperand(1)});
2793 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
2794}
2795
2796static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
2797 Value* A = CI.getArgOperand(0);
2798 Value* B = CI.getArgOperand(1);
2799 Value* Src = CI.getArgOperand(2);
2800 Value* Mask = CI.getArgOperand(3);
2801
2802 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
2803 Value* Cmp = Builder.CreateIsNotNull(AndNode);
2804 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
2805 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
2806 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
2807 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
2808}
2809
2810static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
2811 Value* Op = CI.getArgOperand(0);
2812 Type* ReturnOp = CI.getType();
2813 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2814 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
2815 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
2816}
2817
2818// Replace intrinsic with unmasked version and a select.
2819static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
2820 CallBase &CI, Value *&Rep) {
2821 Name = Name.substr(12); // Remove avx512.mask.
2822
2823 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2824 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2825 Intrinsic::ID IID;
2826 if (Name.starts_with("max.p")) {
2827 if (VecWidth == 128 && EltWidth == 32)
2828 IID = Intrinsic::x86_sse_max_ps;
2829 else if (VecWidth == 128 && EltWidth == 64)
2830 IID = Intrinsic::x86_sse2_max_pd;
2831 else if (VecWidth == 256 && EltWidth == 32)
2832 IID = Intrinsic::x86_avx_max_ps_256;
2833 else if (VecWidth == 256 && EltWidth == 64)
2834 IID = Intrinsic::x86_avx_max_pd_256;
2835 else
2836 llvm_unreachable("Unexpected intrinsic");
2837 } else if (Name.starts_with("min.p")) {
2838 if (VecWidth == 128 && EltWidth == 32)
2839 IID = Intrinsic::x86_sse_min_ps;
2840 else if (VecWidth == 128 && EltWidth == 64)
2841 IID = Intrinsic::x86_sse2_min_pd;
2842 else if (VecWidth == 256 && EltWidth == 32)
2843 IID = Intrinsic::x86_avx_min_ps_256;
2844 else if (VecWidth == 256 && EltWidth == 64)
2845 IID = Intrinsic::x86_avx_min_pd_256;
2846 else
2847 llvm_unreachable("Unexpected intrinsic");
2848 } else if (Name.starts_with("pshuf.b.")) {
2849 if (VecWidth == 128)
2850 IID = Intrinsic::x86_ssse3_pshuf_b_128;
2851 else if (VecWidth == 256)
2852 IID = Intrinsic::x86_avx2_pshuf_b;
2853 else if (VecWidth == 512)
2854 IID = Intrinsic::x86_avx512_pshuf_b_512;
2855 else
2856 llvm_unreachable("Unexpected intrinsic");
2857 } else if (Name.starts_with("pmul.hr.sw.")) {
2858 if (VecWidth == 128)
2859 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
2860 else if (VecWidth == 256)
2861 IID = Intrinsic::x86_avx2_pmul_hr_sw;
2862 else if (VecWidth == 512)
2863 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
2864 else
2865 llvm_unreachable("Unexpected intrinsic");
2866 } else if (Name.starts_with("pmulh.w.")) {
2867 if (VecWidth == 128)
2868 IID = Intrinsic::x86_sse2_pmulh_w;
2869 else if (VecWidth == 256)
2870 IID = Intrinsic::x86_avx2_pmulh_w;
2871 else if (VecWidth == 512)
2872 IID = Intrinsic::x86_avx512_pmulh_w_512;
2873 else
2874 llvm_unreachable("Unexpected intrinsic");
2875 } else if (Name.starts_with("pmulhu.w.")) {
2876 if (VecWidth == 128)
2877 IID = Intrinsic::x86_sse2_pmulhu_w;
2878 else if (VecWidth == 256)
2879 IID = Intrinsic::x86_avx2_pmulhu_w;
2880 else if (VecWidth == 512)
2881 IID = Intrinsic::x86_avx512_pmulhu_w_512;
2882 else
2883 llvm_unreachable("Unexpected intrinsic");
2884 } else if (Name.starts_with("pmaddw.d.")) {
2885 if (VecWidth == 128)
2886 IID = Intrinsic::x86_sse2_pmadd_wd;
2887 else if (VecWidth == 256)
2888 IID = Intrinsic::x86_avx2_pmadd_wd;
2889 else if (VecWidth == 512)
2890 IID = Intrinsic::x86_avx512_pmaddw_d_512;
2891 else
2892 llvm_unreachable("Unexpected intrinsic");
2893 } else if (Name.starts_with("pmaddubs.w.")) {
2894 if (VecWidth == 128)
2895 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
2896 else if (VecWidth == 256)
2897 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
2898 else if (VecWidth == 512)
2899 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
2900 else
2901 llvm_unreachable("Unexpected intrinsic");
2902 } else if (Name.starts_with("packsswb.")) {
2903 if (VecWidth == 128)
2904 IID = Intrinsic::x86_sse2_packsswb_128;
2905 else if (VecWidth == 256)
2906 IID = Intrinsic::x86_avx2_packsswb;
2907 else if (VecWidth == 512)
2908 IID = Intrinsic::x86_avx512_packsswb_512;
2909 else
2910 llvm_unreachable("Unexpected intrinsic");
2911 } else if (Name.starts_with("packssdw.")) {
2912 if (VecWidth == 128)
2913 IID = Intrinsic::x86_sse2_packssdw_128;
2914 else if (VecWidth == 256)
2915 IID = Intrinsic::x86_avx2_packssdw;
2916 else if (VecWidth == 512)
2917 IID = Intrinsic::x86_avx512_packssdw_512;
2918 else
2919 llvm_unreachable("Unexpected intrinsic");
2920 } else if (Name.starts_with("packuswb.")) {
2921 if (VecWidth == 128)
2922 IID = Intrinsic::x86_sse2_packuswb_128;
2923 else if (VecWidth == 256)
2924 IID = Intrinsic::x86_avx2_packuswb;
2925 else if (VecWidth == 512)
2926 IID = Intrinsic::x86_avx512_packuswb_512;
2927 else
2928 llvm_unreachable("Unexpected intrinsic");
2929 } else if (Name.starts_with("packusdw.")) {
2930 if (VecWidth == 128)
2931 IID = Intrinsic::x86_sse41_packusdw;
2932 else if (VecWidth == 256)
2933 IID = Intrinsic::x86_avx2_packusdw;
2934 else if (VecWidth == 512)
2935 IID = Intrinsic::x86_avx512_packusdw_512;
2936 else
2937 llvm_unreachable("Unexpected intrinsic");
2938 } else if (Name.starts_with("vpermilvar.")) {
2939 if (VecWidth == 128 && EltWidth == 32)
2940 IID = Intrinsic::x86_avx_vpermilvar_ps;
2941 else if (VecWidth == 128 && EltWidth == 64)
2942 IID = Intrinsic::x86_avx_vpermilvar_pd;
2943 else if (VecWidth == 256 && EltWidth == 32)
2944 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
2945 else if (VecWidth == 256 && EltWidth == 64)
2946 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
2947 else if (VecWidth == 512 && EltWidth == 32)
2948 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
2949 else if (VecWidth == 512 && EltWidth == 64)
2950 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
2951 else
2952 llvm_unreachable("Unexpected intrinsic");
2953 } else if (Name == "cvtpd2dq.256") {
2954 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
2955 } else if (Name == "cvtpd2ps.256") {
2956 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
2957 } else if (Name == "cvttpd2dq.256") {
2958 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
2959 } else if (Name == "cvttps2dq.128") {
2960 IID = Intrinsic::x86_sse2_cvttps2dq;
2961 } else if (Name == "cvttps2dq.256") {
2962 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
2963 } else if (Name.starts_with("permvar.")) {
2964 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2965 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
2966 IID = Intrinsic::x86_avx2_permps;
2967 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
2968 IID = Intrinsic::x86_avx2_permd;
2969 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
2970 IID = Intrinsic::x86_avx512_permvar_df_256;
2971 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
2972 IID = Intrinsic::x86_avx512_permvar_di_256;
2973 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
2974 IID = Intrinsic::x86_avx512_permvar_sf_512;
2975 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
2976 IID = Intrinsic::x86_avx512_permvar_si_512;
2977 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
2978 IID = Intrinsic::x86_avx512_permvar_df_512;
2979 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
2980 IID = Intrinsic::x86_avx512_permvar_di_512;
2981 else if (VecWidth == 128 && EltWidth == 16)
2982 IID = Intrinsic::x86_avx512_permvar_hi_128;
2983 else if (VecWidth == 256 && EltWidth == 16)
2984 IID = Intrinsic::x86_avx512_permvar_hi_256;
2985 else if (VecWidth == 512 && EltWidth == 16)
2986 IID = Intrinsic::x86_avx512_permvar_hi_512;
2987 else if (VecWidth == 128 && EltWidth == 8)
2988 IID = Intrinsic::x86_avx512_permvar_qi_128;
2989 else if (VecWidth == 256 && EltWidth == 8)
2990 IID = Intrinsic::x86_avx512_permvar_qi_256;
2991 else if (VecWidth == 512 && EltWidth == 8)
2992 IID = Intrinsic::x86_avx512_permvar_qi_512;
2993 else
2994 llvm_unreachable("Unexpected intrinsic");
2995 } else if (Name.starts_with("dbpsadbw.")) {
2996 if (VecWidth == 128)
2997 IID = Intrinsic::x86_avx512_dbpsadbw_128;
2998 else if (VecWidth == 256)
2999 IID = Intrinsic::x86_avx512_dbpsadbw_256;
3000 else if (VecWidth == 512)
3001 IID = Intrinsic::x86_avx512_dbpsadbw_512;
3002 else
3003 llvm_unreachable("Unexpected intrinsic");
3004 } else if (Name.starts_with("pmultishift.qb.")) {
3005 if (VecWidth == 128)
3006 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
3007 else if (VecWidth == 256)
3008 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
3009 else if (VecWidth == 512)
3010 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
3011 else
3012 llvm_unreachable("Unexpected intrinsic");
3013 } else if (Name.starts_with("conflict.")) {
3014 if (Name[9] == 'd' && VecWidth == 128)
3015 IID = Intrinsic::x86_avx512_conflict_d_128;
3016 else if (Name[9] == 'd' && VecWidth == 256)
3017 IID = Intrinsic::x86_avx512_conflict_d_256;
3018 else if (Name[9] == 'd' && VecWidth == 512)
3019 IID = Intrinsic::x86_avx512_conflict_d_512;
3020 else if (Name[9] == 'q' && VecWidth == 128)
3021 IID = Intrinsic::x86_avx512_conflict_q_128;
3022 else if (Name[9] == 'q' && VecWidth == 256)
3023 IID = Intrinsic::x86_avx512_conflict_q_256;
3024 else if (Name[9] == 'q' && VecWidth == 512)
3025 IID = Intrinsic::x86_avx512_conflict_q_512;
3026 else
3027 llvm_unreachable("Unexpected intrinsic");
3028 } else if (Name.starts_with("pavg.")) {
3029 if (Name[5] == 'b' && VecWidth == 128)
3030 IID = Intrinsic::x86_sse2_pavg_b;
3031 else if (Name[5] == 'b' && VecWidth == 256)
3032 IID = Intrinsic::x86_avx2_pavg_b;
3033 else if (Name[5] == 'b' && VecWidth == 512)
3034 IID = Intrinsic::x86_avx512_pavg_b_512;
3035 else if (Name[5] == 'w' && VecWidth == 128)
3036 IID = Intrinsic::x86_sse2_pavg_w;
3037 else if (Name[5] == 'w' && VecWidth == 256)
3038 IID = Intrinsic::x86_avx2_pavg_w;
3039 else if (Name[5] == 'w' && VecWidth == 512)
3040 IID = Intrinsic::x86_avx512_pavg_w_512;
3041 else
3042 llvm_unreachable("Unexpected intrinsic");
3043 } else
3044 return false;
3045
3046 SmallVector<Value *, 4> Args(CI.args());
3047 Args.pop_back();
3048 Args.pop_back();
3049 Rep = Builder.CreateIntrinsic(IID, Args);
3050 unsigned NumArgs = CI.arg_size();
3051 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
3052 CI.getArgOperand(NumArgs - 2));
3053 return true;
3054}
3055
3056/// Upgrade comment in call to inline asm that represents an objc retain release
3057/// marker.
3058void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
3059 size_t Pos;
3060 if (AsmStr->find("mov\tfp") == 0 &&
3061 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
3062 (Pos = AsmStr->find("# marker")) != std::string::npos) {
3063 AsmStr->replace(Pos, 1, ";");
3064 }
3065}
3066
3067static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
3068 Function *F, IRBuilder<> &Builder) {
3069 Value *Rep = nullptr;
3070
3071 if (Name == "abs.i" || Name == "abs.ll") {
3072 Value *Arg = CI->getArgOperand(0);
3073 Rep = Builder.CreateIntrinsic(Intrinsic::abs, {Arg->getType()},
3074 {Arg, Builder.getTrue()},
3075 /*FMFSource=*/nullptr, "abs");
3076 } else if (Name == "abs.bf16" || Name == "abs.bf16x2") {
3077 Type *Ty = (Name == "abs.bf16")
3078 ? Builder.getBFloatTy()
3079 : FixedVectorType::get(Builder.getBFloatTy(), 2);
3080 Value *Arg = Builder.CreateBitCast(CI->getArgOperand(0), Ty);
3081 Value *Abs = Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_fabs, Arg);
3082 Rep = Builder.CreateBitCast(Abs, CI->getType());
3083 } else if (Name == "fabs.f" || Name == "fabs.ftz.f" || Name == "fabs.d") {
3084 Intrinsic::ID IID = (Name == "fabs.ftz.f") ? Intrinsic::nvvm_fabs_ftz
3085 : Intrinsic::nvvm_fabs;
3086 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3087 } else if (Name.consume_front("ex2.approx.")) {
3088 // nvvm.ex2.approx.{f,ftz.f,d,f16x2}
3089 Intrinsic::ID IID = Name.starts_with("ftz") ? Intrinsic::nvvm_ex2_approx_ftz
3090 : Intrinsic::nvvm_ex2_approx;
3091 Rep = Builder.CreateUnaryIntrinsic(IID, CI->getArgOperand(0));
3092 } else if (Name.starts_with("atomic.load.add.f32.p") ||
3093 Name.starts_with("atomic.load.add.f64.p")) {
3094 Value *Ptr = CI->getArgOperand(0);
3095 Value *Val = CI->getArgOperand(1);
3096 Rep = Builder.CreateAtomicRMW(
3097 AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3098 CI->getContext().getOrInsertSyncScopeID("device"));
3099 // The default scope for atomic.load.* intrinsics is device
3100 // (= gpu scope in ptx), but the default LLVM atomic scope is
3101 // "system"
3102 } else if (Name.starts_with("atomic.load.inc.32.p") ||
3103 Name.starts_with("atomic.load.dec.32.p")) {
3104 Value *Ptr = CI->getArgOperand(0);
3105 Value *Val = CI->getArgOperand(1);
3106 auto Op = Name.starts_with("atomic.load.inc") ? AtomicRMWInst::UIncWrap
3107 : AtomicRMWInst::UDecWrap;
3108 Rep = Builder.CreateAtomicRMW(
3109 Op, Ptr, Val, MaybeAlign(), AtomicOrdering::Monotonic,
3110 CI->getContext().getOrInsertSyncScopeID("device"));
3111 // See comment above.
3112 } else if (Name.starts_with("atomic.") && Name.contains(".gen.")) {
3113 // nvvm.atomic.{op}.gen.{i,f}.{cta,sys} -> atomicrmw / cmpxchg.
3114 StringRef Op = Name.substr(StringRef("atomic.").size());
3115 Value *Ptr = CI->getArgOperand(0);
3116 Value *Val = CI->getArgOperand(1);
3117 SyncScope::ID SSID = CI->getContext().getOrInsertSyncScopeID(
3118 Op.contains(".cta.") ? "block" : "");
3119 if (Op.starts_with("cas.")) {
3120 Value *New = CI->getArgOperand(2);
3121 Value *Pair = Builder.CreateAtomicCmpXchg(
3122 Ptr, Val, New, MaybeAlign(), AtomicOrdering::Monotonic,
3123 AtomicOrdering::Monotonic, SSID);
3124 Rep = Builder.CreateExtractValue(Pair, 0);
3125 } else {
3126 // Note we don't upgrade anything to AtomicRMWInst::UMin/UMax. This is
3127 // because we were actually missing those intrinsics!
3128 AtomicRMWInst::BinOp BinOp =
3129 StringSwitch<AtomicRMWInst::BinOp>(Op)
3130 .StartsWith("add.gen.f", AtomicRMWInst::FAdd)
3131 .StartsWith("add.gen.i", AtomicRMWInst::Add)
3132 .StartsWith("exch.", AtomicRMWInst::Xchg)
3133 .StartsWith("max.", AtomicRMWInst::Max)
3134 .StartsWith("min.", AtomicRMWInst::Min)
3135 .StartsWith("inc.", AtomicRMWInst::UIncWrap)
3136 .StartsWith("dec.", AtomicRMWInst::UDecWrap)
3137 .StartsWith("and.", AtomicRMWInst::And)
3138 .StartsWith("or.", AtomicRMWInst::Or)
3139 .StartsWith("xor.", AtomicRMWInst::Xor)
3140 .Default(AtomicRMWInst::BAD_BINOP);
3141 assert(BinOp != AtomicRMWInst::BAD_BINOP &&
3142 "unexpected nvvm scoped atomic intrinsic");
3143 Rep = Builder.CreateAtomicRMW(BinOp, Ptr, Val, MaybeAlign(),
3144 AtomicOrdering::Monotonic, SSID);
3145 }
3146 } else if (Name == "clz.ll") {
3147 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
3148 Value *Arg = CI->getArgOperand(0);
3149 Value *Ctlz = Builder.CreateIntrinsic(Intrinsic::ctlz, {Arg->getType()},
3150 {Arg, Builder.getFalse()},
3151 /*FMFSource=*/nullptr, "ctlz");
3152 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3153 } else if (Name == "popc.ll") {
3154 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
3155 // i64.
3156 Value *Arg = CI->getArgOperand(0);
3157 Value *Popc = Builder.CreateIntrinsic(Intrinsic::ctpop, {Arg->getType()},
3158 Arg, /*FMFSource=*/nullptr, "ctpop");
3159 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3160 } else if (Name == "h2f") {
3161 Value *Cast =
3162 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
3163 Rep = Builder.CreateFPExt(Cast, Builder.getFloatTy());
3164 } else if (Name.consume_front("bitcast.") &&
3165 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
3166 Name == "d2ll")) {
3167 Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
3168 } else if (Name == "rotate.b32") {
3169 Value *Arg = CI->getOperand(0);
3170 Value *ShiftAmt = CI->getOperand(1);
3171 Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
3172 {Arg, Arg, ShiftAmt});
3173 } else if (Name == "rotate.b64") {
3174 Type *Int64Ty = Builder.getInt64Ty();
3175 Value *Arg = CI->getOperand(0);
3176 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3177 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3178 {Arg, Arg, ZExtShiftAmt});
3179 } else if (Name == "rotate.right.b64") {
3180 Type *Int64Ty = Builder.getInt64Ty();
3181 Value *Arg = CI->getOperand(0);
3182 Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
3183 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
3184 {Arg, Arg, ZExtShiftAmt});
3185 } else if (Name == "swap.lo.hi.b64") {
3186 Type *Int64Ty = Builder.getInt64Ty();
3187 Value *Arg = CI->getOperand(0);
3188 Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
3189 {Arg, Arg, Builder.getInt64(32)});
3190 } else if ((Name.consume_front("ptr.gen.to.") &&
3191 consumeNVVMPtrAddrSpace(Name)) ||
3192 (Name.consume_front("ptr.") && consumeNVVMPtrAddrSpace(Name) &&
3193 Name.starts_with(".to.gen"))) {
3194 Rep = Builder.CreateAddrSpaceCast(CI->getArgOperand(0), CI->getType());
3195 } else if (Name.consume_front("ldg.global")) {
3196 Value *Ptr = CI->getArgOperand(0);
3197 Align PtrAlign = cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
3198 // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL
3199 Value *ASC = Builder.CreateAddrSpaceCast(Ptr, Builder.getPtrTy(1));
3200 Instruction *LD = Builder.CreateAlignedLoad(CI->getType(), ASC, PtrAlign);
3201 MDNode *MD = MDNode::get(Builder.getContext(), {});
3202 LD->setMetadata(LLVMContext::MD_invariant_load, MD);
3203 return LD;
3204 } else if (Name == "tanh.approx.f32") {
3205 // nvvm.tanh.approx.f32 -> afn llvm.tanh.f32
3206 FastMathFlags FMF;
3207 FMF.setApproxFunc();
3208 Rep = Builder.CreateUnaryIntrinsic(Intrinsic::tanh, CI->getArgOperand(0),
3209 FMF);
3210 } else if (Name == "barrier0" || Name == "barrier.n" || Name == "bar.sync") {
3211 Value *Arg =
3212 Name.ends_with('0') ? Builder.getInt32(0) : CI->getArgOperand(0);
3213 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_aligned_all,
3214 {}, {Arg});
3215 } else if (Name == "barrier") {
3216 Rep = Builder.CreateIntrinsic(
3217 Intrinsic::nvvm_barrier_cta_sync_aligned_count, {},
3218 {CI->getArgOperand(0), CI->getArgOperand(1)});
3219 } else if (Name == "barrier.sync") {
3220 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_all, {},
3221 {CI->getArgOperand(0)});
3222 } else if (Name == "barrier.sync.cnt") {
3223 Rep = Builder.CreateIntrinsic(Intrinsic::nvvm_barrier_cta_sync_count, {},
3224 {CI->getArgOperand(0), CI->getArgOperand(1)});
3225 } else if (Name == "barrier0.popc" || Name == "barrier0.and" ||
3226 Name == "barrier0.or") {
3227 Value *C = CI->getArgOperand(0);
3228 C = Builder.CreateICmpNE(C, Builder.getInt32(0));
3229
3230 Intrinsic::ID IID =
3231 StringSwitch<Intrinsic::ID>(Name)
3232 .Case("barrier0.popc",
3233 Intrinsic::nvvm_barrier_cta_red_popc_aligned_all)
3234 .Case("barrier0.and",
3235 Intrinsic::nvvm_barrier_cta_red_and_aligned_all)
3236 .Case("barrier0.or",
3237 Intrinsic::nvvm_barrier_cta_red_or_aligned_all);
3238 Value *Bar = Builder.CreateIntrinsic(IID, {}, {Builder.getInt32(0), C});
3239 Rep = Builder.CreateZExt(Bar, CI->getType());
3240 } else {
3241 Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
3242 if (IID != Intrinsic::not_intrinsic &&
3243 isLegacyNVPTXBF16IntSignature(F, IID)) {
3244 rename(F);
3245 Function *NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
3246 SmallVector<Value *, 2> Args;
3247 for (size_t I = 0; I < NewFn->arg_size(); ++I) {
3248 Value *Arg = CI->getArgOperand(I);
3249 Type *OldType = Arg->getType();
3250 Type *NewType = NewFn->getArg(I)->getType();
3251 Args.push_back(
3252 (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
3253 ? Builder.CreateBitCast(Arg, NewType)
3254 : Arg);
3255 }
3256 Rep = Builder.CreateCall(NewFn, Args);
3257 if (F->getReturnType()->isIntegerTy())
3258 Rep = Builder.CreateBitCast(Rep, F->getReturnType());
3259 }
3260 }
3261
3262 return Rep;
3263}
3264
3265static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3266 IRBuilder<> &Builder) {
3267 LLVMContext &C = F->getContext();
3268 Value *Rep = nullptr;
3269
3270 if (Name.starts_with("sse4a.movnt.")) {
3271 SmallVector<Metadata *, 1> Elts;
3272 Elts.push_back(
3273 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3274 MDNode *Node = MDNode::get(C, Elts);
3275
3276 Value *Arg0 = CI->getArgOperand(0);
3277 Value *Arg1 = CI->getArgOperand(1);
3278
3279 // Nontemporal (unaligned) store of the 0'th element of the float/double
3280 // vector.
3281 Value *Extract =
3282 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
3283
3284 StoreInst *SI = Builder.CreateAlignedStore(Extract, Arg0, Align(1));
3285 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3286 } else if (Name.starts_with("avx.movnt.") ||
3287 Name.starts_with("avx512.storent.")) {
3288 SmallVector<Metadata *, 1> Elts;
3289 Elts.push_back(
3290 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3291 MDNode *Node = MDNode::get(C, Elts);
3292
3293 Value *Arg0 = CI->getArgOperand(0);
3294 Value *Arg1 = CI->getArgOperand(1);
3295
3296 StoreInst *SI = Builder.CreateAlignedStore(
3297 Arg1, Arg0,
3298 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3299 SI->setMetadata(LLVMContext::MD_nontemporal, Node);
3300 } else if (Name == "sse2.storel.dq") {
3301 Value *Arg0 = CI->getArgOperand(0);
3302 Value *Arg1 = CI->getArgOperand(1);
3303
3304 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3305 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3306 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
3307 Builder.CreateAlignedStore(Elt, Arg0, Align(1));
3308 } else if (Name.starts_with("sse.storeu.") ||
3309 Name.starts_with("sse2.storeu.") ||
3310 Name.starts_with("avx.storeu.")) {
3311 Value *Arg0 = CI->getArgOperand(0);
3312 Value *Arg1 = CI->getArgOperand(1);
3313 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
3314 } else if (Name == "avx512.mask.store.ss") {
3315 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
3316 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3317 Mask, false);
3318 } else if (Name.starts_with("avx512.mask.store")) {
3319 // "avx512.mask.storeu." or "avx512.mask.store."
3320 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
3321 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3322 CI->getArgOperand(2), Aligned);
3323 } else if (Name.starts_with("sse2.pcmp") || Name.starts_with("avx2.pcmp")) {
3324 // Upgrade packed integer vector compare intrinsics to compare instructions.
3325 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
3326 bool CmpEq = Name[9] == 'e';
3327 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
3328 CI->getArgOperand(0), CI->getArgOperand(1));
3329 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
3330 } else if (Name.starts_with("avx512.broadcastm")) {
3331 Type *ExtTy = Type::getInt32Ty(C);
3332 if (CI->getOperand(0)->getType()->isIntegerTy(8))
3333 ExtTy = Type::getInt64Ty(C);
3334 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
3335 ExtTy->getPrimitiveSizeInBits();
3336 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
3337 Rep = Builder.CreateVectorSplat(NumElts, Rep);
3338 } else if (Name == "sse.sqrt.ss" || Name == "sse2.sqrt.sd") {
3339 Value *Vec = CI->getArgOperand(0);
3340 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
3341 Elt0 = Builder.CreateIntrinsic(Intrinsic::sqrt, Elt0->getType(), Elt0);
3342 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
3343 } else if (Name.starts_with("avx.sqrt.p") ||
3344 Name.starts_with("sse2.sqrt.p") ||
3345 Name.starts_with("sse.sqrt.p")) {
3346 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3347 {CI->getArgOperand(0)});
3348 } else if (Name.starts_with("avx512.mask.sqrt.p")) {
3349 if (CI->arg_size() == 4 &&
3350 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3351 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3352 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
3353 : Intrinsic::x86_avx512_sqrt_pd_512;
3354
3355 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
3356 Rep = Builder.CreateIntrinsic(IID, Args);
3357 } else {
3358 Rep = Builder.CreateIntrinsic(Intrinsic::sqrt, CI->getType(),
3359 {CI->getArgOperand(0)});
3360 }
3361 Rep =
3362 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3363 } else if (Name.starts_with("avx512.ptestm") ||
3364 Name.starts_with("avx512.ptestnm")) {
3365 Value *Op0 = CI->getArgOperand(0);
3366 Value *Op1 = CI->getArgOperand(1);
3367 Value *Mask = CI->getArgOperand(2);
3368 Rep = Builder.CreateAnd(Op0, Op1);
3369 llvm::Type *Ty = Op0->getType();
3370 Value *Zero = llvm::Constant::getNullValue(Ty);
3371 ICmpInst::Predicate Pred = Name.starts_with("avx512.ptestm")
3372 ? ICmpInst::ICMP_NE
3373 : ICmpInst::ICMP_EQ;
3374 Rep = Builder.CreateICmp(Pred, Rep, Zero);
3375 Rep = applyX86MaskOn1BitsVec(Builder, Rep, Mask);
3376 } else if (Name.starts_with("avx512.mask.pbroadcast")) {
3377 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
3378 ->getNumElements();
3379 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
3380 Rep =
3381 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3382 } else if (Name.starts_with("avx512.kunpck")) {
3383 unsigned NumElts = CI->getType()->getScalarSizeInBits();
3384 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
3385 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
3386 int Indices[64];
3387 for (unsigned i = 0; i != NumElts; ++i)
3388 Indices[i] = i;
3389
3390 // First extract half of each vector. This gives better codegen than
3391 // doing it in a single shuffle.
3392 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
3393 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
3394 // Concat the vectors.
3395 // NOTE: Operands have to be swapped to match intrinsic definition.
3396 Rep = Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
3397 Rep = Builder.CreateBitCast(Rep, CI->getType());
3398 } else if (Name == "avx512.kand.w") {
3399 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3400 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3401 Rep = Builder.CreateAnd(LHS, RHS);
3402 Rep = Builder.CreateBitCast(Rep, CI->getType());
3403 } else if (Name == "avx512.kandn.w") {
3404 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3405 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3406 LHS = Builder.CreateNot(LHS);
3407 Rep = Builder.CreateAnd(LHS, RHS);
3408 Rep = Builder.CreateBitCast(Rep, CI->getType());
3409 } else if (Name == "avx512.kor.w") {
3410 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3411 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3412 Rep = Builder.CreateOr(LHS, RHS);
3413 Rep = Builder.CreateBitCast(Rep, CI->getType());
3414 } else if (Name == "avx512.kxor.w") {
3415 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3416 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3417 Rep = Builder.CreateXor(LHS, RHS);
3418 Rep = Builder.CreateBitCast(Rep, CI->getType());
3419 } else if (Name == "avx512.kxnor.w") {
3420 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3421 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3422 LHS = Builder.CreateNot(LHS);
3423 Rep = Builder.CreateXor(LHS, RHS);
3424 Rep = Builder.CreateBitCast(Rep, CI->getType());
3425 } else if (Name == "avx512.knot.w") {
3426 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3427 Rep = Builder.CreateNot(Rep);
3428 Rep = Builder.CreateBitCast(Rep, CI->getType());
3429 } else if (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w") {
3430 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
3431 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
3432 Rep = Builder.CreateOr(LHS, RHS);
3433 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
3434 Value *C;
3435 if (Name[14] == 'c')
3436 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
3437 else
3438 C = ConstantInt::getNullValue(Builder.getInt16Ty());
3439 Rep = Builder.CreateICmpEQ(Rep, C);
3440 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
3441 } else if (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
3442 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
3443 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
3444 Name == "sse.div.ss" || Name == "sse2.div.sd") {
3445 Type *I32Ty = Type::getInt32Ty(C);
3446 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
3447 ConstantInt::get(I32Ty, 0));
3448 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
3449 ConstantInt::get(I32Ty, 0));
3450 Value *EltOp;
3451 if (Name.contains(".add."))
3452 EltOp = Builder.CreateFAdd(Elt0, Elt1);
3453 else if (Name.contains(".sub."))
3454 EltOp = Builder.CreateFSub(Elt0, Elt1);
3455 else if (Name.contains(".mul."))
3456 EltOp = Builder.CreateFMul(Elt0, Elt1);
3457 else
3458 EltOp = Builder.CreateFDiv(Elt0, Elt1);
3459 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
3460 ConstantInt::get(I32Ty, 0));
3461 } else if (Name.starts_with("avx512.mask.pcmp")) {
3462 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
3463 bool CmpEq = Name[16] == 'e';
3464 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
3465 } else if (Name.starts_with("avx512.mask.vpshufbitqmb.")) {
3466 Type *OpTy = CI->getArgOperand(0)->getType();
3467 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3468 Intrinsic::ID IID;
3469 switch (VecWidth) {
3470 default:
3471 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3472 break;
3473 case 128:
3474 IID = Intrinsic::x86_avx512_vpshufbitqmb_128;
3475 break;
3476 case 256:
3477 IID = Intrinsic::x86_avx512_vpshufbitqmb_256;
3478 break;
3479 case 512:
3480 IID = Intrinsic::x86_avx512_vpshufbitqmb_512;
3481 break;
3482 }
3483
3484 Rep =
3485 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3486 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3487 } else if (Name.starts_with("avx512.mask.fpclass.p")) {
3488 Type *OpTy = CI->getArgOperand(0)->getType();
3489 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3490 unsigned EltWidth = OpTy->getScalarSizeInBits();
3491 Intrinsic::ID IID;
3492 if (VecWidth == 128 && EltWidth == 32)
3493 IID = Intrinsic::x86_avx512_fpclass_ps_128;
3494 else if (VecWidth == 256 && EltWidth == 32)
3495 IID = Intrinsic::x86_avx512_fpclass_ps_256;
3496 else if (VecWidth == 512 && EltWidth == 32)
3497 IID = Intrinsic::x86_avx512_fpclass_ps_512;
3498 else if (VecWidth == 128 && EltWidth == 64)
3499 IID = Intrinsic::x86_avx512_fpclass_pd_128;
3500 else if (VecWidth == 256 && EltWidth == 64)
3501 IID = Intrinsic::x86_avx512_fpclass_pd_256;
3502 else if (VecWidth == 512 && EltWidth == 64)
3503 IID = Intrinsic::x86_avx512_fpclass_pd_512;
3504 else
3505 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3506
3507 Rep =
3508 Builder.CreateIntrinsic(IID, {CI->getOperand(0), CI->getArgOperand(1)});
3509 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
3510 } else if (Name.starts_with("avx512.cmp.p")) {
3511 SmallVector<Value *, 4> Args(CI->args());
3512 Type *OpTy = Args[0]->getType();
3513 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
3514 unsigned EltWidth = OpTy->getScalarSizeInBits();
3515 Intrinsic::ID IID;
3516 if (VecWidth == 128 && EltWidth == 32)
3517 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
3518 else if (VecWidth == 256 && EltWidth == 32)
3519 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
3520 else if (VecWidth == 512 && EltWidth == 32)
3521 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
3522 else if (VecWidth == 128 && EltWidth == 64)
3523 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
3524 else if (VecWidth == 256 && EltWidth == 64)
3525 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
3526 else if (VecWidth == 512 && EltWidth == 64)
3527 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
3528 else
3529 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
3530
3531 Value *Mask = Constant::getAllOnesValue(CI->getType());
3532 if (VecWidth == 512)
3533 std::swap(Mask, Args.back());
3534 Args.push_back(Mask);
3535
3536 Rep = Builder.CreateIntrinsic(IID, Args);
3537 } else if (Name.starts_with("avx512.mask.cmp.")) {
3538 // Integer compare intrinsics.
3539 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3540 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
3541 } else if (Name.starts_with("avx512.mask.ucmp.")) {
3542 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3543 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
3544 } else if (Name.starts_with("avx512.cvtb2mask.") ||
3545 Name.starts_with("avx512.cvtw2mask.") ||
3546 Name.starts_with("avx512.cvtd2mask.") ||
3547 Name.starts_with("avx512.cvtq2mask.")) {
3548 Value *Op = CI->getArgOperand(0);
3549 Value *Zero = llvm::Constant::getNullValue(Op->getType());
3550 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
3551 Rep = applyX86MaskOn1BitsVec(Builder, Rep, nullptr);
3552 } else if (Name == "ssse3.pabs.b.128" || Name == "ssse3.pabs.w.128" ||
3553 Name == "ssse3.pabs.d.128" || Name.starts_with("avx2.pabs") ||
3554 Name.starts_with("avx512.mask.pabs")) {
3555 Rep = upgradeAbs(Builder, *CI);
3556 } else if (Name == "sse41.pmaxsb" || Name == "sse2.pmaxs.w" ||
3557 Name == "sse41.pmaxsd" || Name.starts_with("avx2.pmaxs") ||
3558 Name.starts_with("avx512.mask.pmaxs")) {
3559 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
3560 } else if (Name == "sse2.pmaxu.b" || Name == "sse41.pmaxuw" ||
3561 Name == "sse41.pmaxud" || Name.starts_with("avx2.pmaxu") ||
3562 Name.starts_with("avx512.mask.pmaxu")) {
3563 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
3564 } else if (Name == "sse41.pminsb" || Name == "sse2.pmins.w" ||
3565 Name == "sse41.pminsd" || Name.starts_with("avx2.pmins") ||
3566 Name.starts_with("avx512.mask.pmins")) {
3567 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
3568 } else if (Name == "sse2.pminu.b" || Name == "sse41.pminuw" ||
3569 Name == "sse41.pminud" || Name.starts_with("avx2.pminu") ||
3570 Name.starts_with("avx512.mask.pminu")) {
3571 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
3572 } else if (Name == "sse2.pmulu.dq" || Name == "avx2.pmulu.dq" ||
3573 Name == "avx512.pmulu.dq.512" ||
3574 Name.starts_with("avx512.mask.pmulu.dq.")) {
3575 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
3576 } else if (Name == "sse41.pmuldq" || Name == "avx2.pmul.dq" ||
3577 Name == "avx512.pmul.dq.512" ||
3578 Name.starts_with("avx512.mask.pmul.dq.")) {
3579 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
3580 } else if (Name == "sse.cvtsi2ss" || Name == "sse2.cvtsi2sd" ||
3581 Name == "sse.cvtsi642ss" || Name == "sse2.cvtsi642sd") {
3582 Rep =
3583 Builder.CreateSIToFP(CI->getArgOperand(1),
3584 cast<VectorType>(CI->getType())->getElementType());
3585 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3586 } else if (Name == "avx512.cvtusi2sd") {
3587 Rep =
3588 Builder.CreateUIToFP(CI->getArgOperand(1),
3589 cast<VectorType>(CI->getType())->getElementType());
3590 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3591 } else if (Name == "sse2.cvtss2sd") {
3592 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
3593 Rep = Builder.CreateFPExt(
3594 Rep, cast<VectorType>(CI->getType())->getElementType());
3595 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3596 } else if (Name == "sse2.cvtdq2pd" || Name == "sse2.cvtdq2ps" ||
3597 Name == "avx.cvtdq2.pd.256" || Name == "avx.cvtdq2.ps.256" ||
3598 Name.starts_with("avx512.mask.cvtdq2pd.") ||
3599 Name.starts_with("avx512.mask.cvtudq2pd.") ||
3600 Name.starts_with("avx512.mask.cvtdq2ps.") ||
3601 Name.starts_with("avx512.mask.cvtudq2ps.") ||
3602 Name.starts_with("avx512.mask.cvtqq2pd.") ||
3603 Name.starts_with("avx512.mask.cvtuqq2pd.") ||
3604 Name == "avx512.mask.cvtqq2ps.256" ||
3605 Name == "avx512.mask.cvtqq2ps.512" ||
3606 Name == "avx512.mask.cvtuqq2ps.256" ||
3607 Name == "avx512.mask.cvtuqq2ps.512" || Name == "sse2.cvtps2pd" ||
3608 Name == "avx.cvt.ps2.pd.256" ||
3609 Name == "avx512.mask.cvtps2pd.128" ||
3610 Name == "avx512.mask.cvtps2pd.256") {
3611 auto *DstTy = cast<FixedVectorType>(CI->getType());
3612 Rep = CI->getArgOperand(0);
3613 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3614
3615 unsigned NumDstElts = DstTy->getNumElements();
3616 if (NumDstElts < SrcTy->getNumElements()) {
3617 assert(NumDstElts == 2 && "Unexpected vector size");
3618 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
3619 }
3620
3621 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
3622 bool IsUnsigned = Name.contains("cvtu");
3623 if (IsPS2PD)
3624 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
3625 else if (CI->arg_size() == 4 &&
3626 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
3627 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
3628 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
3629 : Intrinsic::x86_avx512_sitofp_round;
3630 Rep = Builder.CreateIntrinsic(IID, {DstTy, SrcTy},
3631 {Rep, CI->getArgOperand(3)});
3632 } else {
3633 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
3634 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
3635 }
3636
3637 if (CI->arg_size() >= 3)
3638 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3639 CI->getArgOperand(1));
3640 } else if (Name.starts_with("avx512.mask.vcvtph2ps.") ||
3641 Name.starts_with("vcvtph2ps.")) {
3642 auto *DstTy = cast<FixedVectorType>(CI->getType());
3643 Rep = CI->getArgOperand(0);
3644 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
3645 unsigned NumDstElts = DstTy->getNumElements();
3646 if (NumDstElts != SrcTy->getNumElements()) {
3647 assert(NumDstElts == 4 && "Unexpected vector size");
3648 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
3649 }
3650 Rep = Builder.CreateBitCast(
3651 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
3652 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
3653 if (CI->arg_size() >= 3)
3654 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3655 CI->getArgOperand(1));
3656 } else if (Name.starts_with("avx512.mask.load")) {
3657 // "avx512.mask.loadu." or "avx512.mask.load."
3658 bool Aligned = Name[16] != 'u'; // "avx512.mask.loadu".
3659 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3660 CI->getArgOperand(2), Aligned);
3661 } else if (Name.starts_with("avx512.mask.expand.load.")) {
3662 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3663 auto *PtrTy = CI->getOperand(0)->getType();
3664 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3665 ResultTy->getNumElements());
3666 Rep = Builder.CreateIntrinsic(
3667 Intrinsic::masked_expandload, {ResultTy, PtrTy},
3668 {CI->getOperand(0), MaskVec, CI->getOperand(1)});
3669 } else if (Name.starts_with("avx512.mask.compress.store.")) {
3670 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
3671 auto *PtrTy = CI->getArgOperand(0)->getType();
3672 Value *MaskVec =
3673 getX86MaskVec(Builder, CI->getArgOperand(2),
3674 cast<FixedVectorType>(ResultTy)->getNumElements());
3675 Rep = Builder.CreateIntrinsic(
3676 Intrinsic::masked_compressstore, {ResultTy, PtrTy},
3677 {CI->getArgOperand(1), CI->getArgOperand(0), MaskVec});
3678 } else if (Name.starts_with("avx512.mask.compress.") ||
3679 Name.starts_with("avx512.mask.expand.")) {
3680 auto *ResultTy = cast<FixedVectorType>(CI->getType());
3681
3682 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
3683 ResultTy->getNumElements());
3684
3685 bool IsCompress = Name[12] == 'c';
3686 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
3687 : Intrinsic::x86_avx512_mask_expand;
3688 Rep = Builder.CreateIntrinsic(
3689 IID, ResultTy, {CI->getOperand(0), CI->getOperand(1), MaskVec});
3690 } else if (Name.starts_with("xop.vpcom")) {
3691 bool IsSigned;
3692 if (Name.ends_with("ub") || Name.ends_with("uw") || Name.ends_with("ud") ||
3693 Name.ends_with("uq"))
3694 IsSigned = false;
3695 else if (Name.ends_with("b") || Name.ends_with("w") ||
3696 Name.ends_with("d") || Name.ends_with("q"))
3697 IsSigned = true;
3698 else
3699 reportFatalUsageErrorWithCI("Intrinsic has unknown suffix", CI);
3700
3701 unsigned Imm;
3702 if (CI->arg_size() == 3) {
3703 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3704 } else {
3705 Name = Name.substr(9); // strip off "xop.vpcom"
3706 if (Name.starts_with("lt"))
3707 Imm = 0;
3708 else if (Name.starts_with("le"))
3709 Imm = 1;
3710 else if (Name.starts_with("gt"))
3711 Imm = 2;
3712 else if (Name.starts_with("ge"))
3713 Imm = 3;
3714 else if (Name.starts_with("eq"))
3715 Imm = 4;
3716 else if (Name.starts_with("ne"))
3717 Imm = 5;
3718 else if (Name.starts_with("false"))
3719 Imm = 6;
3720 else if (Name.starts_with("true"))
3721 Imm = 7;
3722 else
3723 llvm_unreachable("Unknown condition");
3724 }
3725
3726 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
3727 } else if (Name.starts_with("xop.vpcmov")) {
3728 Value *Sel = CI->getArgOperand(2);
3729 Value *NotSel = Builder.CreateNot(Sel);
3730 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
3731 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
3732 Rep = Builder.CreateOr(Sel0, Sel1);
3733 } else if (Name.starts_with("xop.vprot") || Name.starts_with("avx512.prol") ||
3734 Name.starts_with("avx512.mask.prol")) {
3735 Rep = upgradeX86Rotate(Builder, *CI, false);
3736 } else if (Name.starts_with("avx512.pror") ||
3737 Name.starts_with("avx512.mask.pror")) {
3738 Rep = upgradeX86Rotate(Builder, *CI, true);
3739 } else if (Name.starts_with("avx512.vpshld.") ||
3740 Name.starts_with("avx512.mask.vpshld") ||
3741 Name.starts_with("avx512.maskz.vpshld")) {
3742 bool ZeroMask = Name[11] == 'z';
3743 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
3744 } else if (Name.starts_with("avx512.vpshrd.") ||
3745 Name.starts_with("avx512.mask.vpshrd") ||
3746 Name.starts_with("avx512.maskz.vpshrd")) {
3747 bool ZeroMask = Name[11] == 'z';
3748 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
3749 } else if (Name == "sse42.crc32.64.8") {
3750 Value *Trunc0 =
3751 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
3752 Rep = Builder.CreateIntrinsic(Intrinsic::x86_sse42_crc32_32_8,
3753 {Trunc0, CI->getArgOperand(1)});
3754 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
3755 } else if (Name.starts_with("avx.vbroadcast.s") ||
3756 Name.starts_with("avx512.vbroadcast.s")) {
3757 // Replace broadcasts with a series of insertelements.
3758 auto *VecTy = cast<FixedVectorType>(CI->getType());
3759 Type *EltTy = VecTy->getElementType();
3760 unsigned EltNum = VecTy->getNumElements();
3761 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
3762 Type *I32Ty = Type::getInt32Ty(C);
3763 Rep = PoisonValue::get(VecTy);
3764 for (unsigned I = 0; I < EltNum; ++I)
3765 Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I));
3766 } else if (Name.starts_with("sse41.pmovsx") ||
3767 Name.starts_with("sse41.pmovzx") ||
3768 Name.starts_with("avx2.pmovsx") ||
3769 Name.starts_with("avx2.pmovzx") ||
3770 Name.starts_with("avx512.mask.pmovsx") ||
3771 Name.starts_with("avx512.mask.pmovzx")) {
3772 auto *DstTy = cast<FixedVectorType>(CI->getType());
3773 unsigned NumDstElts = DstTy->getNumElements();
3774
3775 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
3776 SmallVector<int, 8> ShuffleMask(NumDstElts);
3777 for (unsigned i = 0; i != NumDstElts; ++i)
3778 ShuffleMask[i] = i;
3779
3780 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
3781
3782 bool DoSext = Name.contains("pmovsx");
3783 Rep =
3784 DoSext ? Builder.CreateSExt(SV, DstTy) : Builder.CreateZExt(SV, DstTy);
3785 // If there are 3 arguments, it's a masked intrinsic so we need a select.
3786 if (CI->arg_size() == 3)
3787 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3788 CI->getArgOperand(1));
3789 } else if (Name == "avx512.mask.pmov.qd.256" ||
3790 Name == "avx512.mask.pmov.qd.512" ||
3791 Name == "avx512.mask.pmov.wb.256" ||
3792 Name == "avx512.mask.pmov.wb.512") {
3793 Type *Ty = CI->getArgOperand(1)->getType();
3794 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
3795 Rep =
3796 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3797 } else if (Name.starts_with("avx.vbroadcastf128") ||
3798 Name == "avx2.vbroadcasti128") {
3799 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
3800 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
3801 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
3802 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
3803 Value *Load = Builder.CreateAlignedLoad(VT, CI->getArgOperand(0), Align(1));
3804 if (NumSrcElts == 2)
3805 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
3806 else
3807 Rep = Builder.CreateShuffleVector(Load,
3808 ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
3809 } else if (Name.starts_with("avx512.mask.shuf.i") ||
3810 Name.starts_with("avx512.mask.shuf.f")) {
3811 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3812 Type *VT = CI->getType();
3813 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
3814 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
3815 unsigned ControlBitsMask = NumLanes - 1;
3816 unsigned NumControlBits = NumLanes / 2;
3817 SmallVector<int, 8> ShuffleMask(0);
3818
3819 for (unsigned l = 0; l != NumLanes; ++l) {
3820 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
3821 // We actually need the other source.
3822 if (l >= NumLanes / 2)
3823 LaneMask += NumLanes;
3824 for (unsigned i = 0; i != NumElementsInLane; ++i)
3825 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
3826 }
3827 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3828 CI->getArgOperand(1), ShuffleMask);
3829 Rep =
3830 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3831 } else if (Name.starts_with("avx512.mask.broadcastf") ||
3832 Name.starts_with("avx512.mask.broadcasti")) {
3833 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
3834 ->getNumElements();
3835 unsigned NumDstElts =
3836 cast<FixedVectorType>(CI->getType())->getNumElements();
3837
3838 SmallVector<int, 8> ShuffleMask(NumDstElts);
3839 for (unsigned i = 0; i != NumDstElts; ++i)
3840 ShuffleMask[i] = i % NumSrcElts;
3841
3842 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
3843 CI->getArgOperand(0), ShuffleMask);
3844 Rep =
3845 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3846 } else if (Name.starts_with("avx2.pbroadcast") ||
3847 Name.starts_with("avx2.vbroadcast") ||
3848 Name.starts_with("avx512.pbroadcast") ||
3849 Name.starts_with("avx512.mask.broadcast.s")) {
3850 // Replace vp?broadcasts with a vector shuffle.
3851 Value *Op = CI->getArgOperand(0);
3852 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
3853 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
3854 SmallVector<int, 8> M;
3855 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
3856 Rep = Builder.CreateShuffleVector(Op, M);
3857
3858 if (CI->arg_size() == 3)
3859 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
3860 CI->getArgOperand(1));
3861 } else if (Name.starts_with("sse2.padds.") ||
3862 Name.starts_with("avx2.padds.") ||
3863 Name.starts_with("avx512.padds.") ||
3864 Name.starts_with("avx512.mask.padds.")) {
3865 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
3866 } else if (Name.starts_with("sse2.psubs.") ||
3867 Name.starts_with("avx2.psubs.") ||
3868 Name.starts_with("avx512.psubs.") ||
3869 Name.starts_with("avx512.mask.psubs.")) {
3870 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
3871 } else if (Name.starts_with("sse2.paddus.") ||
3872 Name.starts_with("avx2.paddus.") ||
3873 Name.starts_with("avx512.mask.paddus.")) {
3874 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
3875 } else if (Name.starts_with("sse2.psubus.") ||
3876 Name.starts_with("avx2.psubus.") ||
3877 Name.starts_with("avx512.mask.psubus.")) {
3878 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
3879 } else if (Name.starts_with("avx512.mask.palignr.")) {
3880 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
3881 CI->getArgOperand(1), CI->getArgOperand(2),
3882 CI->getArgOperand(3), CI->getArgOperand(4),
3883 false);
3884 } else if (Name.starts_with("avx512.mask.valign.")) {
3885 Rep = upgradeX86ALIGNIntrinsics(
3886 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
3887 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
3888 } else if (Name == "sse2.psll.dq" || Name == "avx2.psll.dq") {
3889 // 128/256-bit shift left specified in bits.
3890 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3891 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
3892 Shift / 8); // Shift is in bits.
3893 } else if (Name == "sse2.psrl.dq" || Name == "avx2.psrl.dq") {
3894 // 128/256-bit shift right specified in bits.
3895 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3896 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
3897 Shift / 8); // Shift is in bits.
3898 } else if (Name == "sse2.psll.dq.bs" || Name == "avx2.psll.dq.bs" ||
3899 Name == "avx512.psll.dq.512") {
3900 // 128/256/512-bit shift left specified in bytes.
3901 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3902 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3903 } else if (Name == "sse2.psrl.dq.bs" || Name == "avx2.psrl.dq.bs" ||
3904 Name == "avx512.psrl.dq.512") {
3905 // 128/256/512-bit shift right specified in bytes.
3906 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3907 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
3908 } else if (Name == "sse41.pblendw" || Name.starts_with("sse41.blendp") ||
3909 Name.starts_with("avx.blend.p") || Name == "avx2.pblendw" ||
3910 Name.starts_with("avx2.pblendd.")) {
3911 Value *Op0 = CI->getArgOperand(0);
3912 Value *Op1 = CI->getArgOperand(1);
3913 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3914 auto *VecTy = cast<FixedVectorType>(CI->getType());
3915 unsigned NumElts = VecTy->getNumElements();
3916
3917 SmallVector<int, 16> Idxs(NumElts);
3918 for (unsigned i = 0; i != NumElts; ++i)
3919 Idxs[i] = ((Imm >> (i % 8)) & 1) ? i + NumElts : i;
3920
3921 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
3922 } else if (Name.starts_with("avx.vinsertf128.") ||
3923 Name == "avx2.vinserti128" ||
3924 Name.starts_with("avx512.mask.insert")) {
3925 Value *Op0 = CI->getArgOperand(0);
3926 Value *Op1 = CI->getArgOperand(1);
3927 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3928 unsigned DstNumElts =
3929 cast<FixedVectorType>(CI->getType())->getNumElements();
3930 unsigned SrcNumElts =
3931 cast<FixedVectorType>(Op1->getType())->getNumElements();
3932 unsigned Scale = DstNumElts / SrcNumElts;
3933
3934 // Mask off the high bits of the immediate value; hardware ignores those.
3935 Imm = Imm % Scale;
3936
3937 // Extend the second operand into a vector the size of the destination.
3938 SmallVector<int, 8> Idxs(DstNumElts);
3939 for (unsigned i = 0; i != SrcNumElts; ++i)
3940 Idxs[i] = i;
3941 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
3942 Idxs[i] = SrcNumElts;
3943 Rep = Builder.CreateShuffleVector(Op1, Idxs);
3944
3945 // Insert the second operand into the first operand.
3946
3947 // Note that there is no guarantee that instruction lowering will actually
3948 // produce a vinsertf128 instruction for the created shuffles. In
3949 // particular, the 0 immediate case involves no lane changes, so it can
3950 // be handled as a blend.
3951
3952 // Example of shuffle mask for 32-bit elements:
3953 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
3954 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
3955
3956 // First fill with identify mask.
3957 for (unsigned i = 0; i != DstNumElts; ++i)
3958 Idxs[i] = i;
3959 // Then replace the elements where we need to insert.
3960 for (unsigned i = 0; i != SrcNumElts; ++i)
3961 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
3962 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
3963
3964 // If the intrinsic has a mask operand, handle that.
3965 if (CI->arg_size() == 5)
3966 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
3967 CI->getArgOperand(3));
3968 } else if (Name.starts_with("avx.vextractf128.") ||
3969 Name == "avx2.vextracti128" ||
3970 Name.starts_with("avx512.mask.vextract")) {
3971 Value *Op0 = CI->getArgOperand(0);
3972 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3973 unsigned DstNumElts =
3974 cast<FixedVectorType>(CI->getType())->getNumElements();
3975 unsigned SrcNumElts =
3976 cast<FixedVectorType>(Op0->getType())->getNumElements();
3977 unsigned Scale = SrcNumElts / DstNumElts;
3978
3979 // Mask off the high bits of the immediate value; hardware ignores those.
3980 Imm = Imm % Scale;
3981
3982 // Get indexes for the subvector of the input vector.
3983 SmallVector<int, 8> Idxs(DstNumElts);
3984 for (unsigned i = 0; i != DstNumElts; ++i) {
3985 Idxs[i] = i + (Imm * DstNumElts);
3986 }
3987 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
3988
3989 // If the intrinsic has a mask operand, handle that.
3990 if (CI->arg_size() == 4)
3991 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3992 CI->getArgOperand(2));
3993 } else if (Name.starts_with("avx512.mask.perm.df.") ||
3994 Name.starts_with("avx512.mask.perm.di.")) {
3995 Value *Op0 = CI->getArgOperand(0);
3996 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3997 auto *VecTy = cast<FixedVectorType>(CI->getType());
3998 unsigned NumElts = VecTy->getNumElements();
3999
4000 SmallVector<int, 8> Idxs(NumElts);
4001 for (unsigned i = 0; i != NumElts; ++i)
4002 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
4003
4004 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4005
4006 if (CI->arg_size() == 4)
4007 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4008 CI->getArgOperand(2));
4009 } else if (Name.starts_with("avx.vperm2f128.") || Name == "avx2.vperm2i128") {
4010 // The immediate permute control byte looks like this:
4011 // [1:0] - select 128 bits from sources for low half of destination
4012 // [2] - ignore
4013 // [3] - zero low half of destination
4014 // [5:4] - select 128 bits from sources for high half of destination
4015 // [6] - ignore
4016 // [7] - zero high half of destination
4017
4018 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4019
4020 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4021 unsigned HalfSize = NumElts / 2;
4022 SmallVector<int, 8> ShuffleMask(NumElts);
4023
4024 // Determine which operand(s) are actually in use for this instruction.
4025 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4026 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
4027
4028 // If needed, replace operands based on zero mask.
4029 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
4030 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
4031
4032 // Permute low half of result.
4033 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
4034 for (unsigned i = 0; i < HalfSize; ++i)
4035 ShuffleMask[i] = StartIndex + i;
4036
4037 // Permute high half of result.
4038 StartIndex = (Imm & 0x10) ? HalfSize : 0;
4039 for (unsigned i = 0; i < HalfSize; ++i)
4040 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
4041
4042 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4043
4044 } else if (Name.starts_with("avx.vpermil.") || Name == "sse2.pshuf.d" ||
4045 Name.starts_with("avx512.mask.vpermil.p") ||
4046 Name.starts_with("avx512.mask.pshuf.d.")) {
4047 Value *Op0 = CI->getArgOperand(0);
4048 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4049 auto *VecTy = cast<FixedVectorType>(CI->getType());
4050 unsigned NumElts = VecTy->getNumElements();
4051 // Calculate the size of each index in the immediate.
4052 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
4053 unsigned IdxMask = ((1 << IdxSize) - 1);
4054
4055 SmallVector<int, 8> Idxs(NumElts);
4056 // Lookup the bits for this element, wrapping around the immediate every
4057 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
4058 // to offset by the first index of each group.
4059 for (unsigned i = 0; i != NumElts; ++i)
4060 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
4061
4062 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4063
4064 if (CI->arg_size() == 4)
4065 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4066 CI->getArgOperand(2));
4067 } else if (Name == "sse2.pshufl.w" ||
4068 Name.starts_with("avx512.mask.pshufl.w.")) {
4069 Value *Op0 = CI->getArgOperand(0);
4070 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4071 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4072
4073 if (Name == "sse2.pshufl.w" && NumElts % 8 != 0)
4074 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4075
4076 SmallVector<int, 16> Idxs(NumElts);
4077 for (unsigned l = 0; l != NumElts; l += 8) {
4078 for (unsigned i = 0; i != 4; ++i)
4079 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
4080 for (unsigned i = 4; i != 8; ++i)
4081 Idxs[i + l] = i + l;
4082 }
4083
4084 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4085
4086 if (CI->arg_size() == 4)
4087 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4088 CI->getArgOperand(2));
4089 } else if (Name == "sse2.pshufh.w" ||
4090 Name.starts_with("avx512.mask.pshufh.w.")) {
4091 Value *Op0 = CI->getArgOperand(0);
4092 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4093 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4094
4095 if (Name == "sse2.pshufh.w" && NumElts % 8 != 0)
4096 reportFatalUsageErrorWithCI("Intrinsic has invalid signature", CI);
4097
4098 SmallVector<int, 16> Idxs(NumElts);
4099 for (unsigned l = 0; l != NumElts; l += 8) {
4100 for (unsigned i = 0; i != 4; ++i)
4101 Idxs[i + l] = i + l;
4102 for (unsigned i = 0; i != 4; ++i)
4103 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
4104 }
4105
4106 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4107
4108 if (CI->arg_size() == 4)
4109 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
4110 CI->getArgOperand(2));
4111 } else if (Name.starts_with("avx512.mask.shuf.p")) {
4112 Value *Op0 = CI->getArgOperand(0);
4113 Value *Op1 = CI->getArgOperand(1);
4114 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
4115 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4116
4117 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4118 unsigned HalfLaneElts = NumLaneElts / 2;
4119
4120 SmallVector<int, 16> Idxs(NumElts);
4121 for (unsigned i = 0; i != NumElts; ++i) {
4122 // Base index is the starting element of the lane.
4123 Idxs[i] = i - (i % NumLaneElts);
4124 // If we are half way through the lane switch to the other source.
4125 if ((i % NumLaneElts) >= HalfLaneElts)
4126 Idxs[i] += NumElts;
4127 // Now select the specific element. By adding HalfLaneElts bits from
4128 // the immediate. Wrapping around the immediate every 8-bits.
4129 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
4130 }
4131
4132 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4133
4134 Rep =
4135 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
4136 } else if (Name.starts_with("avx512.mask.movddup") ||
4137 Name.starts_with("avx512.mask.movshdup") ||
4138 Name.starts_with("avx512.mask.movsldup")) {
4139 Value *Op0 = CI->getArgOperand(0);
4140 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4141 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4142
4143 unsigned Offset = 0;
4144 if (Name.starts_with("avx512.mask.movshdup."))
4145 Offset = 1;
4146
4147 SmallVector<int, 16> Idxs(NumElts);
4148 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
4149 for (unsigned i = 0; i != NumLaneElts; i += 2) {
4150 Idxs[i + l + 0] = i + l + Offset;
4151 Idxs[i + l + 1] = i + l + Offset;
4152 }
4153
4154 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
4155
4156 Rep =
4157 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4158 } else if (Name.starts_with("avx512.mask.punpckl") ||
4159 Name.starts_with("avx512.mask.unpckl.")) {
4160 Value *Op0 = CI->getArgOperand(0);
4161 Value *Op1 = CI->getArgOperand(1);
4162 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4163 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4164
4165 SmallVector<int, 64> Idxs(NumElts);
4166 for (int l = 0; l != NumElts; l += NumLaneElts)
4167 for (int i = 0; i != NumLaneElts; ++i)
4168 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
4169
4170 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4171
4172 Rep =
4173 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4174 } else if (Name.starts_with("avx512.mask.punpckh") ||
4175 Name.starts_with("avx512.mask.unpckh.")) {
4176 Value *Op0 = CI->getArgOperand(0);
4177 Value *Op1 = CI->getArgOperand(1);
4178 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4179 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
4180
4181 SmallVector<int, 64> Idxs(NumElts);
4182 for (int l = 0; l != NumElts; l += NumLaneElts)
4183 for (int i = 0; i != NumLaneElts; ++i)
4184 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
4185
4186 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
4187
4188 Rep =
4189 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4190 } else if (Name.starts_with("avx512.mask.and.") ||
4191 Name.starts_with("avx512.mask.pand.")) {
4192 VectorType *FTy = cast<VectorType>(CI->getType());
4193 VectorType *ITy = VectorType::getInteger(FTy);
4194 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4195 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4196 Rep = Builder.CreateBitCast(Rep, FTy);
4197 Rep =
4198 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4199 } else if (Name.starts_with("avx512.mask.andn.") ||
4200 Name.starts_with("avx512.mask.pandn.")) {
4201 VectorType *FTy = cast<VectorType>(CI->getType());
4202 VectorType *ITy = VectorType::getInteger(FTy);
4203 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
4204 Rep = Builder.CreateAnd(Rep,
4205 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4206 Rep = Builder.CreateBitCast(Rep, FTy);
4207 Rep =
4208 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4209 } else if (Name.starts_with("avx512.mask.or.") ||
4210 Name.starts_with("avx512.mask.por.")) {
4211 VectorType *FTy = cast<VectorType>(CI->getType());
4212 VectorType *ITy = VectorType::getInteger(FTy);
4213 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4214 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4215 Rep = Builder.CreateBitCast(Rep, FTy);
4216 Rep =
4217 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4218 } else if (Name.starts_with("avx512.mask.xor.") ||
4219 Name.starts_with("avx512.mask.pxor.")) {
4220 VectorType *FTy = cast<VectorType>(CI->getType());
4221 VectorType *ITy = VectorType::getInteger(FTy);
4222 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
4223 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
4224 Rep = Builder.CreateBitCast(Rep, FTy);
4225 Rep =
4226 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4227 } else if (Name.starts_with("avx512.mask.padd.")) {
4228 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4229 Rep =
4230 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4231 } else if (Name.starts_with("avx512.mask.psub.")) {
4232 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
4233 Rep =
4234 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4235 } else if (Name.starts_with("avx512.mask.pmull.")) {
4236 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
4237 Rep =
4238 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4239 } else if (Name.starts_with("avx512.mask.add.p")) {
4240 if (Name.ends_with(".512")) {
4241 Intrinsic::ID IID;
4242 if (Name[17] == 's')
4243 IID = Intrinsic::x86_avx512_add_ps_512;
4244 else
4245 IID = Intrinsic::x86_avx512_add_pd_512;
4246
4247 Rep = Builder.CreateIntrinsic(
4248 IID,
4249 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4250 } else {
4251 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
4252 }
4253 Rep =
4254 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4255 } else if (Name.starts_with("avx512.mask.div.p")) {
4256 if (Name.ends_with(".512")) {
4257 Intrinsic::ID IID;
4258 if (Name[17] == 's')
4259 IID = Intrinsic::x86_avx512_div_ps_512;
4260 else
4261 IID = Intrinsic::x86_avx512_div_pd_512;
4262
4263 Rep = Builder.CreateIntrinsic(
4264 IID,
4265 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4266 } else {
4267 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
4268 }
4269 Rep =
4270 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4271 } else if (Name.starts_with("avx512.mask.mul.p")) {
4272 if (Name.ends_with(".512")) {
4273 Intrinsic::ID IID;
4274 if (Name[17] == 's')
4275 IID = Intrinsic::x86_avx512_mul_ps_512;
4276 else
4277 IID = Intrinsic::x86_avx512_mul_pd_512;
4278
4279 Rep = Builder.CreateIntrinsic(
4280 IID,
4281 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4282 } else {
4283 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
4284 }
4285 Rep =
4286 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4287 } else if (Name.starts_with("avx512.mask.sub.p")) {
4288 if (Name.ends_with(".512")) {
4289 Intrinsic::ID IID;
4290 if (Name[17] == 's')
4291 IID = Intrinsic::x86_avx512_sub_ps_512;
4292 else
4293 IID = Intrinsic::x86_avx512_sub_pd_512;
4294
4295 Rep = Builder.CreateIntrinsic(
4296 IID,
4297 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4298 } else {
4299 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
4300 }
4301 Rep =
4302 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4303 } else if ((Name.starts_with("avx512.mask.max.p") ||
4304 Name.starts_with("avx512.mask.min.p")) &&
4305 Name.drop_front(18) == ".512") {
4306 bool IsDouble = Name[17] == 'd';
4307 bool IsMin = Name[13] == 'i';
4308 static const Intrinsic::ID MinMaxTbl[2][2] = {
4309 {Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512},
4310 {Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512}};
4311 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
4312
4313 Rep = Builder.CreateIntrinsic(
4314 IID,
4315 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
4316 Rep =
4317 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
4318 } else if (Name.starts_with("avx512.mask.lzcnt.")) {
4319 Rep =
4320 Builder.CreateIntrinsic(Intrinsic::ctlz, CI->getType(),
4321 {CI->getArgOperand(0), Builder.getInt1(false)});
4322 Rep =
4323 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
4324 } else if (Name.starts_with("avx512.mask.psll")) {
4325 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4326 bool IsVariable = Name[16] == 'v';
4327 char Size = Name[16] == '.' ? Name[17]
4328 : Name[17] == '.' ? Name[18]
4329 : Name[18] == '.' ? Name[19]
4330 : Name[20];
4331
4332 Intrinsic::ID IID;
4333 if (IsVariable && Name[17] != '.') {
4334 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
4335 IID = Intrinsic::x86_avx2_psllv_q;
4336 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
4337 IID = Intrinsic::x86_avx2_psllv_q_256;
4338 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
4339 IID = Intrinsic::x86_avx2_psllv_d;
4340 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
4341 IID = Intrinsic::x86_avx2_psllv_d_256;
4342 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
4343 IID = Intrinsic::x86_avx512_psllv_w_128;
4344 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
4345 IID = Intrinsic::x86_avx512_psllv_w_256;
4346 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
4347 IID = Intrinsic::x86_avx512_psllv_w_512;
4348 else
4349 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4350 } else if (Name.ends_with(".128")) {
4351 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
4352 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
4353 : Intrinsic::x86_sse2_psll_d;
4354 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
4355 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
4356 : Intrinsic::x86_sse2_psll_q;
4357 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
4358 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
4359 : Intrinsic::x86_sse2_psll_w;
4360 else
4361 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4362 } else if (Name.ends_with(".256")) {
4363 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
4364 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
4365 : Intrinsic::x86_avx2_psll_d;
4366 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
4367 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
4368 : Intrinsic::x86_avx2_psll_q;
4369 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
4370 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
4371 : Intrinsic::x86_avx2_psll_w;
4372 else
4373 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4374 } else {
4375 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
4376 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512
4377 : IsVariable ? Intrinsic::x86_avx512_psllv_d_512
4378 : Intrinsic::x86_avx512_psll_d_512;
4379 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
4380 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512
4381 : IsVariable ? Intrinsic::x86_avx512_psllv_q_512
4382 : Intrinsic::x86_avx512_psll_q_512;
4383 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
4384 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
4385 : Intrinsic::x86_avx512_psll_w_512;
4386 else
4387 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4388 }
4389
4390 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4391 } else if (Name.starts_with("avx512.mask.psrl")) {
4392 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4393 bool IsVariable = Name[16] == 'v';
4394 char Size = Name[16] == '.' ? Name[17]
4395 : Name[17] == '.' ? Name[18]
4396 : Name[18] == '.' ? Name[19]
4397 : Name[20];
4398
4399 Intrinsic::ID IID;
4400 if (IsVariable && Name[17] != '.') {
4401 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
4402 IID = Intrinsic::x86_avx2_psrlv_q;
4403 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
4404 IID = Intrinsic::x86_avx2_psrlv_q_256;
4405 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
4406 IID = Intrinsic::x86_avx2_psrlv_d;
4407 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
4408 IID = Intrinsic::x86_avx2_psrlv_d_256;
4409 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
4410 IID = Intrinsic::x86_avx512_psrlv_w_128;
4411 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
4412 IID = Intrinsic::x86_avx512_psrlv_w_256;
4413 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
4414 IID = Intrinsic::x86_avx512_psrlv_w_512;
4415 else
4416 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4417 } else if (Name.ends_with(".128")) {
4418 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
4419 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
4420 : Intrinsic::x86_sse2_psrl_d;
4421 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
4422 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
4423 : Intrinsic::x86_sse2_psrl_q;
4424 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
4425 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
4426 : Intrinsic::x86_sse2_psrl_w;
4427 else
4428 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4429 } else if (Name.ends_with(".256")) {
4430 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
4431 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
4432 : Intrinsic::x86_avx2_psrl_d;
4433 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
4434 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
4435 : Intrinsic::x86_avx2_psrl_q;
4436 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
4437 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
4438 : Intrinsic::x86_avx2_psrl_w;
4439 else
4440 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4441 } else {
4442 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
4443 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512
4444 : IsVariable ? Intrinsic::x86_avx512_psrlv_d_512
4445 : Intrinsic::x86_avx512_psrl_d_512;
4446 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
4447 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512
4448 : IsVariable ? Intrinsic::x86_avx512_psrlv_q_512
4449 : Intrinsic::x86_avx512_psrl_q_512;
4450 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
4451 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
4452 : Intrinsic::x86_avx512_psrl_w_512;
4453 else
4454 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4455 }
4456
4457 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4458 } else if (Name.starts_with("avx512.mask.psra")) {
4459 bool IsImmediate = Name[16] == 'i' || (Name.size() > 18 && Name[18] == 'i');
4460 bool IsVariable = Name[16] == 'v';
4461 char Size = Name[16] == '.' ? Name[17]
4462 : Name[17] == '.' ? Name[18]
4463 : Name[18] == '.' ? Name[19]
4464 : Name[20];
4465
4466 Intrinsic::ID IID;
4467 if (IsVariable && Name[17] != '.') {
4468 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
4469 IID = Intrinsic::x86_avx2_psrav_d;
4470 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
4471 IID = Intrinsic::x86_avx2_psrav_d_256;
4472 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
4473 IID = Intrinsic::x86_avx512_psrav_w_128;
4474 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
4475 IID = Intrinsic::x86_avx512_psrav_w_256;
4476 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
4477 IID = Intrinsic::x86_avx512_psrav_w_512;
4478 else
4479 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4480 } else if (Name.ends_with(".128")) {
4481 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
4482 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
4483 : Intrinsic::x86_sse2_psra_d;
4484 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
4485 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128
4486 : IsVariable ? Intrinsic::x86_avx512_psrav_q_128
4487 : Intrinsic::x86_avx512_psra_q_128;
4488 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
4489 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
4490 : Intrinsic::x86_sse2_psra_w;
4491 else
4492 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4493 } else if (Name.ends_with(".256")) {
4494 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
4495 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
4496 : Intrinsic::x86_avx2_psra_d;
4497 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
4498 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256
4499 : IsVariable ? Intrinsic::x86_avx512_psrav_q_256
4500 : Intrinsic::x86_avx512_psra_q_256;
4501 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
4502 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
4503 : Intrinsic::x86_avx2_psra_w;
4504 else
4505 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4506 } else {
4507 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
4508 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512
4509 : IsVariable ? Intrinsic::x86_avx512_psrav_d_512
4510 : Intrinsic::x86_avx512_psra_d_512;
4511 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
4512 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512
4513 : IsVariable ? Intrinsic::x86_avx512_psrav_q_512
4514 : Intrinsic::x86_avx512_psra_q_512;
4515 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
4516 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
4517 : Intrinsic::x86_avx512_psra_w_512;
4518 else
4519 reportFatalUsageErrorWithCI("Intrinsic has unexpected size", CI);
4520 }
4521
4522 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
4523 } else if (Name.starts_with("avx512.mask.move.s")) {
4524 Rep = upgradeMaskedMove(Builder, *CI);
4525 } else if (Name.starts_with("avx512.cvtmask2")) {
4526 Rep = upgradeMaskToInt(Builder, *CI);
4527 } else if (Name.ends_with(".movntdqa")) {
4528 MDNode *Node = MDNode::get(
4529 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
4530
4531 LoadInst *LI = Builder.CreateAlignedLoad(
4532 CI->getType(), CI->getArgOperand(0),
4533 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
4534 LI->setMetadata(LLVMContext::MD_nontemporal, Node);
4535 Rep = LI;
4536 } else if (Name.starts_with("fma.vfmadd.") ||
4537 Name.starts_with("fma.vfmsub.") ||
4538 Name.starts_with("fma.vfnmadd.") ||
4539 Name.starts_with("fma.vfnmsub.")) {
4540 bool NegMul = Name[6] == 'n';
4541 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
4542 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
4543
4544 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4545 CI->getArgOperand(2)};
4546
4547 if (IsScalar) {
4548 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4549 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4550 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4551 }
4552
4553 if (NegMul && !IsScalar)
4554 Ops[0] = Builder.CreateFNeg(Ops[0]);
4555 if (NegMul && IsScalar)
4556 Ops[1] = Builder.CreateFNeg(Ops[1]);
4557 if (NegAcc)
4558 Ops[2] = Builder.CreateFNeg(Ops[2]);
4559
4560 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4561
4562 if (IsScalar)
4563 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
4564 } else if (Name.starts_with("fma4.vfmadd.s")) {
4565 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4566 CI->getArgOperand(2)};
4567
4568 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
4569 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
4570 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
4571
4572 Rep = Builder.CreateIntrinsic(Intrinsic::fma, Ops[0]->getType(), Ops);
4573
4574 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
4575 Rep, (uint64_t)0);
4576 } else if (Name.starts_with("avx512.mask.vfmadd.s") ||
4577 Name.starts_with("avx512.maskz.vfmadd.s") ||
4578 Name.starts_with("avx512.mask3.vfmadd.s") ||
4579 Name.starts_with("avx512.mask3.vfmsub.s") ||
4580 Name.starts_with("avx512.mask3.vfnmsub.s")) {
4581 bool IsMask3 = Name[11] == '3';
4582 bool IsMaskZ = Name[11] == 'z';
4583 // Drop the "avx512.mask." to make it easier.
4584 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4585 bool NegMul = Name[2] == 'n';
4586 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4587
4588 Value *A = CI->getArgOperand(0);
4589 Value *B = CI->getArgOperand(1);
4590 Value *C = CI->getArgOperand(2);
4591
4592 if (NegMul && (IsMask3 || IsMaskZ))
4593 A = Builder.CreateFNeg(A);
4594 if (NegMul && !(IsMask3 || IsMaskZ))
4595 B = Builder.CreateFNeg(B);
4596 if (NegAcc)
4597 C = Builder.CreateFNeg(C);
4598
4599 A = Builder.CreateExtractElement(A, (uint64_t)0);
4600 B = Builder.CreateExtractElement(B, (uint64_t)0);
4601 C = Builder.CreateExtractElement(C, (uint64_t)0);
4602
4603 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4604 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
4605 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
4606
4607 Intrinsic::ID IID;
4608 if (Name.back() == 'd')
4609 IID = Intrinsic::x86_avx512_vfmadd_f64;
4610 else
4611 IID = Intrinsic::x86_avx512_vfmadd_f32;
4612 Rep = Builder.CreateIntrinsic(IID, Ops);
4613 } else {
4614 Rep = Builder.CreateFMA(A, B, C);
4615 }
4616
4617 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType())
4618 : IsMask3 ? C
4619 : A;
4620
4621 // For Mask3 with NegAcc, we need to create a new extractelement that
4622 // avoids the negation above.
4623 if (NegAcc && IsMask3)
4624 PassThru =
4625 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
4626
4627 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
4628 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
4629 (uint64_t)0);
4630 } else if (Name.starts_with("avx512.mask.vfmadd.p") ||
4631 Name.starts_with("avx512.mask.vfnmadd.p") ||
4632 Name.starts_with("avx512.mask.vfnmsub.p") ||
4633 Name.starts_with("avx512.mask3.vfmadd.p") ||
4634 Name.starts_with("avx512.mask3.vfmsub.p") ||
4635 Name.starts_with("avx512.mask3.vfnmsub.p") ||
4636 Name.starts_with("avx512.maskz.vfmadd.p")) {
4637 bool IsMask3 = Name[11] == '3';
4638 bool IsMaskZ = Name[11] == 'z';
4639 // Drop the "avx512.mask." to make it easier.
4640 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4641 bool NegMul = Name[2] == 'n';
4642 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
4643
4644 Value *A = CI->getArgOperand(0);
4645 Value *B = CI->getArgOperand(1);
4646 Value *C = CI->getArgOperand(2);
4647
4648 if (NegMul && (IsMask3 || IsMaskZ))
4649 A = Builder.CreateFNeg(A);
4650 if (NegMul && !(IsMask3 || IsMaskZ))
4651 B = Builder.CreateFNeg(B);
4652 if (NegAcc)
4653 C = Builder.CreateFNeg(C);
4654
4655 if (CI->arg_size() == 5 &&
4656 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
4657 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
4658 Intrinsic::ID IID;
4659 // Check the character before ".512" in string.
4660 if (Name[Name.size() - 5] == 's')
4661 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
4662 else
4663 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
4664
4665 Rep = Builder.CreateIntrinsic(IID, {A, B, C, CI->getArgOperand(4)});
4666 } else {
4667 Rep = Builder.CreateFMA(A, B, C);
4668 }
4669
4670 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4671 : IsMask3 ? CI->getArgOperand(2)
4672 : CI->getArgOperand(0);
4673
4674 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4675 } else if (Name.starts_with("fma.vfmsubadd.p")) {
4676 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4677 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4678 Intrinsic::ID IID;
4679 if (VecWidth == 128 && EltWidth == 32)
4680 IID = Intrinsic::x86_fma_vfmaddsub_ps;
4681 else if (VecWidth == 256 && EltWidth == 32)
4682 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
4683 else if (VecWidth == 128 && EltWidth == 64)
4684 IID = Intrinsic::x86_fma_vfmaddsub_pd;
4685 else if (VecWidth == 256 && EltWidth == 64)
4686 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
4687 else
4688 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4689
4690 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4691 CI->getArgOperand(2)};
4692 Ops[2] = Builder.CreateFNeg(Ops[2]);
4693 Rep = Builder.CreateIntrinsic(IID, Ops);
4694 } else if (Name.starts_with("avx512.mask.vfmaddsub.p") ||
4695 Name.starts_with("avx512.mask3.vfmaddsub.p") ||
4696 Name.starts_with("avx512.maskz.vfmaddsub.p") ||
4697 Name.starts_with("avx512.mask3.vfmsubadd.p")) {
4698 bool IsMask3 = Name[11] == '3';
4699 bool IsMaskZ = Name[11] == 'z';
4700 // Drop the "avx512.mask." to make it easier.
4701 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
4702 bool IsSubAdd = Name[3] == 's';
4703 if (CI->arg_size() == 5) {
4704 Intrinsic::ID IID;
4705 // Check the character before ".512" in string.
4706 if (Name[Name.size() - 5] == 's')
4707 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
4708 else
4709 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
4710
4711 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4712 CI->getArgOperand(2), CI->getArgOperand(4)};
4713 if (IsSubAdd)
4714 Ops[2] = Builder.CreateFNeg(Ops[2]);
4715
4716 Rep = Builder.CreateIntrinsic(IID, Ops);
4717 } else {
4718 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4719
4720 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4721 CI->getArgOperand(2)};
4722
4723 Function *FMA = Intrinsic::getOrInsertDeclaration(
4724 CI->getModule(), Intrinsic::fma, Ops[0]->getType());
4725 Value *Odd = Builder.CreateCall(FMA, Ops);
4726 Ops[2] = Builder.CreateFNeg(Ops[2]);
4727 Value *Even = Builder.CreateCall(FMA, Ops);
4728
4729 if (IsSubAdd)
4730 std::swap(Even, Odd);
4731
4732 SmallVector<int, 32> Idxs(NumElts);
4733 for (int i = 0; i != NumElts; ++i)
4734 Idxs[i] = i + (i % 2) * NumElts;
4735
4736 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
4737 }
4738
4739 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
4740 : IsMask3 ? CI->getArgOperand(2)
4741 : CI->getArgOperand(0);
4742
4743 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4744 } else if (Name.starts_with("avx512.mask.pternlog.") ||
4745 Name.starts_with("avx512.maskz.pternlog.")) {
4746 bool ZeroMask = Name[11] == 'z';
4747 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4748 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
4749 Intrinsic::ID IID;
4750 if (VecWidth == 128 && EltWidth == 32)
4751 IID = Intrinsic::x86_avx512_pternlog_d_128;
4752 else if (VecWidth == 256 && EltWidth == 32)
4753 IID = Intrinsic::x86_avx512_pternlog_d_256;
4754 else if (VecWidth == 512 && EltWidth == 32)
4755 IID = Intrinsic::x86_avx512_pternlog_d_512;
4756 else if (VecWidth == 128 && EltWidth == 64)
4757 IID = Intrinsic::x86_avx512_pternlog_q_128;
4758 else if (VecWidth == 256 && EltWidth == 64)
4759 IID = Intrinsic::x86_avx512_pternlog_q_256;
4760 else if (VecWidth == 512 && EltWidth == 64)
4761 IID = Intrinsic::x86_avx512_pternlog_q_512;
4762 else
4763 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4764
4765 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4766 CI->getArgOperand(2), CI->getArgOperand(3)};
4767 Rep = Builder.CreateIntrinsic(IID, Args);
4768 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4769 : CI->getArgOperand(0);
4770 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
4771 } else if (Name.starts_with("avx512.mask.vpmadd52") ||
4772 Name.starts_with("avx512.maskz.vpmadd52")) {
4773 bool ZeroMask = Name[11] == 'z';
4774 bool High = Name[20] == 'h' || Name[21] == 'h';
4775 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4776 Intrinsic::ID IID;
4777 if (VecWidth == 128 && !High)
4778 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
4779 else if (VecWidth == 256 && !High)
4780 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
4781 else if (VecWidth == 512 && !High)
4782 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
4783 else if (VecWidth == 128 && High)
4784 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
4785 else if (VecWidth == 256 && High)
4786 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
4787 else if (VecWidth == 512 && High)
4788 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
4789 else
4790 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4791
4792 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4793 CI->getArgOperand(2)};
4794 Rep = Builder.CreateIntrinsic(IID, Args);
4795 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4796 : CI->getArgOperand(0);
4797 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4798 } else if (Name.starts_with("avx512.mask.vpermi2var.") ||
4799 Name.starts_with("avx512.mask.vpermt2var.") ||
4800 Name.starts_with("avx512.maskz.vpermt2var.")) {
4801 bool ZeroMask = Name[11] == 'z';
4802 bool IndexForm = Name[17] == 'i';
4803 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
4804 } else if (Name.starts_with("avx512.mask.vpdpbusd.") ||
4805 Name.starts_with("avx512.maskz.vpdpbusd.") ||
4806 Name.starts_with("avx512.mask.vpdpbusds.") ||
4807 Name.starts_with("avx512.maskz.vpdpbusds.")) {
4808 bool ZeroMask = Name[11] == 'z';
4809 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4810 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4811 Intrinsic::ID IID;
4812 if (VecWidth == 128 && !IsSaturating)
4813 IID = Intrinsic::x86_avx512_vpdpbusd_128;
4814 else if (VecWidth == 256 && !IsSaturating)
4815 IID = Intrinsic::x86_avx512_vpdpbusd_256;
4816 else if (VecWidth == 512 && !IsSaturating)
4817 IID = Intrinsic::x86_avx512_vpdpbusd_512;
4818 else if (VecWidth == 128 && IsSaturating)
4819 IID = Intrinsic::x86_avx512_vpdpbusds_128;
4820 else if (VecWidth == 256 && IsSaturating)
4821 IID = Intrinsic::x86_avx512_vpdpbusds_256;
4822 else if (VecWidth == 512 && IsSaturating)
4823 IID = Intrinsic::x86_avx512_vpdpbusds_512;
4824 else
4825 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4826
4827 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4828 CI->getArgOperand(2)};
4829
4830 // Input arguments types were incorrectly set to vectors of i32 before but
4831 // they should be vectors of i8. Insert bit cast when encountering the old
4832 // types
4833 if (Args[1]->getType()->isVectorTy() &&
4834 cast<VectorType>(Args[1]->getType())
4835 ->getElementType()
4836 ->isIntegerTy(32) &&
4837 Args[2]->getType()->isVectorTy() &&
4838 cast<VectorType>(Args[2]->getType())
4839 ->getElementType()
4840 ->isIntegerTy(32)) {
4841 Type *NewArgType = nullptr;
4842 if (VecWidth == 128)
4843 NewArgType = VectorType::get(Builder.getInt8Ty(), 16, false);
4844 else if (VecWidth == 256)
4845 NewArgType = VectorType::get(Builder.getInt8Ty(), 32, false);
4846 else if (VecWidth == 512)
4847 NewArgType = VectorType::get(Builder.getInt8Ty(), 64, false);
4848 else
4849 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4850 CI);
4851
4852 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4853 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4854 }
4855
4856 Rep = Builder.CreateIntrinsic(IID, Args);
4857 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4858 : CI->getArgOperand(0);
4859 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4860 } else if (Name.starts_with("avx512.mask.vpdpwssd.") ||
4861 Name.starts_with("avx512.maskz.vpdpwssd.") ||
4862 Name.starts_with("avx512.mask.vpdpwssds.") ||
4863 Name.starts_with("avx512.maskz.vpdpwssds.")) {
4864 bool ZeroMask = Name[11] == 'z';
4865 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
4866 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
4867 Intrinsic::ID IID;
4868 if (VecWidth == 128 && !IsSaturating)
4869 IID = Intrinsic::x86_avx512_vpdpwssd_128;
4870 else if (VecWidth == 256 && !IsSaturating)
4871 IID = Intrinsic::x86_avx512_vpdpwssd_256;
4872 else if (VecWidth == 512 && !IsSaturating)
4873 IID = Intrinsic::x86_avx512_vpdpwssd_512;
4874 else if (VecWidth == 128 && IsSaturating)
4875 IID = Intrinsic::x86_avx512_vpdpwssds_128;
4876 else if (VecWidth == 256 && IsSaturating)
4877 IID = Intrinsic::x86_avx512_vpdpwssds_256;
4878 else if (VecWidth == 512 && IsSaturating)
4879 IID = Intrinsic::x86_avx512_vpdpwssds_512;
4880 else
4881 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4882
4883 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4884 CI->getArgOperand(2)};
4885
4886 // Input arguments types were incorrectly set to vectors of i32 before but
4887 // they should be vectors of i16. Insert bit cast when encountering the old
4888 // types
4889 if (Args[1]->getType()->isVectorTy() &&
4890 cast<VectorType>(Args[1]->getType())
4891 ->getElementType()
4892 ->isIntegerTy(32) &&
4893 Args[2]->getType()->isVectorTy() &&
4894 cast<VectorType>(Args[2]->getType())
4895 ->getElementType()
4896 ->isIntegerTy(32)) {
4897 Type *NewArgType = nullptr;
4898 if (VecWidth == 128)
4899 NewArgType = VectorType::get(Builder.getInt16Ty(), 8, false);
4900 else if (VecWidth == 256)
4901 NewArgType = VectorType::get(Builder.getInt16Ty(), 16, false);
4902 else if (VecWidth == 512)
4903 NewArgType = VectorType::get(Builder.getInt16Ty(), 32, false);
4904 else
4905 reportFatalUsageErrorWithCI("Intrinsic has unexpected vector bit width",
4906 CI);
4907
4908 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
4909 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
4910 }
4911
4912 Rep = Builder.CreateIntrinsic(IID, Args);
4913 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
4914 : CI->getArgOperand(0);
4915 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
4916 } else if (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
4917 Name == "addcarry.u32" || Name == "addcarry.u64" ||
4918 Name == "subborrow.u32" || Name == "subborrow.u64") {
4919 Intrinsic::ID IID;
4920 if (Name[0] == 'a' && Name.back() == '2')
4921 IID = Intrinsic::x86_addcarry_32;
4922 else if (Name[0] == 'a' && Name.back() == '4')
4923 IID = Intrinsic::x86_addcarry_64;
4924 else if (Name[0] == 's' && Name.back() == '2')
4925 IID = Intrinsic::x86_subborrow_32;
4926 else if (Name[0] == 's' && Name.back() == '4')
4927 IID = Intrinsic::x86_subborrow_64;
4928 else
4929 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4930
4931 // Make a call with 3 operands.
4932 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
4933 CI->getArgOperand(2)};
4934 Value *NewCall = Builder.CreateIntrinsic(IID, Args);
4935
4936 // Extract the second result and store it.
4937 Value *Data = Builder.CreateExtractValue(NewCall, 1);
4938 Builder.CreateAlignedStore(Data, CI->getArgOperand(3), Align(1));
4939 // Replace the original call result with the first result of the new call.
4940 Value *CF = Builder.CreateExtractValue(NewCall, 0);
4941
4942 CI->replaceAllUsesWith(CF);
4943 Rep = nullptr;
4944 } else if (Name.starts_with("avx512.mask.") &&
4945 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
4946 // Rep will be updated by the call in the condition.
4947 } else if (Name.starts_with("bmi.pdep.")) {
4948 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pdep);
4949 } else if (Name.starts_with("bmi.pext.")) {
4950 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::pext);
4951 } else
4952 reportFatalUsageErrorWithCI("Unexpected intrinsic", CI);
4953
4954 return Rep;
4955}
4956
4957static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
4958 Function *F, IRBuilder<> &Builder) {
4959 if (Name.starts_with("neon.bfcvt")) {
4960 if (Name.starts_with("neon.bfcvtn2")) {
4961 SmallVector<int, 32> LoMask(4);
4962 std::iota(LoMask.begin(), LoMask.end(), 0);
4963 SmallVector<int, 32> ConcatMask(8);
4964 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4965 Value *Inactive = Builder.CreateShuffleVector(CI->getOperand(0), LoMask);
4966 Value *Trunc =
4967 Builder.CreateFPTrunc(CI->getOperand(1), Inactive->getType());
4968 return Builder.CreateShuffleVector(Inactive, Trunc, ConcatMask);
4969 } else if (Name.starts_with("neon.bfcvtn")) {
4970 SmallVector<int, 32> ConcatMask(8);
4971 std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
4972 Type *V4BF16 =
4973 FixedVectorType::get(Type::getBFloatTy(F->getContext()), 4);
4974 Value *Trunc = Builder.CreateFPTrunc(CI->getOperand(0), V4BF16);
4975 dbgs() << "Trunc: " << *Trunc << "\n";
4976 return Builder.CreateShuffleVector(
4977 Trunc, ConstantAggregateZero::get(V4BF16), ConcatMask);
4978 } else {
4979 return Builder.CreateFPTrunc(CI->getOperand(0),
4980 Type::getBFloatTy(F->getContext()));
4981 }
4982 } else if (Name.starts_with("sve.fcvt")) {
4983 Intrinsic::ID NewID =
4984 StringSwitch<Intrinsic::ID>(Name)
4985 .Case("sve.fcvt.bf16f32", Intrinsic::aarch64_sve_fcvt_bf16f32_v2)
4986 .Case("sve.fcvtnt.bf16f32",
4987 Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2)
4988 .Default(Intrinsic::not_intrinsic);
4989 if (NewID == Intrinsic::not_intrinsic)
4990 llvm_unreachable("Unhandled Intrinsic!");
4991
4992 SmallVector<Value *, 3> Args(CI->args());
4993
4994 // The original intrinsics incorrectly used a predicate based on the
4995 // smallest element type rather than the largest.
4996 Type *BadPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 8);
4997 Type *GoodPredTy = ScalableVectorType::get(Builder.getInt1Ty(), 4);
4998
4999 if (Args[1]->getType() != BadPredTy)
5000 llvm_unreachable("Unexpected predicate type!");
5001
5002 Args[1] = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool,
5003 BadPredTy, Args[1]);
5004 Args[1] = Builder.CreateIntrinsic(
5005 Intrinsic::aarch64_sve_convert_from_svbool, GoodPredTy, Args[1]);
5006
5007 return Builder.CreateIntrinsic(NewID, Args, /*FMFSource=*/nullptr,
5008 CI->getName());
5009 }
5010
5011 if (Name == "neon.vcvtfp2hf")
5012 return Builder.CreateBitCast(
5013 Builder.CreateFPTrunc(
5014 CI->getOperand(0),
5015 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5016 FixedVectorType::get(Type::getInt16Ty(F->getContext()), 4));
5017 if (Name == "neon.vcvthf2fp")
5018 return Builder.CreateFPExt(
5019 Builder.CreateBitCast(
5020 CI->getOperand(0),
5021 FixedVectorType::get(Type::getHalfTy(F->getContext()), 4)),
5022 FixedVectorType::get(Type::getFloatTy(F->getContext()), 4));
5023
5024 llvm_unreachable("Unhandled Intrinsic!");
5025}
5026
5027static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
5028 IRBuilder<> &Builder) {
5029 if (Name == "mve.vctp64.old") {
5030 // Replace the old v4i1 vctp64 with a v2i1 vctp and predicate-casts to the
5031 // correct type.
5032 Value *VCTP = Builder.CreateIntrinsic(Intrinsic::arm_mve_vctp64, {},
5033 CI->getArgOperand(0),
5034 /*FMFSource=*/nullptr, CI->getName());
5035 Value *C1 = Builder.CreateIntrinsic(
5036 Intrinsic::arm_mve_pred_v2i,
5037 {VectorType::get(Builder.getInt1Ty(), 2, false)}, VCTP);
5038 return Builder.CreateIntrinsic(
5039 Intrinsic::arm_mve_pred_i2v,
5040 {VectorType::get(Builder.getInt1Ty(), 4, false)}, C1);
5041 } else if (Name == "mve.mull.int.predicated.v2i64.v4i32.v4i1" ||
5042 Name == "mve.vqdmull.predicated.v2i64.v4i32.v4i1" ||
5043 Name == "mve.vldr.gather.base.predicated.v2i64.v2i64.v4i1" ||
5044 Name == "mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1" ||
5045 Name ==
5046 "mve.vldr.gather.offset.predicated.v2i64.p0i64.v2i64.v4i1" ||
5047 Name == "mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v4i1" ||
5048 Name == "mve.vstr.scatter.base.predicated.v2i64.v2i64.v4i1" ||
5049 Name == "mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v4i1" ||
5050 Name ==
5051 "mve.vstr.scatter.offset.predicated.p0i64.v2i64.v2i64.v4i1" ||
5052 Name == "mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v4i1" ||
5053 Name == "cde.vcx1q.predicated.v2i64.v4i1" ||
5054 Name == "cde.vcx1qa.predicated.v2i64.v4i1" ||
5055 Name == "cde.vcx2q.predicated.v2i64.v4i1" ||
5056 Name == "cde.vcx2qa.predicated.v2i64.v4i1" ||
5057 Name == "cde.vcx3q.predicated.v2i64.v4i1" ||
5058 Name == "cde.vcx3qa.predicated.v2i64.v4i1") {
5059 std::vector<Type *> Tys;
5060 unsigned ID = CI->getIntrinsicID();
5061 Type *V2I1Ty = FixedVectorType::get(Builder.getInt1Ty(), 2);
5062 switch (ID) {
5063 case Intrinsic::arm_mve_mull_int_predicated:
5064 case Intrinsic::arm_mve_vqdmull_predicated:
5065 case Intrinsic::arm_mve_vldr_gather_base_predicated:
5066 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
5067 break;
5068 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated:
5069 case Intrinsic::arm_mve_vstr_scatter_base_predicated:
5070 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated:
5071 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
5072 V2I1Ty};
5073 break;
5074 case Intrinsic::arm_mve_vldr_gather_offset_predicated:
5075 Tys = {CI->getType(), CI->getOperand(0)->getType(),
5076 CI->getOperand(1)->getType(), V2I1Ty};
5077 break;
5078 case Intrinsic::arm_mve_vstr_scatter_offset_predicated:
5079 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
5080 CI->getOperand(2)->getType(), V2I1Ty};
5081 break;
5082 case Intrinsic::arm_cde_vcx1q_predicated:
5083 case Intrinsic::arm_cde_vcx1qa_predicated:
5084 case Intrinsic::arm_cde_vcx2q_predicated:
5085 case Intrinsic::arm_cde_vcx2qa_predicated:
5086 case Intrinsic::arm_cde_vcx3q_predicated:
5087 case Intrinsic::arm_cde_vcx3qa_predicated:
5088 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
5089 break;
5090 default:
5091 llvm_unreachable("Unhandled Intrinsic!");
5092 }
5093
5094 std::vector<Value *> Ops;
5095 for (Value *Op : CI->args()) {
5096 Type *Ty = Op->getType();
5097 if (Ty->getScalarSizeInBits() == 1) {
5098 Value *C1 = Builder.CreateIntrinsic(
5099 Intrinsic::arm_mve_pred_v2i,
5100 {VectorType::get(Builder.getInt1Ty(), 4, false)}, Op);
5101 Op = Builder.CreateIntrinsic(Intrinsic::arm_mve_pred_i2v, {V2I1Ty}, C1);
5102 }
5103 Ops.push_back(Op);
5104 }
5105
5106 return Builder.CreateIntrinsic(ID, Tys, Ops, /*FMFSource=*/nullptr,
5107 CI->getName());
5108 }
5109 llvm_unreachable("Unknown function for ARM CallBase upgrade.");
5110}
5111
5112// These are expected to have the arguments:
5113// atomic.intrin (ptr, rmw_value, ordering, scope, isVolatile)
5114//
5115// Except for int_amdgcn_ds_fadd_v2bf16 which only has (ptr, rmw_value).
5116//
5117static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
5118 Function *F, IRBuilder<> &Builder) {
5119 // Legacy WMMA iu intrinsics missed the optional clamp operand. Append clamp=0
5120 // for compatibility.
5121 auto UpgradeLegacyWMMAIUIntrinsicCall =
5122 [](Function *F, CallBase *CI, IRBuilder<> &Builder,
5123 ArrayRef<Type *> OverloadTys) -> Value * {
5124 // Prepare arguments, append clamp=0 for compatibility
5125 SmallVector<Value *, 10> Args(CI->args().begin(), CI->args().end());
5126 Args.push_back(Builder.getFalse());
5127
5128 // Insert the declaration for the right overload types
5129 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5130 F->getParent(), F->getIntrinsicID(), OverloadTys);
5131
5132 // Copy operand bundles if any
5133 SmallVector<OperandBundleDef, 1> Bundles;
5134 CI->getOperandBundlesAsDefs(Bundles);
5135
5136 // Create the new call and copy calling properties
5137 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5138 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5139 NewCall->setCallingConv(CI->getCallingConv());
5140 NewCall->setAttributes(CI->getAttributes());
5141 NewCall->copyMetadata(*CI);
5142 return NewCall;
5143 };
5144
5145 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_i32_16x16x64_iu8) {
5146 assert(CI->arg_size() == 7 && "Legacy int_amdgcn_wmma_i32_16x16x64_iu8 "
5147 "intrinsic should have 7 arguments");
5148 Type *T1 = CI->getArgOperand(4)->getType();
5149 Type *T2 = CI->getArgOperand(1)->getType();
5150 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2});
5151 }
5152 if (F->getIntrinsicID() == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8) {
5153 assert(CI->arg_size() == 8 && "Legacy int_amdgcn_swmmac_i32_16x16x128_iu8 "
5154 "intrinsic should have 8 arguments");
5155 Type *T1 = CI->getArgOperand(4)->getType();
5156 Type *T2 = CI->getArgOperand(1)->getType();
5157 Type *T3 = CI->getArgOperand(3)->getType();
5158 Type *T4 = CI->getArgOperand(5)->getType();
5159 return UpgradeLegacyWMMAIUIntrinsicCall(F, CI, Builder, {T1, T2, T3, T4});
5160 }
5161
5162 switch (F->getIntrinsicID()) {
5163 default:
5164 break;
5165 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
5166 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
5167 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
5168 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
5169 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
5170 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16: {
5171 // Drop src0 and src1 modifiers.
5172 const Value *Op0 = CI->getArgOperand(0);
5173 const Value *Op2 = CI->getArgOperand(2);
5174 assert(Op0->getType()->isIntegerTy() && Op2->getType()->isIntegerTy());
5175 const ConstantInt *ModA = dyn_cast<ConstantInt>(Op0);
5176 const ConstantInt *ModB = dyn_cast<ConstantInt>(Op2);
5177 if (!ModA->isZero() || !ModB->isZero())
5178 reportFatalUsageError(Name + " matrix A and B modifiers shall be zero");
5179
5180 SmallVector<Value *, 8> Args{CI->getArgOperand(1), CI->getArgOperand(3)};
5181 for (int I = 4, E = CI->arg_size(); I < E; ++I)
5182 Args.push_back(CI->getArgOperand(I));
5183
5184 SmallVector<Type *, 3> Overloads{F->getReturnType(), Args[0]->getType()};
5185 if (F->getIntrinsicID() == Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16)
5186 Overloads.push_back(Args[3]->getType());
5187 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
5188 F->getParent(), F->getIntrinsicID(), Overloads);
5189
5190 SmallVector<OperandBundleDef, 1> Bundles;
5191 CI->getOperandBundlesAsDefs(Bundles);
5192
5193 auto *NewCall = cast<CallInst>(Builder.CreateCall(NewDecl, Args, Bundles));
5194 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5195 NewCall->setCallingConv(CI->getCallingConv());
5196 NewCall->setAttributes(CI->getAttributes());
5197 NewCall->copyMetadata(*CI);
5198 NewCall->takeName(CI);
5199 return NewCall;
5200 }
5201 }
5202
5203 if (Name.starts_with("fcmp.") || Name.starts_with("icmp.")) {
5204 Value *LHS = CI->getArgOperand(0);
5205 Value *RHS = CI->getArgOperand(1);
5206 CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(
5207 cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue());
5208 Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS);
5209 CallInst *NewCall = Builder.CreateIntrinsicWithoutFolding(
5210 CI->getType(), Intrinsic::amdgcn_ballot, Cmp);
5211 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
5212 NewCall->setCallingConv(CI->getCallingConv());
5213 NewCall->copyMetadata(*CI);
5214 NewCall->takeName(CI);
5215 return NewCall;
5216 }
5217
5218 AtomicRMWInst::BinOp RMWOp =
5219 StringSwitch<AtomicRMWInst::BinOp>(Name)
5220 .StartsWith("ds.fadd", AtomicRMWInst::FAdd)
5221 .StartsWith("ds.fmin", AtomicRMWInst::FMin)
5222 .StartsWith("ds.fmax", AtomicRMWInst::FMax)
5223 .StartsWith("atomic.inc.", AtomicRMWInst::UIncWrap)
5224 .StartsWith("atomic.dec.", AtomicRMWInst::UDecWrap)
5225 .StartsWith("global.atomic.fadd", AtomicRMWInst::FAdd)
5226 .StartsWith("flat.atomic.fadd", AtomicRMWInst::FAdd)
5227 .StartsWith("global.atomic.fmin", AtomicRMWInst::FMin)
5228 .StartsWith("flat.atomic.fmin", AtomicRMWInst::FMin)
5229 .StartsWith("global.atomic.fmax", AtomicRMWInst::FMax)
5230 .StartsWith("flat.atomic.fmax", AtomicRMWInst::FMax)
5231 .StartsWith("atomic.cond.sub", AtomicRMWInst::USubCond)
5232 .StartsWith("atomic.csub", AtomicRMWInst::USubSat);
5233
5234 unsigned NumOperands = CI->getNumOperands();
5235 if (NumOperands < 3) // Malformed bitcode.
5236 return nullptr;
5237
5238 Value *Ptr = CI->getArgOperand(0);
5239 PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
5240 if (!PtrTy) // Malformed.
5241 return nullptr;
5242
5243 Value *Val = CI->getArgOperand(1);
5244 if (Val->getType() != CI->getType()) // Malformed.
5245 return nullptr;
5246
5247 ConstantInt *OrderArg = nullptr;
5248 bool IsVolatile = false;
5249
5250 // These should have 5 arguments (plus the callee). A separate version of the
5251 // ds_fadd intrinsic was defined for bf16 which was missing arguments.
5252 if (NumOperands > 3)
5253 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5254
5255 // Ignore scope argument at 3
5256
5257 if (NumOperands > 5) {
5258 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
5259 IsVolatile = !VolatileArg || !VolatileArg->isZero();
5260 }
5261
5262 AtomicOrdering Order = AtomicOrdering::SequentiallyConsistent;
5263 if (OrderArg && isValidAtomicOrdering(OrderArg->getZExtValue()))
5264 Order = static_cast<AtomicOrdering>(OrderArg->getZExtValue());
5265 if (Order == AtomicOrdering::NotAtomic || Order == AtomicOrdering::Unordered)
5266 Order = AtomicOrdering::SequentiallyConsistent;
5267
5268 LLVMContext &Ctx = F->getContext();
5269
5270 // Handle the v2bf16 intrinsic which used <2 x i16> instead of <2 x bfloat>
5271 Type *RetTy = CI->getType();
5272 if (VectorType *VT = dyn_cast<VectorType>(RetTy)) {
5273 if (VT->getElementType()->isIntegerTy(16)) {
5274 VectorType *AsBF16 =
5275 VectorType::get(Type::getBFloatTy(Ctx), VT->getElementCount());
5276 Val = Builder.CreateBitCast(Val, AsBF16);
5277 }
5278 }
5279
5280 // The scope argument never really worked correctly. Use agent as the most
5281 // conservative option which should still always produce the instruction.
5282 SyncScope::ID SSID = Ctx.getOrInsertSyncScopeID("agent");
5283 AtomicRMWInst *RMW =
5284 Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);
5285
5286 unsigned AddrSpace = PtrTy->getAddressSpace();
5287 if (AddrSpace != AMDGPUAS::LOCAL_ADDRESS) {
5288 MDNode *EmptyMD = MDNode::get(F->getContext(), {});
5289 RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
5290 if (RMWOp == AtomicRMWInst::FAdd && RetTy->isFloatTy())
5291 RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
5292 }
5293
5294 if (AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
5295 MDBuilder MDB(F->getContext());
5296 MDNode *RangeNotPrivate =
5297 MDB.createRange(APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
5298 APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
5299 RMW->setMetadata(LLVMContext::MD_noalias_addrspace, RangeNotPrivate);
5300 }
5301
5302 if (IsVolatile)
5303 RMW->setVolatile(true);
5304
5305 return Builder.CreateBitCast(RMW, RetTy);
5306}
5307
5308/// Helper to unwrap intrinsic call MetadataAsValue operands. Return as a
5309/// plain MDNode, as it's the verifier's job to check these are the correct
5310/// types later.
5311static MDNode *unwrapMAVOp(CallBase *CI, unsigned Op) {
5312 if (Op < CI->arg_size()) {
5313 if (MetadataAsValue *MAV =
5314 dyn_cast<MetadataAsValue>(CI->getArgOperand(Op))) {
5315 Metadata *MD = MAV->getMetadata();
5316 return dyn_cast_if_present<MDNode>(MD);
5317 }
5318 }
5319 return nullptr;
5320}
5321
5322/// Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
5323static Metadata *unwrapMAVMetadataOp(CallBase *CI, unsigned Op) {
5324 if (Op < CI->arg_size())
5325 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
5326 return MAV->getMetadata();
5327 return nullptr;
5328}
5329
5330/// Convert debug intrinsic calls to non-instruction debug records.
5331/// \p Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value.
5332/// \p CI - The debug intrinsic call.
5333static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
5334 DbgRecord *DR = nullptr;
5335 if (Name == "label") {
5336 DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(unwrapMAVOp(CI, 0));
5337 } else if (Name == "assign") {
5338 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5339 DbgVariableRecord::LocationType::Assign, unwrapMAVMetadataOp(CI, 0),
5340 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), unwrapMAVOp(CI, 3),
5341 unwrapMAVMetadataOp(CI, 4),
5342 /*The address is a Value ref, it will be stored as a Metadata */
5343 unwrapMAVOp(CI, 5));
5344 } else if (Name == "declare") {
5345 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5346 DbgVariableRecord::LocationType::Declare, unwrapMAVMetadataOp(CI, 0),
5347 unwrapMAVOp(CI, 1), unwrapMAVOp(CI, 2), nullptr, nullptr, nullptr);
5348 } else if (Name == "addr") {
5349 // Upgrade dbg.addr to dbg.value with DW_OP_deref.
5350 MDNode *ExprNode = unwrapMAVOp(CI, 2);
5351 // Don't try to add something to the expression if it's not an expression.
5352 // Instead, allow the verifier to fail later.
5353 if (DIExpression *Expr = dyn_cast<DIExpression>(ExprNode)) {
5354 ExprNode = DIExpression::append(Expr, dwarf::DW_OP_deref);
5355 }
5356 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5357 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5358 unwrapMAVOp(CI, 1), ExprNode, nullptr, nullptr, nullptr);
5359 } else if (Name == "value") {
5360 // An old version of dbg.value had an extra offset argument.
5361 unsigned VarOp = 1;
5362 unsigned ExprOp = 2;
5363 if (CI->arg_size() == 4) {
5364 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
5365 // Nonzero offset dbg.values get dropped without a replacement.
5366 if (!Offset || !Offset->isNullValue())
5367 return;
5368 VarOp = 2;
5369 ExprOp = 3;
5370 }
5371 DR = DbgVariableRecord::createUnresolvedDbgVariableRecord(
5372 DbgVariableRecord::LocationType::Value, unwrapMAVMetadataOp(CI, 0),
5373 unwrapMAVOp(CI, VarOp), unwrapMAVOp(CI, ExprOp), nullptr, nullptr,
5374 nullptr);
5375 }
5376 DR->setDebugLoc(CI->getDebugLoc());
5377 assert(DR && "Unhandled intrinsic kind in upgrade to DbgRecord");
5378 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
5379}
5380
5381static Value *upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder) {
5382 auto *Offset = dyn_cast<ConstantInt>(CI->getArgOperand(2));
5383 if (!Offset)
5384 reportFatalUsageError("Invalid llvm.vector.splice offset argument");
5385 int64_t OffsetVal = Offset->getSExtValue();
5386 return Builder.CreateIntrinsic(OffsetVal >= 0
5387 ? Intrinsic::vector_splice_left
5388 : Intrinsic::vector_splice_right,
5389 CI->getType(),
5390 {CI->getArgOperand(0), CI->getArgOperand(1),
5391 Builder.getInt32(std::abs(OffsetVal))});
5392}
5393
5394static Value *upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI,
5395 Function *F, IRBuilder<> &Builder) {
5396 if (Name.starts_with("to.fp16")) {
5397 Value *Cast =
5398 Builder.CreateFPTrunc(CI->getArgOperand(0), Builder.getHalfTy());
5399 return Builder.CreateBitCast(Cast, CI->getType());
5400 }
5401
5402 if (Name.starts_with("from.fp16")) {
5403 Value *Cast =
5404 Builder.CreateBitCast(CI->getArgOperand(0), Builder.getHalfTy());
5405 return Builder.CreateFPExt(Cast, CI->getType());
5406 }
5407
5408 return nullptr;
5409}
5410
5411static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op) {
5412 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5413 if (!MD || !isa<MDString>(MD))
5414 return ICmpInst::BAD_ICMP_PREDICATE;
5415 return StringSwitch<ICmpInst::Predicate>(cast<MDString>(MD)->getString())
5416 .Case("eq", ICmpInst::ICMP_EQ)
5417 .Case("ne", ICmpInst::ICMP_NE)
5418 .Case("ugt", ICmpInst::ICMP_UGT)
5419 .Case("uge", ICmpInst::ICMP_UGE)
5420 .Case("ult", ICmpInst::ICMP_ULT)
5421 .Case("ule", ICmpInst::ICMP_ULE)
5422 .Case("sgt", ICmpInst::ICMP_SGT)
5423 .Case("sge", ICmpInst::ICMP_SGE)
5424 .Case("slt", ICmpInst::ICMP_SLT)
5425 .Case("sle", ICmpInst::ICMP_SLE)
5426 .Default(ICmpInst::BAD_ICMP_PREDICATE);
5427}
5428
5429static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op) {
5430 Metadata *MD = cast<MetadataAsValue>(Op)->getMetadata();
5431 if (!MD || !isa<MDString>(MD))
5432 return FCmpInst::BAD_FCMP_PREDICATE;
5433 return StringSwitch<FCmpInst::Predicate>(cast<MDString>(MD)->getString())
5434 .Case("oeq", FCmpInst::FCMP_OEQ)
5435 .Case("ogt", FCmpInst::FCMP_OGT)
5436 .Case("oge", FCmpInst::FCMP_OGE)
5437 .Case("olt", FCmpInst::FCMP_OLT)
5438 .Case("ole", FCmpInst::FCMP_OLE)
5439 .Case("one", FCmpInst::FCMP_ONE)
5440 .Case("ord", FCmpInst::FCMP_ORD)
5441 .Case("uno", FCmpInst::FCMP_UNO)
5442 .Case("ueq", FCmpInst::FCMP_UEQ)
5443 .Case("ugt", FCmpInst::FCMP_UGT)
5444 .Case("uge", FCmpInst::FCMP_UGE)
5445 .Case("ult", FCmpInst::FCMP_ULT)
5446 .Case("ule", FCmpInst::FCMP_ULE)
5447 .Case("une", FCmpInst::FCMP_UNE)
5448 .Default(FCmpInst::BAD_FCMP_PREDICATE);
5449}
5450
5451static Value *upgradeVPIntrinsicCall(StringRef Name, CallBase *CI,
5452 IRBuilder<> &Builder) {
5453 Value *Rep;
5454 unsigned Opcode = getFunctionalOpcodeForVP(Name);
5455 if (Opcode && Instruction::isUnaryOp(Opcode))
5456 Rep =
5457 Builder.CreateUnOp((Instruction::UnaryOps)Opcode, CI->getArgOperand(0));
5458 else if (Opcode && Instruction::isBinaryOp(Opcode))
5459 Rep = Builder.CreateBinOp((Instruction::BinaryOps)Opcode,
5460 CI->getArgOperand(0), CI->getArgOperand(1));
5461 else if (Opcode && Instruction::isCast(Opcode))
5462 Rep = Builder.CreateCast((Instruction::CastOps)Opcode, CI->getArgOperand(0),
5463 CI->getType());
5464 else if (Opcode == Instruction::ICmp)
5465 Rep = Builder.CreateICmp(getVPIntPredicateFromMD(CI->getArgOperand(2)),
5466 CI->getArgOperand(0), CI->getArgOperand(1));
5467 else if (Opcode == Instruction::FCmp)
5468 Rep = Builder.CreateFCmp(getVPFPPredicateFromMD(CI->getArgOperand(2)),
5469 CI->getArgOperand(0), CI->getArgOperand(1));
5470 else if (Opcode == Instruction::Select)
5471 Rep = Builder.CreateSelect(CI->getArgOperand(0), CI->getArgOperand(1),
5472 CI->getArgOperand(2));
5473 else if (auto IntrinsicID = getFunctionalIntrinsicIDForVP(Name)) {
5474 SmallVector<Value *, 2> Args(drop_end(CI->args(), 2));
5475 Rep = Builder.CreateIntrinsic(CI->getType(), IntrinsicID, Args, {});
5476 } else
5477 llvm_unreachable("Unexpected vp intrinsic");
5478 Rep->takeName(CI);
5479 return Rep;
5480}
5481
5482static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn,
5483 IRBuilder<> &Builder) {
5484 Intrinsic::ID IID = NewFn->getIntrinsicID();
5485
5486 auto [FirstDefault, Defaults] = Intrinsic::getAllDefaultArgValues(IID);
5487 if (Defaults.empty())
5488 return false;
5489
5490 unsigned OldArgCount = CI->arg_size();
5491 unsigned NewArgCount = NewFn->arg_size();
5492
5493 // If the caller already supplied all arguments (or more), nothing to do.
5494 // This mirrors C++ semantics: an explicitly-passed value is never overridden.
5495 if (OldArgCount >= NewArgCount)
5496 return false;
5497
5498 // Start with the existing arguments from the old call.
5499 SmallVector<Value *, 8> NewArgs(CI->args());
5500
5501 // Defaults are a contiguous trailing block, so checking the first missing
5502 // argument is enough.
5503 if (OldArgCount < FirstDefault)
5504 return false;
5505
5506 // Fill in each missing trailing argument from the table.
5507 FunctionType *NewFT = NewFn->getFunctionType();
5508 for (unsigned Idx = OldArgCount; Idx < NewArgCount; ++Idx) {
5509 assert(Idx >= FirstDefault && Idx - FirstDefault < Defaults.size() &&
5510 "missing argument outside the default range");
5511 Type *ParamTy = NewFT->getParamType(Idx);
5512
5513 // Only integer types are supported (i1, i8, i16, i32, i64).
5514 if (!ParamTy->isIntegerTy())
5515 return false;
5516 NewArgs.push_back(ConstantInt::get(ParamTy, Defaults[Idx - FirstDefault]));
5517 }
5518
5519 // Preserve operand bundles by creating the call with them.
5520 SmallVector<OperandBundleDef, 1> OpBundles;
5521 CI->getOperandBundlesAsDefs(OpBundles);
5522 CallInst *NewCall = Builder.CreateCall(NewFn, NewArgs, OpBundles);
5523
5524 NewCall->takeName(CI);
5525 NewCall->setCallingConv(CI->getCallingConv());
5526 NewCall->copyMetadata(*CI);
5527 if (auto *OldCI = dyn_cast<CallInst>(CI))
5528 NewCall->setTailCallKind(OldCI->getTailCallKind());
5529
5530 CI->replaceAllUsesWith(NewCall);
5531 CI->eraseFromParent();
5532 return true;
5533}
5534
5535/// Upgrade a call to an old intrinsic. All argument and return casting must be
5536/// provided to seamlessly integrate with existing context.
5537void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
5538 // Note dyn_cast to Function is not quite the same as getCalledFunction, which
5539 // checks the callee's function type matches. It's likely we need to handle
5540 // type changes here.
5541 Function *F = dyn_cast<Function>(CI->getCalledOperand());
5542 if (!F)
5543 return;
5544
5545 LLVMContext &C = CI->getContext();
5546 IRBuilder<> Builder(C);
5547 if (isa<FPMathOperator>(CI))
5548 Builder.setFastMathFlags(CI->getFastMathFlags());
5549 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
5550
5551 if (!NewFn) {
5552 // Get the Function's name.
5553 StringRef Name = F->getName();
5554 if (!Name.consume_front("llvm."))
5555 llvm_unreachable("intrinsic doesn't start with 'llvm.'");
5556
5557 bool IsX86 = Name.consume_front("x86.");
5558 bool IsNVVM = Name.consume_front("nvvm.");
5559 bool IsAArch64 = Name.consume_front("aarch64.");
5560 bool IsARM = Name.consume_front("arm.");
5561 bool IsAMDGCN = Name.consume_front("amdgcn.");
5562 bool IsDbg = Name.consume_front("dbg.");
5563 bool IsOldSplice =
5564 (Name.consume_front("experimental.vector.splice") ||
5565 Name.consume_front("vector.splice")) &&
5566 !(Name.starts_with(".left") || Name.starts_with(".right"));
5567 Value *Rep = nullptr;
5568
5569 if (!IsX86 && Name == "stackprotectorcheck") {
5570 Rep = nullptr;
5571 } else if (IsNVVM) {
5572 Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
5573 } else if (IsX86) {
5574 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
5575 } else if (IsAArch64) {
5576 Rep = upgradeAArch64IntrinsicCall(Name, CI, F, Builder);
5577 } else if (IsARM) {
5578 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
5579 } else if (IsAMDGCN) {
5580 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
5581 } else if (IsDbg) {
5582 upgradeDbgIntrinsicToDbgRecord(Name, CI);
5583 } else if (IsOldSplice) {
5584 Rep = upgradeVectorSplice(CI, Builder);
5585 } else if (Name.consume_front("convert.")) {
5586 Rep = upgradeConvertIntrinsicCall(Name, CI, F, Builder);
5587 } else if (Name == "lifetime.start.i64" || Name == "lifetime.end.i64") {
5588 // Delete calls to invalid @llvm.lifetime.{start,end}.i64 intrinsics.
5589 Rep = nullptr;
5590 } else if (shouldUpgradeVPIntrinsic(Name)) {
5591 Rep = upgradeVPIntrinsicCall(Name, CI, Builder);
5592 } else {
5593 llvm_unreachable("Unknown function for CallBase upgrade.");
5594 }
5595
5596 if (Rep)
5597 CI->replaceAllUsesWith(Rep);
5598 CI->eraseFromParent();
5599 return;
5600 }
5601
5602 const auto &DefaultCase = [&]() -> void {
5603 if (F == NewFn)
5604 return;
5605
5606 if (CI->getFunctionType() == NewFn->getFunctionType()) {
5607 // Handle generic mangling change.
5608 assert(
5609 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
5610 "Unknown function for CallBase upgrade and isn't just a name change");
5611 CI->setCalledFunction(NewFn);
5612 return;
5613 }
5614
5615 // This must be an upgrade from a named to a literal struct.
5616 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
5617 assert(OldST != NewFn->getReturnType() &&
5618 "Return type must have changed");
5619 assert(OldST->getNumElements() ==
5620 cast<StructType>(NewFn->getReturnType())->getNumElements() &&
5621 "Must have same number of elements");
5622
5623 SmallVector<Value *> Args(CI->args());
5624 CallInst *NewCI = Builder.CreateCall(NewFn, Args);
5625 NewCI->setAttributes(CI->getAttributes());
5626 Value *Res = PoisonValue::get(OldST);
5627 for (unsigned Idx = 0; Idx < OldST->getNumElements(); ++Idx) {
5628 Value *Elem = Builder.CreateExtractValue(NewCI, Idx);
5629 Res = Builder.CreateInsertValue(Res, Elem, Idx);
5630 }
5631 CI->replaceAllUsesWith(Res);
5632 CI->eraseFromParent();
5633 return;
5634 }
5635
5636 // We're probably about to produce something invalid. Let the verifier catch
5637 // it instead of dying here.
5638 CI->setCalledOperand(
5639 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
5640 return;
5641 };
5642 CallInst *NewCall = nullptr;
5643 switch (NewFn->getIntrinsicID()) {
5644 default: {
5645 // Last resort: try the data-driven default-arg upgrade.
5646 // Handles any intrinsic annotated with ImmArg<..., DefaultValue<...>>
5647 // in its .td definition, without needing a dedicated case.
5648 if (upgradeIntrinsicCallWithDefaultArgs(CI, NewFn, Builder))
5649 return;
5650 DefaultCase();
5651 return;
5652 }
5653 case Intrinsic::arm_neon_vst1:
5654 case Intrinsic::arm_neon_vst2:
5655 case Intrinsic::arm_neon_vst3:
5656 case Intrinsic::arm_neon_vst4:
5657 case Intrinsic::arm_neon_vst2lane:
5658 case Intrinsic::arm_neon_vst3lane:
5659 case Intrinsic::arm_neon_vst4lane: {
5660 SmallVector<Value *, 4> Args(CI->args());
5661 NewCall = Builder.CreateCall(NewFn, Args);
5662 break;
5663 }
5664 case Intrinsic::aarch64_sve_bfmlalb_lane_v2:
5665 case Intrinsic::aarch64_sve_bfmlalt_lane_v2:
5666 case Intrinsic::aarch64_sve_bfdot_lane_v2: {
5667 LLVMContext &Ctx = F->getParent()->getContext();
5668 SmallVector<Value *, 4> Args(CI->args());
5669 Args[3] = ConstantInt::get(Type::getInt32Ty(Ctx),
5670 cast<ConstantInt>(Args[3])->getZExtValue());
5671 NewCall = Builder.CreateCall(NewFn, Args);
5672 break;
5673 }
5674 case Intrinsic::aarch64_sve_ld3_sret:
5675 case Intrinsic::aarch64_sve_ld4_sret:
5676 case Intrinsic::aarch64_sve_ld2_sret: {
5677 // Is this a trivial remangle of the name to support ptr address spaces?
5678 if (isa<StructType>(F->getReturnType())) {
5679 DefaultCase();
5680 return;
5681 }
5682
5683 StringRef Name = F->getName();
5684 Name = Name.substr(5);
5685 unsigned N = StringSwitch<unsigned>(Name)
5686 .StartsWith("aarch64.sve.ld2", 2)
5687 .StartsWith("aarch64.sve.ld3", 3)
5688 .StartsWith("aarch64.sve.ld4", 4)
5689 .Default(0);
5690 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5691 unsigned MinElts = RetTy->getMinNumElements() / N;
5692 SmallVector<Value *, 2> Args(CI->args());
5693 Value *NewLdCall = Builder.CreateCall(NewFn, Args);
5694 Value *Ret = llvm::PoisonValue::get(RetTy);
5695 for (unsigned I = 0; I < N; I++) {
5696 Value *SRet = Builder.CreateExtractValue(NewLdCall, I);
5697 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, I * MinElts);
5698 }
5699 NewCall = dyn_cast<CallInst>(Ret);
5700 break;
5701 }
5702
5703 case Intrinsic::coro_end_async:
5704 case Intrinsic::coro_end: {
5705 SmallVector<Value *, 3> Args(CI->args());
5706 if (NewFn->getIntrinsicID() == Intrinsic::coro_end && Args.size() == 2)
5707 Args.push_back(ConstantTokenNone::get(CI->getContext()));
5708 NewCall = Builder.CreateCall(NewFn, Args);
5709
5710 if (!CI->getType()->isVoidTy()) {
5711 if (!CI->use_empty()) {
5712 Function *IsInRamp = Intrinsic::getOrInsertDeclaration(
5713 CI->getModule(), Intrinsic::coro_is_in_ramp);
5714 Value *InRamp = Builder.CreateCall(IsInRamp);
5715 CI->replaceAllUsesWith(Builder.CreateNot(InRamp));
5716 }
5717 CI->eraseFromParent();
5718 return;
5719 }
5720
5721 break;
5722 }
5723
5724 case Intrinsic::vector_extract: {
5725 StringRef Name = F->getName();
5726 Name = Name.substr(5); // Strip llvm
5727 if (!Name.starts_with("aarch64.sve.tuple.get")) {
5728 DefaultCase();
5729 return;
5730 }
5731 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5732 unsigned MinElts = RetTy->getMinNumElements();
5733 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5734 Value *NewIdx = ConstantInt::get(Type::getInt64Ty(C), I * MinElts);
5735 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
5736 break;
5737 }
5738
5739 case Intrinsic::vector_insert: {
5740 StringRef Name = F->getName();
5741 Name = Name.substr(5);
5742 if (!Name.starts_with("aarch64.sve.tuple")) {
5743 DefaultCase();
5744 return;
5745 }
5746 if (Name.starts_with("aarch64.sve.tuple.set")) {
5747 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
5748 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
5749 Value *NewIdx =
5750 ConstantInt::get(Type::getInt64Ty(C), I * Ty->getMinNumElements());
5751 NewCall = Builder.CreateCall(
5752 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
5753 break;
5754 }
5755 if (Name.starts_with("aarch64.sve.tuple.create")) {
5756 unsigned N = StringSwitch<unsigned>(Name)
5757 .StartsWith("aarch64.sve.tuple.create2", 2)
5758 .StartsWith("aarch64.sve.tuple.create3", 3)
5759 .StartsWith("aarch64.sve.tuple.create4", 4)
5760 .Default(0);
5761 assert(N > 1 && "Create is expected to be between 2-4");
5762 auto *RetTy = cast<ScalableVectorType>(F->getReturnType());
5763 Value *Ret = llvm::PoisonValue::get(RetTy);
5764 unsigned MinElts = RetTy->getMinNumElements() / N;
5765 for (unsigned I = 0; I < N; I++) {
5766 Value *V = CI->getArgOperand(I);
5767 Ret = Builder.CreateInsertVector(RetTy, Ret, V, I * MinElts);
5768 }
5769 NewCall = dyn_cast<CallInst>(Ret);
5770 }
5771 break;
5772 }
5773
5774 case Intrinsic::arm_neon_bfdot:
5775 case Intrinsic::arm_neon_bfmmla:
5776 case Intrinsic::arm_neon_bfmlalb:
5777 case Intrinsic::arm_neon_bfmlalt:
5778 case Intrinsic::aarch64_neon_bfdot:
5779 case Intrinsic::aarch64_neon_bfmmla:
5780 case Intrinsic::aarch64_neon_bfmlalb:
5781 case Intrinsic::aarch64_neon_bfmlalt: {
5782 SmallVector<Value *, 3> Args;
5783 assert(CI->arg_size() == 3 &&
5784 "Mismatch between function args and call args");
5785 size_t OperandWidth =
5786 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
5787 assert((OperandWidth == 64 || OperandWidth == 128) &&
5788 "Unexpected operand width");
5789 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
5790 auto Iter = CI->args().begin();
5791 Args.push_back(*Iter++);
5792 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5793 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
5794 NewCall = Builder.CreateCall(NewFn, Args);
5795 break;
5796 }
5797
5798 case Intrinsic::bitreverse:
5799 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5800 break;
5801
5802 case Intrinsic::ctlz:
5803 case Intrinsic::cttz: {
5804 if (CI->arg_size() != 1) {
5805 DefaultCase();
5806 return;
5807 }
5808
5809 NewCall =
5810 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
5811 break;
5812 }
5813
5814 case Intrinsic::objectsize: {
5815 Value *NullIsUnknownSize =
5816 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
5817 Value *Dynamic =
5818 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
5819 NewCall = Builder.CreateCall(
5820 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
5821 break;
5822 }
5823
5824 case Intrinsic::ctpop:
5825 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
5826 break;
5827 case Intrinsic::dbg_value: {
5828 StringRef Name = F->getName();
5829 Name = Name.substr(5); // Strip llvm.
5830 // Upgrade `dbg.addr` to `dbg.value` with `DW_OP_deref`.
5831 if (Name.starts_with("dbg.addr")) {
5832 DIExpression *Expr = cast<DIExpression>(
5833 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
5834 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
5835 NewCall =
5836 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
5837 MetadataAsValue::get(C, Expr)});
5838 break;
5839 }
5840
5841 // Upgrade from the old version that had an extra offset argument.
5842 assert(CI->arg_size() == 4);
5843 // Drop nonzero offsets instead of attempting to upgrade them.
5844 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
5845 if (Offset->isNullValue()) {
5846 NewCall = Builder.CreateCall(
5847 NewFn,
5848 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
5849 break;
5850 }
5851 CI->eraseFromParent();
5852 return;
5853 }
5854
5855 case Intrinsic::ptr_annotation:
5856 // Upgrade from versions that lacked the annotation attribute argument.
5857 if (CI->arg_size() != 4) {
5858 DefaultCase();
5859 return;
5860 }
5861
5862 // Create a new call with an added null annotation attribute argument.
5863 NewCall = Builder.CreateCall(
5864 NewFn,
5865 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5866 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5867 NewCall->takeName(CI);
5868 CI->replaceAllUsesWith(NewCall);
5869 CI->eraseFromParent();
5870 return;
5871
5872 case Intrinsic::var_annotation:
5873 // Upgrade from versions that lacked the annotation attribute argument.
5874 if (CI->arg_size() != 4) {
5875 DefaultCase();
5876 return;
5877 }
5878 // Create a new call with an added null annotation attribute argument.
5879 NewCall = Builder.CreateCall(
5880 NewFn,
5881 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
5882 CI->getArgOperand(3), ConstantPointerNull::get(Builder.getPtrTy())});
5883 NewCall->takeName(CI);
5884 CI->replaceAllUsesWith(NewCall);
5885 CI->eraseFromParent();
5886 return;
5887
5888 case Intrinsic::riscv_aes32dsi:
5889 case Intrinsic::riscv_aes32dsmi:
5890 case Intrinsic::riscv_aes32esi:
5891 case Intrinsic::riscv_aes32esmi:
5892 case Intrinsic::riscv_sm4ks:
5893 case Intrinsic::riscv_sm4ed: {
5894 // The last argument to these intrinsics used to be i8 and changed to i32.
5895 // The type overload for sm4ks and sm4ed was removed.
5896 Value *Arg2 = CI->getArgOperand(2);
5897 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
5898 return;
5899
5900 Value *Arg0 = CI->getArgOperand(0);
5901 Value *Arg1 = CI->getArgOperand(1);
5902 if (CI->getType()->isIntegerTy(64)) {
5903 Arg0 = Builder.CreateTrunc(Arg0, Builder.getInt32Ty());
5904 Arg1 = Builder.CreateTrunc(Arg1, Builder.getInt32Ty());
5905 }
5906
5907 Arg2 = ConstantInt::get(Type::getInt32Ty(C),
5908 cast<ConstantInt>(Arg2)->getZExtValue());
5909
5910 NewCall = Builder.CreateCall(NewFn, {Arg0, Arg1, Arg2});
5911 Value *Res = NewCall;
5912 if (Res->getType() != CI->getType())
5913 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
5914 NewCall->takeName(CI);
5915 CI->replaceAllUsesWith(Res);
5916 CI->eraseFromParent();
5917 return;
5918 }
5919 case Intrinsic::nvvm_mapa_shared_cluster: {
5920 // Create a new call with the correct address space.
5921 NewCall =
5922 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1)});
5923 Value *Res = NewCall;
5924 Res = Builder.CreateAddrSpaceCast(
5925 Res, Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED));
5926 NewCall->takeName(CI);
5927 CI->replaceAllUsesWith(Res);
5928 CI->eraseFromParent();
5929 return;
5930 }
5931 case Intrinsic::nvvm_cp_async_bulk_global_to_shared_cluster:
5932 case Intrinsic::nvvm_cp_async_bulk_shared_cta_to_cluster: {
5933 // Create a new call with the correct address space.
5934 SmallVector<Value *, 4> Args(CI->args());
5935 Args[0] = Builder.CreateAddrSpaceCast(
5936 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5937
5938 NewCall = Builder.CreateCall(NewFn, Args);
5939 NewCall->takeName(CI);
5940 CI->replaceAllUsesWith(NewCall);
5941 CI->eraseFromParent();
5942 return;
5943 }
5944 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_3d:
5945 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_4d:
5946 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_im2col_5d:
5947 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_1d:
5948 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_2d:
5949 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_3d:
5950 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_4d:
5951 case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_tile_5d: {
5952 SmallVector<Value *, 16> Args(CI->args());
5953
5954 // Create AddrSpaceCast to shared_cluster if needed.
5955 // This handles case (1) in shouldUpgradeNVPTXTMAG2SIntrinsics().
5956 unsigned AS = CI->getArgOperand(0)->getType()->getPointerAddressSpace();
5957 if (AS == NVPTXAS::ADDRESS_SPACE_SHARED)
5958 Args[0] = Builder.CreateAddrSpaceCast(
5959 Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
5960
5961 // Attach the flag argument for cta_group, with a
5962 // default value of 0. This handles case (2) in
5963 // shouldUpgradeNVPTXTMAG2SIntrinsics().
5964 size_t NumArgs = CI->arg_size();
5965 Value *FlagArg = CI->getArgOperand(NumArgs - 3);
5966 if (!FlagArg->getType()->isIntegerTy(1))
5967 Args.push_back(ConstantInt::get(Builder.getInt32Ty(), 0));
5968
5969 NewCall = Builder.CreateCall(NewFn, Args);
5970 NewCall->takeName(CI);
5971 CI->replaceAllUsesWith(NewCall);
5972 CI->eraseFromParent();
5973 return;
5974 }
5975 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_1d:
5976 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_2d:
5977 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_3d:
5978 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_4d:
5979 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_tile_5d:
5980 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_3d:
5981 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_4d:
5982 case Intrinsic::nvvm_cp_async_bulk_tensor_reduce_im2col_5d: {
5983 StringRef Name = F->getName();
5984 Name.consume_front("llvm.nvvm.cp.async.bulk.tensor.reduce.");
5985 auto RedOp = getNVPTXTMAReductionOp(Name.split('.').first);
5986
5987 SmallVector<Value *, 16> Args(CI->args());
5988 Args.insert(Args.end() - 1, Builder.getInt32(*RedOp));
5989 NewCall = Builder.CreateCall(NewFn, Args);
5990 break;
5991 }
5992 case Intrinsic::nvvm_tcgen05_mma_shared:
5993 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
5994 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
5995 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale:
5996 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4_block_scale_block32:
5997 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block16:
5998 case Intrinsic::nvvm_tcgen05_mma_shared_mxf4nvf4_block_scale_block32:
5999 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale:
6000 case Intrinsic::nvvm_tcgen05_mma_shared_mxf8f6f4_block_scale_block32:
6001 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d:
6002 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
6003 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
6004 case Intrinsic::nvvm_tcgen05_mma_sp_shared:
6005 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
6006 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
6007 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale:
6008 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4_block_scale_block32:
6009 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block16:
6010 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf4nvf4_block_scale_block32:
6011 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale:
6012 case Intrinsic::nvvm_tcgen05_mma_sp_shared_mxf8f6f4_block_scale_block32:
6013 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d:
6014 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
6015 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
6016 case Intrinsic::nvvm_tcgen05_mma_sp_tensor:
6017 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_ashift:
6018 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
6019 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
6020 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
6021 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
6022 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale:
6023 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4_block_scale_block32:
6024 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block16:
6025 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf4nvf4_block_scale_block32:
6026 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale:
6027 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_mxf8f6f4_block_scale_block32:
6028 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d:
6029 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_ashift:
6030 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
6031 case Intrinsic::
6032 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
6033 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
6034 case Intrinsic::
6035 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
6036 case Intrinsic::nvvm_tcgen05_mma_tensor:
6037 case Intrinsic::nvvm_tcgen05_mma_tensor_ashift:
6038 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
6039 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
6040 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
6041 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
6042 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale:
6043 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4_block_scale_block32:
6044 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block16:
6045 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf4nvf4_block_scale_block32:
6046 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale:
6047 case Intrinsic::nvvm_tcgen05_mma_tensor_mxf8f6f4_block_scale_block32:
6048 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d:
6049 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_ashift:
6050 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
6051 case Intrinsic::
6052 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
6053 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
6054 case Intrinsic::
6055 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift: {
6056 SmallVector<Value *, 12> Args(CI->args());
6057 Args.push_back(Builder.getInt32(0)); // collector_usage_b = discard(0)
6058 NewCall = Builder.CreateCall(NewFn, Args);
6059 break;
6060 }
6061 case Intrinsic::nvvm_tcgen05_alloc_cg1:
6062 case Intrinsic::nvvm_tcgen05_alloc_cg2:
6063 case Intrinsic::nvvm_tcgen05_dealloc_cg1:
6064 case Intrinsic::nvvm_tcgen05_dealloc_cg2:
6065 NewCall =
6066 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
6067 Builder.getFalse()});
6068 break;
6069 case Intrinsic::riscv_sha256sig0:
6070 case Intrinsic::riscv_sha256sig1:
6071 case Intrinsic::riscv_sha256sum0:
6072 case Intrinsic::riscv_sha256sum1:
6073 case Intrinsic::riscv_sm3p0:
6074 case Intrinsic::riscv_sm3p1: {
6075 // The last argument to these intrinsics used to be i8 and changed to i32.
6076 // The type overload for sm4ks and sm4ed was removed.
6077 if (!CI->getType()->isIntegerTy(64))
6078 return;
6079
6080 Value *Arg =
6081 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
6082
6083 NewCall = Builder.CreateCall(NewFn, Arg);
6084 Value *Res =
6085 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
6086 NewCall->takeName(CI);
6087 CI->replaceAllUsesWith(Res);
6088 CI->eraseFromParent();
6089 return;
6090 }
6091
6092 case Intrinsic::x86_xop_vfrcz_ss:
6093 case Intrinsic::x86_xop_vfrcz_sd:
6094 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
6095 break;
6096
6097 case Intrinsic::x86_xop_vpermil2pd:
6098 case Intrinsic::x86_xop_vpermil2ps:
6099 case Intrinsic::x86_xop_vpermil2pd_256:
6100 case Intrinsic::x86_xop_vpermil2ps_256: {
6101 SmallVector<Value *, 4> Args(CI->args());
6102 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
6103 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
6104 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
6105 NewCall = Builder.CreateCall(NewFn, Args);
6106 break;
6107 }
6108
6109 case Intrinsic::x86_sse41_ptestc:
6110 case Intrinsic::x86_sse41_ptestz:
6111 case Intrinsic::x86_sse41_ptestnzc: {
6112 // The arguments for these intrinsics used to be v4f32, and changed
6113 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
6114 // So, the only thing required is a bitcast for both arguments.
6115 // First, check the arguments have the old type.
6116 Value *Arg0 = CI->getArgOperand(0);
6117 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
6118 return;
6119
6120 // Old intrinsic, add bitcasts
6121 Value *Arg1 = CI->getArgOperand(1);
6122
6123 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
6124
6125 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
6126 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
6127
6128 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
6129 break;
6130 }
6131
6132 case Intrinsic::x86_rdtscp: {
6133 // This used to take 1 arguments. If we have no arguments, it is already
6134 // upgraded.
6135 if (CI->getNumOperands() == 0)
6136 return;
6137
6138 NewCall = Builder.CreateCall(NewFn);
6139 // Extract the second result and store it.
6140 Value *Data = Builder.CreateExtractValue(NewCall, 1);
6141 Builder.CreateAlignedStore(Data, CI->getArgOperand(0), Align(1));
6142 // Replace the original call result with the first result of the new call.
6143 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
6144
6145 NewCall->takeName(CI);
6146 CI->replaceAllUsesWith(TSC);
6147 CI->eraseFromParent();
6148 return;
6149 }
6150
6151 case Intrinsic::x86_sse41_insertps:
6152 case Intrinsic::x86_sse41_dppd:
6153 case Intrinsic::x86_sse41_dpps:
6154 case Intrinsic::x86_sse41_mpsadbw:
6155 case Intrinsic::x86_avx_dp_ps_256:
6156 case Intrinsic::x86_avx2_mpsadbw: {
6157 // Need to truncate the last argument from i32 to i8 -- this argument models
6158 // an inherently 8-bit immediate operand to these x86 instructions.
6159 SmallVector<Value *, 4> Args(CI->args());
6160
6161 // Replace the last argument with a trunc.
6162 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
6163 NewCall = Builder.CreateCall(NewFn, Args);
6164 break;
6165 }
6166
6167 case Intrinsic::x86_avx512_mask_cmp_pd_128:
6168 case Intrinsic::x86_avx512_mask_cmp_pd_256:
6169 case Intrinsic::x86_avx512_mask_cmp_pd_512:
6170 case Intrinsic::x86_avx512_mask_cmp_ps_128:
6171 case Intrinsic::x86_avx512_mask_cmp_ps_256:
6172 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
6173 SmallVector<Value *, 4> Args(CI->args());
6174 unsigned NumElts =
6175 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
6176 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
6177
6178 NewCall = Builder.CreateCall(NewFn, Args);
6179 Value *Res = applyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
6180
6181 NewCall->takeName(CI);
6182 CI->replaceAllUsesWith(Res);
6183 CI->eraseFromParent();
6184 return;
6185 }
6186
6187 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_128:
6188 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_256:
6189 case Intrinsic::x86_avx512bf16_cvtne2ps2bf16_512:
6190 case Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128:
6191 case Intrinsic::x86_avx512bf16_cvtneps2bf16_256:
6192 case Intrinsic::x86_avx512bf16_cvtneps2bf16_512: {
6193 SmallVector<Value *, 4> Args(CI->args());
6194 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
6195 if (NewFn->getIntrinsicID() ==
6196 Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128)
6197 Args[1] = Builder.CreateBitCast(
6198 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6199
6200 NewCall = Builder.CreateCall(NewFn, Args);
6201 Value *Res = Builder.CreateBitCast(
6202 NewCall, FixedVectorType::get(Builder.getInt16Ty(), NumElts));
6203
6204 NewCall->takeName(CI);
6205 CI->replaceAllUsesWith(Res);
6206 CI->eraseFromParent();
6207 return;
6208 }
6209 case Intrinsic::x86_avx512bf16_dpbf16ps_128:
6210 case Intrinsic::x86_avx512bf16_dpbf16ps_256:
6211 case Intrinsic::x86_avx512bf16_dpbf16ps_512:{
6212 SmallVector<Value *, 4> Args(CI->args());
6213 unsigned NumElts =
6214 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
6215 Args[1] = Builder.CreateBitCast(
6216 Args[1], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6217 Args[2] = Builder.CreateBitCast(
6218 Args[2], FixedVectorType::get(Builder.getBFloatTy(), NumElts));
6219
6220 NewCall = Builder.CreateCall(NewFn, Args);
6221 break;
6222 }
6223
6224 case Intrinsic::thread_pointer: {
6225 NewCall = Builder.CreateCall(NewFn, {});
6226 break;
6227 }
6228
6229 case Intrinsic::memcpy:
6230 case Intrinsic::memmove:
6231 case Intrinsic::memset: {
6232 // We have to make sure that the call signature is what we're expecting.
6233 // We only want to change the old signatures by removing the alignment arg:
6234 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
6235 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
6236 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
6237 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
6238 // Note: i8*'s in the above can be any pointer type
6239 if (CI->arg_size() != 5) {
6240 DefaultCase();
6241 return;
6242 }
6243 // Remove alignment argument (3), and add alignment attributes to the
6244 // dest/src pointers.
6245 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
6246 CI->getArgOperand(2), CI->getArgOperand(4)};
6247 NewCall = Builder.CreateCall(NewFn, Args);
6248 AttributeList OldAttrs = CI->getAttributes();
6249 AttributeList NewAttrs = AttributeList::get(
6250 C, OldAttrs.getFnAttrs(), OldAttrs.getRetAttrs(),
6251 {OldAttrs.getParamAttrs(0), OldAttrs.getParamAttrs(1),
6252 OldAttrs.getParamAttrs(2), OldAttrs.getParamAttrs(4)});
6253 NewCall->setAttributes(NewAttrs);
6254 auto *MemCI = cast<MemIntrinsic>(NewCall);
6255 // All mem intrinsics support dest alignment.
6256 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
6257 MemCI->setDestAlignment(Align->getMaybeAlignValue());
6258 // Memcpy/Memmove also support source alignment.
6259 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
6260 MTI->setSourceAlignment(Align->getMaybeAlignValue());
6261 break;
6262 }
6263
6264 case Intrinsic::masked_load:
6265 case Intrinsic::masked_gather:
6266 case Intrinsic::masked_store:
6267 case Intrinsic::masked_scatter: {
6268 if (CI->arg_size() != 4) {
6269 DefaultCase();
6270 return;
6271 }
6272
6273 auto GetMaybeAlign = [](Value *Op) {
6274 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
6275 uint64_t Val = CI->getZExtValue();
6276 if (Val == 0)
6277 return MaybeAlign();
6278 if (isPowerOf2_64(Val))
6279 return MaybeAlign(Val);
6280 }
6281 reportFatalUsageError("Invalid alignment argument");
6282 };
6283 auto GetAlign = [&](Value *Op) {
6284 MaybeAlign Align = GetMaybeAlign(Op);
6285 if (Align)
6286 return *Align;
6287 reportFatalUsageError("Invalid zero alignment argument");
6288 };
6289
6290 const DataLayout &DL = CI->getDataLayout();
6291 switch (NewFn->getIntrinsicID()) {
6292 case Intrinsic::masked_load:
6293 NewCall = Builder.CreateMaskedLoad(
6294 CI->getType(), CI->getArgOperand(0), GetAlign(CI->getArgOperand(1)),
6295 CI->getArgOperand(2), CI->getArgOperand(3));
6296 break;
6297 case Intrinsic::masked_gather:
6298 NewCall = Builder.CreateMaskedGather(
6299 CI->getType(), CI->getArgOperand(0),
6300 DL.getValueOrABITypeAlignment(GetMaybeAlign(CI->getArgOperand(1)),
6301 CI->getType()->getScalarType()),
6302 CI->getArgOperand(2), CI->getArgOperand(3));
6303 break;
6304 case Intrinsic::masked_store:
6305 NewCall = Builder.CreateMaskedStore(
6306 CI->getArgOperand(0), CI->getArgOperand(1),
6307 GetAlign(CI->getArgOperand(2)), CI->getArgOperand(3));
6308 break;
6309 case Intrinsic::masked_scatter:
6310 NewCall = Builder.CreateMaskedScatter(
6311 CI->getArgOperand(0), CI->getArgOperand(1),
6312 DL.getValueOrABITypeAlignment(
6313 GetMaybeAlign(CI->getArgOperand(2)),
6314 CI->getArgOperand(0)->getType()->getScalarType()),
6315 CI->getArgOperand(3));
6316 break;
6317 default:
6318 llvm_unreachable("Unexpected intrinsic ID");
6319 }
6320 // Previous metadata is still valid.
6321 NewCall->copyMetadata(*CI);
6322 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6323 break;
6324 }
6325
6326 case Intrinsic::lifetime_start:
6327 case Intrinsic::lifetime_end: {
6328 if (CI->arg_size() != 2) {
6329 DefaultCase();
6330 return;
6331 }
6332
6333 Value *Ptr = CI->getArgOperand(1);
6334 // Try to strip pointer casts, such that the lifetime works on an alloca.
6335 Ptr = Ptr->stripPointerCasts();
6336 if (isa<AllocaInst>(Ptr)) {
6337 // Don't use NewFn, as we might have looked through an addrspacecast.
6338 if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
6339 NewCall = Builder.CreateLifetimeStart(Ptr);
6340 else
6341 NewCall = Builder.CreateLifetimeEnd(Ptr);
6342 break;
6343 }
6344
6345 // Otherwise remove the lifetime marker.
6346 CI->eraseFromParent();
6347 return;
6348 }
6349
6350 case Intrinsic::x86_avx512_vpdpbusd_128:
6351 case Intrinsic::x86_avx512_vpdpbusd_256:
6352 case Intrinsic::x86_avx512_vpdpbusd_512:
6353 case Intrinsic::x86_avx512_vpdpbusds_128:
6354 case Intrinsic::x86_avx512_vpdpbusds_256:
6355 case Intrinsic::x86_avx512_vpdpbusds_512:
6356 case Intrinsic::x86_avx2_vpdpbssd_128:
6357 case Intrinsic::x86_avx2_vpdpbssd_256:
6358 case Intrinsic::x86_avx10_vpdpbssd_512:
6359 case Intrinsic::x86_avx2_vpdpbssds_128:
6360 case Intrinsic::x86_avx2_vpdpbssds_256:
6361 case Intrinsic::x86_avx10_vpdpbssds_512:
6362 case Intrinsic::x86_avx2_vpdpbsud_128:
6363 case Intrinsic::x86_avx2_vpdpbsud_256:
6364 case Intrinsic::x86_avx10_vpdpbsud_512:
6365 case Intrinsic::x86_avx2_vpdpbsuds_128:
6366 case Intrinsic::x86_avx2_vpdpbsuds_256:
6367 case Intrinsic::x86_avx10_vpdpbsuds_512:
6368 case Intrinsic::x86_avx2_vpdpbuud_128:
6369 case Intrinsic::x86_avx2_vpdpbuud_256:
6370 case Intrinsic::x86_avx10_vpdpbuud_512:
6371 case Intrinsic::x86_avx2_vpdpbuuds_128:
6372 case Intrinsic::x86_avx2_vpdpbuuds_256:
6373 case Intrinsic::x86_avx10_vpdpbuuds_512: {
6374 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 8;
6375 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6376 CI->getArgOperand(2)};
6377 Type *NewArgType = VectorType::get(Builder.getInt8Ty(), NumElts, false);
6378 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6379 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6380
6381 NewCall = Builder.CreateCall(NewFn, Args);
6382 break;
6383 }
6384 case Intrinsic::x86_avx512_vpdpwssd_128:
6385 case Intrinsic::x86_avx512_vpdpwssd_256:
6386 case Intrinsic::x86_avx512_vpdpwssd_512:
6387 case Intrinsic::x86_avx512_vpdpwssds_128:
6388 case Intrinsic::x86_avx512_vpdpwssds_256:
6389 case Intrinsic::x86_avx512_vpdpwssds_512:
6390 case Intrinsic::x86_avx2_vpdpwsud_128:
6391 case Intrinsic::x86_avx2_vpdpwsud_256:
6392 case Intrinsic::x86_avx10_vpdpwsud_512:
6393 case Intrinsic::x86_avx2_vpdpwsuds_128:
6394 case Intrinsic::x86_avx2_vpdpwsuds_256:
6395 case Intrinsic::x86_avx10_vpdpwsuds_512:
6396 case Intrinsic::x86_avx2_vpdpwusd_128:
6397 case Intrinsic::x86_avx2_vpdpwusd_256:
6398 case Intrinsic::x86_avx10_vpdpwusd_512:
6399 case Intrinsic::x86_avx2_vpdpwusds_128:
6400 case Intrinsic::x86_avx2_vpdpwusds_256:
6401 case Intrinsic::x86_avx10_vpdpwusds_512:
6402 case Intrinsic::x86_avx2_vpdpwuud_128:
6403 case Intrinsic::x86_avx2_vpdpwuud_256:
6404 case Intrinsic::x86_avx10_vpdpwuud_512:
6405 case Intrinsic::x86_avx2_vpdpwuuds_128:
6406 case Intrinsic::x86_avx2_vpdpwuuds_256:
6407 case Intrinsic::x86_avx10_vpdpwuuds_512:
6408 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 16;
6409 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
6410 CI->getArgOperand(2)};
6411 Type *NewArgType = VectorType::get(Builder.getInt16Ty(), NumElts, false);
6412 Args[1] = Builder.CreateBitCast(Args[1], NewArgType);
6413 Args[2] = Builder.CreateBitCast(Args[2], NewArgType);
6414
6415 NewCall = Builder.CreateCall(NewFn, Args);
6416 break;
6417 }
6418 assert(NewCall && "Should have either set this variable or returned through "
6419 "the default case");
6420 NewCall->takeName(CI);
6421 CI->replaceAllUsesWith(NewCall);
6422 CI->eraseFromParent();
6423}
6424
6425void llvm::UpgradeCallsToIntrinsic(Function *F) {
6426 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
6427
6428 // Check if this function should be upgraded and get the replacement function
6429 // if there is one.
6430 Function *NewFn;
6431 if (UpgradeIntrinsicFunction(F, NewFn)) {
6432 // Replace all users of the old function with the new function or new
6433 // instructions. This is not a range loop because the call is deleted.
6434 for (User *U : make_early_inc_range(F->users()))
6435 if (CallBase *CB = dyn_cast<CallBase>(U))
6436 UpgradeIntrinsicCall(CB, NewFn);
6437
6438 // Remove old function, no longer used, from the module.
6439 if (F != NewFn)
6440 F->eraseFromParent();
6441 }
6442}
6443
6444MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
6445 const unsigned NumOperands = MD.getNumOperands();
6446 if (NumOperands == 0)
6447 return &MD; // Invalid, punt to a verifier error.
6448
6449 // Check if the tag uses struct-path aware TBAA format.
6450 if (isa<MDNode>(MD.getOperand(0)) && NumOperands >= 3)
6451 return &MD;
6452
6453 auto &Context = MD.getContext();
6454 if (NumOperands == 3) {
6455 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
6456 MDNode *ScalarType = MDNode::get(Context, Elts);
6457 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
6458 Metadata *Elts2[] = {ScalarType, ScalarType,
6459 ConstantAsMetadata::get(
6460 Constant::getNullValue(Type::getInt64Ty(Context))),
6461 MD.getOperand(2)};
6462 return MDNode::get(Context, Elts2);
6463 }
6464 // Create a MDNode <MD, MD, offset 0>
6465 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
6466 Type::getInt64Ty(Context)))};
6467 return MDNode::get(Context, Elts);
6468}
6469
6470Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
6471 Instruction *&Temp) {
6472 if (Opc != Instruction::BitCast)
6473 return nullptr;
6474
6475 Temp = nullptr;
6476 Type *SrcTy = V->getType();
6477 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6478 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6479 LLVMContext &Context = V->getContext();
6480
6481 // We have no information about target data layout, so we assume that
6482 // the maximum pointer size is 64bit.
6483 Type *MidTy = Type::getInt64Ty(Context);
6484 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
6485
6486 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
6487 }
6488
6489 return nullptr;
6490}
6491
6492Constant *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
6493 if (Opc != Instruction::BitCast)
6494 return nullptr;
6495
6496 Type *SrcTy = C->getType();
6497 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
6498 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
6499 LLVMContext &Context = C->getContext();
6500
6501 // We have no information about target data layout, so we assume that
6502 // the maximum pointer size is 64bit.
6503 Type *MidTy = Type::getInt64Ty(Context);
6504
6505 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
6506 DestTy);
6507 }
6508
6509 return nullptr;
6510}
6511
6512static std::optional<StringRef> getModuleFlagNameSafely(const MDNode &Flag) {
6513 if (Flag.getNumOperands() < 3)
6514 return std::nullopt;
6515 if (MDString *Name = dyn_cast_or_null<MDString>(Flag.getOperand(1)))
6516 return Name->getString();
6517 return std::nullopt;
6518}
6519
6520/// Check the debug info version number, if it is out-dated, drop the debug
6521/// info. Return true if module is modified.
6522bool llvm::UpgradeDebugInfo(Module &M) {
6523 if (DisableAutoUpgradeDebugInfo)
6524 return false;
6525
6526 llvm::TimeTraceScope timeScope("Upgrade debug info");
6527 // We need to get metadata before the module is verified (i.e., getModuleFlag
6528 // makes assumptions that we haven't verified yet). Carefully extract the flag
6529 // from the metadata.
6530 unsigned Version = 0;
6531 if (NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6532 auto OpIt = find_if(ModFlags->operands(), [](const MDNode *Flag) {
6533 if (auto Name = getModuleFlagNameSafely(*Flag))
6534 return *Name == "Debug Info Version";
6535 return false;
6536 });
6537 if (OpIt != ModFlags->op_end()) {
6538 const MDOperand &ValOp = (*OpIt)->getOperand(2);
6539 if (auto *CI = mdconst::dyn_extract_or_null<ConstantInt>(ValOp))
6540 Version = CI->getZExtValue();
6541 }
6542 }
6543
6544 if (Version == DEBUG_METADATA_VERSION) {
6545 bool BrokenDebugInfo = false;
6546 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
6547 report_fatal_error("Broken module found, compilation aborted!");
6548 if (!BrokenDebugInfo)
6549 // Everything is ok.
6550 return false;
6551 else {
6552 // Diagnose malformed debug info.
6553 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
6554 M.getContext().diagnose(Diag);
6555 }
6556 }
6557 bool Modified = StripDebugInfo(M);
6558 if (Modified && Version != DEBUG_METADATA_VERSION) {
6559 // Diagnose a version mismatch.
6560 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
6561 M.getContext().diagnose(DiagVersion);
6562 }
6563 return Modified;
6564}
6565
6566static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC,
6567 GlobalValue *GV, const Metadata *V) {
6568 Function *F = cast<Function>(GV);
6569
6570 constexpr StringLiteral DefaultValue = "1";
6571 StringRef Vect3[3] = {DefaultValue, DefaultValue, DefaultValue};
6572 unsigned Length = 0;
6573
6574 if (F->hasFnAttribute(Attr)) {
6575 // We expect the existing attribute to have the form "x[,y[,z]]". Here we
6576 // parse these elements placing them into Vect3
6577 StringRef S = F->getFnAttribute(Attr).getValueAsString();
6578 for (; Length < 3 && !S.empty(); Length++) {
6579 auto [Part, Rest] = S.split(',');
6580 Vect3[Length] = Part.trim();
6581 S = Rest;
6582 }
6583 }
6584
6585 const unsigned Dim = DimC - 'x';
6586 assert(Dim < 3 && "Unexpected dim char");
6587
6588 const uint64_t VInt = mdconst::extract<ConstantInt>(V)->getZExtValue();
6589
6590 // local variable required for StringRef in Vect3 to point to.
6591 const std::string VStr = llvm::utostr(VInt);
6592 Vect3[Dim] = VStr;
6593 Length = std::max(Length, Dim + 1);
6594
6595 const std::string NewAttr = llvm::join(ArrayRef(Vect3, Length), ",");
6596 F->addFnAttr(Attr, NewAttr);
6597}
6598
6599static inline bool isXYZ(StringRef S) {
6600 return S == "x" || S == "y" || S == "z";
6601}
6602
6603bool static upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K,
6604 const Metadata *V) {
6605 if (K == "kernel") {
6606 if (!mdconst::extract<ConstantInt>(V)->isZero())
6607 cast<Function>(GV)->setCallingConv(CallingConv::PTX_Kernel);
6608 return true;
6609 }
6610 if (K == "align") {
6611 // V is a bitfeild specifying two 16-bit values. The alignment value is
6612 // specfied in low 16-bits, The index is specified in the high bits. For the
6613 // index, 0 indicates the return value while higher values correspond to
6614 // each parameter (idx = param + 1).
6615 const uint64_t AlignIdxValuePair =
6616 mdconst::extract<ConstantInt>(V)->getZExtValue();
6617 const unsigned Idx = (AlignIdxValuePair >> 16);
6618 const Align StackAlign = Align(AlignIdxValuePair & 0xFFFF);
6619 cast<Function>(GV)->addAttributeAtIndex(
6620 Idx, Attribute::getWithStackAlignment(GV->getContext(), StackAlign));
6621 return true;
6622 }
6623 if (K == "maxclusterrank" || K == "cluster_max_blocks") {
6624 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6625 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxClusterRank, llvm::utostr(CV));
6626 return true;
6627 }
6628 if (K == "minctasm") {
6629 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6630 cast<Function>(GV)->addFnAttr(NVVMAttr::MinCTASm, llvm::utostr(CV));
6631 return true;
6632 }
6633 if (K == "maxnreg") {
6634 const auto CV = mdconst::extract<ConstantInt>(V)->getZExtValue();
6635 cast<Function>(GV)->addFnAttr(NVVMAttr::MaxNReg, llvm::utostr(CV));
6636 return true;
6637 }
6638 if (K.consume_front("maxntid") && isXYZ(K)) {
6639 upgradeNVVMFnVectorAttr(NVVMAttr::MaxNTID, K[0], GV, V);
6640 return true;
6641 }
6642 if (K.consume_front("reqntid") && isXYZ(K)) {
6643 upgradeNVVMFnVectorAttr(NVVMAttr::ReqNTID, K[0], GV, V);
6644 return true;
6645 }
6646 if (K.consume_front("cluster_dim_") && isXYZ(K)) {
6647 upgradeNVVMFnVectorAttr(NVVMAttr::ClusterDim, K[0], GV, V);
6648 return true;
6649 }
6650 if (K == "grid_constant") {
6651 const auto Attr = Attribute::get(GV->getContext(), NVVMAttr::GridConstant);
6652 for (const auto &Op : cast<MDNode>(V)->operands()) {
6653 // For some reason, the index is 1-based in the metadata. Good thing we're
6654 // able to auto-upgrade it!
6655 const auto Index = mdconst::extract<ConstantInt>(Op)->getZExtValue() - 1;
6656 cast<Function>(GV)->addParamAttr(Index, Attr);
6657 }
6658 return true;
6659 }
6660
6661 return false;
6662}
6663
6664void llvm::UpgradeNVVMAnnotations(Module &M) {
6665 NamedMDNode *NamedMD = M.getNamedMetadata("nvvm.annotations");
6666 if (!NamedMD)
6667 return;
6668
6669 SmallVector<MDNode *, 8> NewNodes;
6670 SmallPtrSet<const MDNode *, 8> SeenNodes;
6671 for (MDNode *MD : NamedMD->operands()) {
6672 if (!SeenNodes.insert(MD).second)
6673 continue;
6674
6675 auto *GV = mdconst::dyn_extract_or_null<GlobalValue>(MD->getOperand(0));
6676 if (!GV)
6677 continue;
6678
6679 assert((MD->getNumOperands() % 2) == 1 && "Invalid number of operands");
6680
6681 SmallVector<Metadata *, 8> NewOperands{MD->getOperand(0)};
6682 // Each nvvm.annotations metadata entry will be of the following form:
6683 // !{ ptr @gv, !"key1", value1, !"key2", value2, ... }
6684 // start index = 1, to skip the global variable key
6685 // increment = 2, to skip the value for each property-value pairs
6686 for (unsigned j = 1, je = MD->getNumOperands(); j < je; j += 2) {
6687 MDString *K = cast<MDString>(MD->getOperand(j));
6688 const MDOperand &V = MD->getOperand(j + 1);
6689 bool Upgraded = upgradeSingleNVVMAnnotation(GV, K->getString(), V);
6690 if (!Upgraded)
6691 NewOperands.append({K, V});
6692 }
6693
6694 if (NewOperands.size() > 1)
6695 NewNodes.push_back(MDNode::get(M.getContext(), NewOperands));
6696 }
6697
6698 NamedMD->clearOperands();
6699 for (MDNode *N : NewNodes)
6700 NamedMD->addOperand(N);
6701}
6702
6703/// This checks for objc retain release marker which should be upgraded. It
6704/// returns true if module is modified.
6705static bool upgradeRetainReleaseMarker(Module &M) {
6706 bool Changed = false;
6707 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
6708 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
6709 if (ModRetainReleaseMarker) {
6710 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
6711 if (Op) {
6712 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
6713 if (ID) {
6714 SmallVector<StringRef, 4> ValueComp;
6715 ID->getString().split(ValueComp, "#");
6716 if (ValueComp.size() == 2) {
6717 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
6718 ID = MDString::get(M.getContext(), NewValue);
6719 }
6720 M.addModuleFlag(Module::Error, MarkerKey, ID);
6721 M.eraseNamedMetadata(ModRetainReleaseMarker);
6722 Changed = true;
6723 }
6724 }
6725 }
6726 return Changed;
6727}
6728
6729void llvm::UpgradeARCRuntime(Module &M) {
6730 // This lambda converts normal function calls to ARC runtime functions to
6731 // intrinsic calls.
6732 auto UpgradeToIntrinsic = [&](const char *OldFunc,
6733 llvm::Intrinsic::ID IntrinsicFunc) {
6734 Function *Fn = M.getFunction(OldFunc);
6735
6736 if (!Fn)
6737 return;
6738
6739 Function *NewFn =
6740 llvm::Intrinsic::getOrInsertDeclaration(&M, IntrinsicFunc);
6741
6742 for (User *U : make_early_inc_range(Fn->users())) {
6743 CallInst *CI = dyn_cast<CallInst>(U);
6744 if (!CI || CI->getCalledFunction() != Fn)
6745 continue;
6746
6747 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
6748 FunctionType *NewFuncTy = NewFn->getFunctionType();
6749 SmallVector<Value *, 2> Args;
6750
6751 // Don't upgrade the intrinsic if it's not valid to bitcast the return
6752 // value to the return type of the old function.
6753 if (NewFuncTy->getReturnType() != CI->getType() &&
6754 !CastInst::castIsValid(Instruction::BitCast, CI,
6755 NewFuncTy->getReturnType()))
6756 continue;
6757
6758 bool InvalidCast = false;
6759
6760 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
6761 Value *Arg = CI->getArgOperand(I);
6762
6763 // Bitcast argument to the parameter type of the new function if it's
6764 // not a variadic argument.
6765 if (I < NewFuncTy->getNumParams()) {
6766 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
6767 // to the parameter type of the new function.
6768 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
6769 NewFuncTy->getParamType(I))) {
6770 InvalidCast = true;
6771 break;
6772 }
6773 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
6774 }
6775 Args.push_back(Arg);
6776 }
6777
6778 if (InvalidCast)
6779 continue;
6780
6781 // Create a call instruction that calls the new function.
6782 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
6783 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
6784 NewCall->takeName(CI);
6785
6786 // Bitcast the return value back to the type of the old call.
6787 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
6788
6789 if (!CI->use_empty())
6790 CI->replaceAllUsesWith(NewRetVal);
6791 CI->eraseFromParent();
6792 }
6793
6794 if (Fn->use_empty())
6795 Fn->eraseFromParent();
6796 };
6797
6798 // Unconditionally convert a call to "clang.arc.use" to a call to
6799 // "llvm.objc.clang.arc.use".
6800 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
6801
6802 // Upgrade the retain release marker. If there is no need to upgrade
6803 // the marker, that means either the module is already new enough to contain
6804 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
6805 if (!upgradeRetainReleaseMarker(M))
6806 return;
6807
6808 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
6809 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
6810 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
6811 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
6812 {"objc_autoreleaseReturnValue",
6813 llvm::Intrinsic::objc_autoreleaseReturnValue},
6814 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
6815 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
6816 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
6817 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
6818 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
6819 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
6820 {"objc_release", llvm::Intrinsic::objc_release},
6821 {"objc_retain", llvm::Intrinsic::objc_retain},
6822 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
6823 {"objc_retainAutoreleaseReturnValue",
6824 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
6825 {"objc_retainAutoreleasedReturnValue",
6826 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
6827 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
6828 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
6829 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
6830 {"objc_unsafeClaimAutoreleasedReturnValue",
6831 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
6832 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
6833 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
6834 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
6835 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
6836 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
6837 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
6838 {"objc_arc_annotation_topdown_bbstart",
6839 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
6840 {"objc_arc_annotation_topdown_bbend",
6841 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
6842 {"objc_arc_annotation_bottomup_bbstart",
6843 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
6844 {"objc_arc_annotation_bottomup_bbend",
6845 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
6846
6847 for (auto &I : RuntimeFuncs)
6848 UpgradeToIntrinsic(I.first, I.second);
6849}
6850
6851// Upgrade the way signing of pointers to init/fini functions is described.
6852//
6853// Originally, the `@llvm.global_(ctors|dtors)` arrays contained `ptrauth`
6854// constants, if signing was requested. After the upgrade, these arrays contain
6855// plain function pointers and the desired signing schema is described via a
6856// pair of module flags.
6857//
6858// Note that the upgrade is only performed if all elements of *both* arrays
6859// agree on a common signing schema.
6860static bool upgradePtrauthInitFiniArrays(Module &M) {
6861 // As we cannot always decide whether the particular module should have
6862 // ptrauth-init-fini flags, we have to treat absent flags as having zero
6863 // values for compatibility reasons. Thus, upgradePtrauthInitFiniArrays
6864 // returns as soon as it spots any non-signed init/fini pointer: either we
6865 // should request non-signed pointers (safe to omit both flags) or there is
6866 // no common schema (and thus we do not modify anything).
6867 //
6868 // UseAddressDisc's value either represents "not decided yet" state (nullopt)
6869 // or whether we should request address diversity in addition to the basic
6870 // constant diversity. There is no value representing "decided not to sign"
6871 // for the reasons explained above.
6872 std::optional<bool> UseAddressDisc;
6873
6874 // Do not attempt upgrading if the new module flags already exist.
6875 if (const NamedMDNode *ModFlags = M.getModuleFlagsMetadata()) {
6876 for (const MDNode *Flag : ModFlags->operands()) {
6877 std::optional<StringRef> Name = getModuleFlagNameSafely(*Flag);
6878 if (Name && (*Name == "ptrauth-init-fini" ||
6879 *Name == "ptrauth-init-fini-address-discrimination"))
6880 return false;
6881 }
6882 }
6883
6884 auto UpgradeSinglePointer = [&UseAddressDisc](Constant *CV) -> Constant * {
6885 constexpr unsigned ExpectedConstDisc = 0xD9D4;
6886 constexpr unsigned ExpectedAddressMarker = 1;
6887
6888 auto *CPA = dyn_cast<ConstantPtrAuth>(CV);
6889 if (!CPA || !CPA->getDiscriminator()->equalsInt(ExpectedConstDisc))
6890 return nullptr; // Nothing to upgrade or unknown pattern found.
6891
6892 bool HasAddressDisc;
6893 if (!CPA->hasAddressDiscriminator())
6894 HasAddressDisc = false;
6895 else if (CPA->hasSpecialAddressDiscriminator(ExpectedAddressMarker))
6896 HasAddressDisc = true;
6897 else
6898 return nullptr; // Unknown pattern.
6899
6900 if (UseAddressDisc && *UseAddressDisc != HasAddressDisc)
6901 return nullptr; // Disagreement with the decided mode.
6902
6903 UseAddressDisc = HasAddressDisc;
6904 return CPA->getPointer();
6905 };
6906
6907 // Do not apply any changes until we know the upgrade is non-ambiguous.
6908 using PendingUpgrade = std::pair<GlobalVariable *, Constant *>;
6909 SmallVector<PendingUpgrade, 2> GlobalArraysToUpgrade;
6910
6911 for (const char *Name : {"llvm.global_ctors", "llvm.global_dtors"}) {
6912 auto *GV = dyn_cast_if_present<GlobalVariable>(M.getNamedValue(Name));
6913 if (!GV || !GV->hasInitializer())
6914 continue; // Skip, but it is okay to upgrade the other variable.
6915
6916 auto *OldStructorsArray = dyn_cast<ConstantArray>(GV->getInitializer());
6917 if (!OldStructorsArray || OldStructorsArray->getNumOperands() == 0)
6918 return false;
6919
6920 std::vector<Constant *> NewStructors;
6921 NewStructors.reserve(OldStructorsArray->getNumOperands());
6922
6923 for (Use &U : OldStructorsArray->operands()) {
6924 ConstantStruct *Structor = dyn_cast<ConstantStruct>(U.get());
6925 if (!Structor || Structor->getNumOperands() != 3)
6926 return false;
6927
6928 Constant *Prio = Structor->getOperand(0);
6929 Constant *Func = Structor->getOperand(1);
6930 Constant *Arg = Structor->getOperand(2);
6931
6932 Func = UpgradeSinglePointer(Func);
6933 if (!Func)
6934 return false;
6935
6936 NewStructors.push_back(
6937 ConstantStruct::get(Structor->getType(), {Prio, Func, Arg}));
6938 }
6939
6940 Constant *NewInit =
6941 ConstantArray::get(OldStructorsArray->getType(), NewStructors);
6942 GlobalArraysToUpgrade.emplace_back(GV, NewInit);
6943 }
6944
6945 if (GlobalArraysToUpgrade.empty())
6946 return false;
6947 assert(UseAddressDisc.has_value());
6948
6949 for (auto [GV, NewInit] : GlobalArraysToUpgrade)
6950 GV->setInitializer(NewInit);
6951
6952 M.addModuleFlag(Module::Error, "ptrauth-init-fini", 1);
6953 M.addModuleFlag(Module::Error, "ptrauth-init-fini-address-discrimination",
6954 *UseAddressDisc);
6955
6956 return true;
6957}
6958
6959bool llvm::UpgradeModuleFlags(Module &M) {
6960 bool Changed = false;
6961 Changed |= upgradePtrauthInitFiniArrays(M);
6962
6963 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
6964 if (!ModFlags)
6965 return Changed;
6966
6967 bool HasObjCFlag = false, HasClassProperties = false;
6968 bool HasSwiftVersionFlag = false;
6969 uint8_t SwiftMajorVersion, SwiftMinorVersion;
6970 uint32_t SwiftABIVersion;
6971 auto Int8Ty = Type::getInt8Ty(M.getContext());
6972 auto Int32Ty = Type::getInt32Ty(M.getContext());
6973
6974 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
6975 MDNode *Op = ModFlags->getOperand(I);
6976 if (Op->getNumOperands() != 3)
6977 continue;
6978 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
6979 if (!ID)
6980 continue;
6981 auto SetBehavior = [&](Module::ModFlagBehavior B) {
6982 Metadata *Ops[3] = {ConstantAsMetadata::get(ConstantInt::get(
6983 Type::getInt32Ty(M.getContext()), B)),
6984 MDString::get(M.getContext(), ID->getString()),
6985 Op->getOperand(2)};
6986 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
6987 Changed = true;
6988 };
6989
6990 if (ID->getString() == "Objective-C Image Info Version")
6991 HasObjCFlag = true;
6992 if (ID->getString() == "Objective-C Class Properties")
6993 HasClassProperties = true;
6994 // Upgrade PIC from Error/Max to Min.
6995 if (ID->getString() == "PIC Level") {
6996 if (auto *Behavior =
6997 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
6998 uint64_t V = Behavior->getLimitedValue();
6999 if (V == Module::Error || V == Module::Max)
7000 SetBehavior(Module::Min);
7001 }
7002 }
7003 // Upgrade "PIE Level" from Error to Max.
7004 if (ID->getString() == "PIE Level")
7005 if (auto *Behavior =
7006 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0)))
7007 if (Behavior->getLimitedValue() == Module::Error)
7008 SetBehavior(Module::Max);
7009
7010 // Upgrade branch protection and return address signing module flags. The
7011 // module flag behavior for these fields were Error and now they are Min.
7012 if (ID->getString() == "branch-target-enforcement" ||
7013 ID->getString().starts_with("sign-return-address")) {
7014 if (auto *Behavior =
7015 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
7016 if (Behavior->getLimitedValue() == Module::Error) {
7017 Type *Int32Ty = Type::getInt32Ty(M.getContext());
7018 Metadata *Ops[3] = {
7019 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Min)),
7020 Op->getOperand(1), Op->getOperand(2)};
7021 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7022 Changed = true;
7023 }
7024 }
7025 }
7026
7027 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
7028 // section name so that llvm-lto will not complain about mismatching
7029 // module flags that is functionally the same.
7030 if (ID->getString() == "Objective-C Image Info Section") {
7031 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
7032 SmallVector<StringRef, 4> ValueComp;
7033 Value->getString().split(ValueComp, " ");
7034 if (ValueComp.size() != 1) {
7035 std::string NewValue;
7036 for (auto &S : ValueComp)
7037 NewValue += S.str();
7038 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
7039 MDString::get(M.getContext(), NewValue)};
7040 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7041 Changed = true;
7042 }
7043 }
7044 }
7045
7046 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
7047 // If the higher bits are set, it adds new module flag for swift info.
7048 if (ID->getString() == "Objective-C Garbage Collection") {
7049 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
7050 if (Md) {
7051 assert(Md->getValue() && "Expected non-empty metadata");
7052 auto Type = Md->getValue()->getType();
7053 if (Type == Int8Ty)
7054 continue;
7055 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
7056 if ((Val & 0xff) != Val) {
7057 HasSwiftVersionFlag = true;
7058 SwiftABIVersion = (Val & 0xff00) >> 8;
7059 SwiftMajorVersion = (Val & 0xff000000) >> 24;
7060 SwiftMinorVersion = (Val & 0xff0000) >> 16;
7061 }
7062 Metadata *Ops[3] = {
7063 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
7064 Op->getOperand(1),
7065 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
7066 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7067 Changed = true;
7068 }
7069 }
7070
7071 if (ID->getString() == "amdgpu_code_object_version") {
7072 Metadata *Ops[3] = {
7073 Op->getOperand(0),
7074 MDString::get(M.getContext(), "amdhsa_code_object_version"),
7075 Op->getOperand(2)};
7076 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7077 Changed = true;
7078 }
7079
7080 // clang/PowerPC used to use "float-abi" to describe the long double format;
7081 // it has been renamed to "long-double-type", with its values changed to the
7082 // corresponding IR floating-point type names.
7083 if (M.getTargetTriple().isPPC() && ID->getString() == "float-abi") {
7084 StringRef Format;
7085 if (auto *S = dyn_cast_or_null<MDString>(Op->getOperand(2)))
7086 Format = S->getString();
7087
7088 // The "float-abi" key is now reserved for the target-independent
7089 // soft/hard ABI flag, so leave a valid value alone. Map any other value
7090 // (including unrecognized ones, which were never valid) to the default.
7091 if (!FloatABI::parseABIType(Format)) {
7092 LongDoubleFormat NewFormat =
7093 StringSwitch<LongDoubleFormat>(Format)
7094 .Case("ieeequad", LongDoubleFormat::IEEEquad)
7095 .Case("ieeedouble", LongDoubleFormat::IEEEdouble)
7096 .Default(LongDoubleFormat::PPCDoubleDouble);
7097 Metadata *Ops[3] = {
7098 Op->getOperand(0),
7099 MDString::get(M.getContext(), "long-double-type"),
7100 MDString::get(M.getContext(), getLongDoubleFormatName(NewFormat))};
7101 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
7102 Changed = true;
7103 }
7104 }
7105 }
7106
7107 // "Objective-C Class Properties" is recently added for Objective-C. We
7108 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
7109 // flag of value 0, so we can correclty downgrade this flag when trying to
7110 // link an ObjC bitcode without this module flag with an ObjC bitcode with
7111 // this module flag.
7112 if (HasObjCFlag && !HasClassProperties) {
7113 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
7114 (uint32_t)0);
7115 Changed = true;
7116 }
7117
7118 if (HasSwiftVersionFlag) {
7119 M.addModuleFlag(Module::Error, "Swift ABI Version",
7120 SwiftABIVersion);
7121 M.addModuleFlag(Module::Error, "Swift Major Version",
7122 ConstantInt::get(Int8Ty, SwiftMajorVersion));
7123 M.addModuleFlag(Module::Error, "Swift Minor Version",
7124 ConstantInt::get(Int8Ty, SwiftMinorVersion));
7125 Changed = true;
7126 }
7127
7128 return Changed;
7129}
7130
7131bool llvm::UpgradeCFIFunctionsMetadata(Module &M) {
7132 NamedMDNode *CFIConsts = M.getNamedMetadata("cfi.functions");
7133 // If this metadata has operands, we expect all of them to be either from
7134 // before or from after the format change handled here, so we can bail out
7135 // fast if the first (if any) operands is of the new format.
7136 auto MatchesVersion = [](const MDNode *Op) {
7137 return Op->getNumOperands() >= 3 &&
7138 isa<ConstantAsMetadata>(Op->getOperand(2)) &&
7139 cast<ConstantAsMetadata>(Op->getOperand(2))
7140 ->getType()
7141 ->isIntegerTy(64);
7142 };
7143
7144 if (!CFIConsts || !CFIConsts->getNumOperands() ||
7145 MatchesVersion(CFIConsts->getOperand(0)))
7146 return false;
7147
7148 bool Changed = false;
7149 for (unsigned I = 0, E = CFIConsts->getNumOperands(); I != E; ++I) {
7150 MDNode *Op = CFIConsts->getOperand(I);
7151 assert(!MatchesVersion(Op) && "Unexpected mix of CFIConstant formats");
7152 assert(Op->getNumOperands() >= 2 &&
7153 "Expected at least 2 operands - name and linkage type");
7154 MDString *NameMD = dyn_cast<MDString>(Op->getOperand(0));
7155 StringRef Name = NameMD->getString();
7156 GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(
7157 GlobalValue::dropLLVMManglingEscape(Name));
7158
7159 SmallVector<Metadata *, 4> Elts;
7160 Elts.push_back(Op->getOperand(0));
7161 Elts.push_back(Op->getOperand(1));
7162 Elts.push_back(ConstantAsMetadata::get(
7163 ConstantInt::get(Type::getInt64Ty(M.getContext()), GUID)));
7164
7165 for (unsigned J = 2, EJ = Op->getNumOperands(); J != EJ; ++J)
7166 Elts.push_back(Op->getOperand(J));
7167
7168 CFIConsts->setOperand(I, MDNode::get(M.getContext(), Elts));
7169 Changed = true;
7170 }
7171
7172 return Changed;
7173}
7174
7175void llvm::UpgradeSectionAttributes(Module &M) {
7176 auto TrimSpaces = [](StringRef Section) -> std::string {
7177 SmallVector<StringRef, 5> Components;
7178 Section.split(Components, ',');
7179
7180 SmallString<32> Buffer;
7181 raw_svector_ostream OS(Buffer);
7182
7183 for (auto Component : Components)
7184 OS << ',' << Component.trim();
7185
7186 return std::string(OS.str().substr(1));
7187 };
7188
7189 for (auto &GV : M.globals()) {
7190 if (!GV.hasSection())
7191 continue;
7192
7193 StringRef Section = GV.getSection();
7194
7195 if (!Section.starts_with("__DATA, __objc_catlist"))
7196 continue;
7197
7198 // __DATA, __objc_catlist, regular, no_dead_strip
7199 // __DATA,__objc_catlist,regular,no_dead_strip
7200 GV.setSection(TrimSpaces(Section));
7201 }
7202}
7203
7204namespace {
7205// Prior to LLVM 10.0, the strictfp attribute could be used on individual
7206// callsites within a function that did not also have the strictfp attribute.
7207// Since 10.0, if strict FP semantics are needed within a function, the
7208// function must have the strictfp attribute and all calls within the function
7209// must also have the strictfp attribute. This latter restriction is
7210// necessary to prevent unwanted libcall simplification when a function is
7211// being cloned (such as for inlining).
7212//
7213// The "dangling" strictfp attribute usage was only used to prevent constant
7214// folding and other libcall simplification. The nobuiltin attribute on the
7215// callsite has the same effect.
7216struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
7217 StrictFPUpgradeVisitor() = default;
7218
7219 void visitCallBase(CallBase &Call) {
7220 if (!Call.isStrictFP())
7221 return;
7222 if (isa<ConstrainedFPIntrinsic>(&Call))
7223 return;
7224 // If we get here, the caller doesn't have the strictfp attribute
7225 // but this callsite does. Replace the strictfp attribute with nobuiltin.
7226 Call.removeFnAttr(Attribute::StrictFP);
7227 Call.addFnAttr(Attribute::NoBuiltin);
7228 }
7229};
7230
7231/// Replace "amdgpu-unsafe-fp-atomics" metadata with atomicrmw metadata
7232struct AMDGPUUnsafeFPAtomicsUpgradeVisitor
7233 : public InstVisitor<AMDGPUUnsafeFPAtomicsUpgradeVisitor> {
7234 AMDGPUUnsafeFPAtomicsUpgradeVisitor() = default;
7235
7236 void visitAtomicRMWInst(AtomicRMWInst &RMW) {
7237 if (!RMW.isFloatingPointOperation())
7238 return;
7239
7240 MDNode *Empty = MDNode::get(RMW.getContext(), {});
7241 RMW.setMetadata("amdgpu.no.fine.grained.host.memory", Empty);
7242 RMW.setMetadata("amdgpu.no.remote.memory.access", Empty);
7243 RMW.setMetadata("amdgpu.ignore.denormal.mode", Empty);
7244 }
7245};
7246} // namespace
7247
7248void llvm::UpgradeFunctionAttributes(Function &F) {
7249 // If a function definition doesn't have the strictfp attribute,
7250 // convert any callsite strictfp attributes to nobuiltin.
7251 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
7252 StrictFPUpgradeVisitor SFPV;
7253 SFPV.visit(F);
7254 }
7255
7256 // Remove all incompatibile attributes from function.
7257 F.removeRetAttrs(AttributeFuncs::typeIncompatible(
7258 F.getReturnType(), F.getAttributes().getRetAttrs()));
7259 for (auto &Arg : F.args())
7260 Arg.removeAttrs(
7261 AttributeFuncs::typeIncompatible(Arg.getType(), Arg.getAttributes()));
7262
7263 bool AddingAttrs = false, RemovingAttrs = false;
7264 AttrBuilder AttrsToAdd(F.getContext());
7265 AttributeMask AttrsToRemove;
7266
7267 // Older versions of LLVM treated an "implicit-section-name" attribute
7268 // similarly to directly setting the section on a Function.
7269 if (Attribute A = F.getFnAttribute("implicit-section-name");
7270 A.isValid() && A.isStringAttribute()) {
7271 F.setSection(A.getValueAsString());
7272 AttrsToRemove.addAttribute("implicit-section-name");
7273 RemovingAttrs = true;
7274 }
7275
7276 if (Attribute A = F.getFnAttribute("nooutline");
7277 A.isValid() && A.isStringAttribute()) {
7278 AttrsToRemove.addAttribute("nooutline");
7279 AttrsToAdd.addAttribute(Attribute::NoOutline);
7280 AddingAttrs = RemovingAttrs = true;
7281 }
7282
7283 if (Attribute A = F.getFnAttribute("uniform-work-group-size");
7284 A.isValid() && A.isStringAttribute() && !A.getValueAsString().empty()) {
7285 AttrsToRemove.addAttribute("uniform-work-group-size");
7286 RemovingAttrs = true;
7287 if (A.getValueAsString() == "true") {
7288 AttrsToAdd.addAttribute("uniform-work-group-size");
7289 AddingAttrs = true;
7290 }
7291 }
7292
7293 if (!F.empty()) {
7294 // For some reason this is called twice, and the first time is before any
7295 // instructions are loaded into the body.
7296
7297 if (Attribute A = F.getFnAttribute("amdgpu-unsafe-fp-atomics");
7298 A.isValid()) {
7299
7300 if (A.getValueAsBool()) {
7301 AMDGPUUnsafeFPAtomicsUpgradeVisitor Visitor;
7302 Visitor.visit(F);
7303 }
7304
7305 // We will leave behind dead attribute uses on external declarations, but
7306 // clang never added these to declarations anyway.
7307 AttrsToRemove.addAttribute("amdgpu-unsafe-fp-atomics");
7308 RemovingAttrs = true;
7309 }
7310 }
7311
7312 DenormalMode DenormalFPMath = DenormalMode::getIEEE();
7313 DenormalMode DenormalFPMathF32 = DenormalMode::getInvalid();
7314
7315 bool HandleDenormalMode = false;
7316
7317 if (Attribute Attr = F.getFnAttribute("denormal-fp-math"); Attr.isValid()) {
7318 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7319 if (ParsedMode.isValid()) {
7320 DenormalFPMath = ParsedMode;
7321 AttrsToRemove.addAttribute("denormal-fp-math");
7322 AddingAttrs = RemovingAttrs = true;
7323 HandleDenormalMode = true;
7324 }
7325 }
7326
7327 if (Attribute Attr = F.getFnAttribute("denormal-fp-math-f32");
7328 Attr.isValid()) {
7329 DenormalMode ParsedMode = parseDenormalFPAttribute(Attr.getValueAsString());
7330 if (ParsedMode.isValid()) {
7331 DenormalFPMathF32 = ParsedMode;
7332 AttrsToRemove.addAttribute("denormal-fp-math-f32");
7333 AddingAttrs = RemovingAttrs = true;
7334 HandleDenormalMode = true;
7335 }
7336 }
7337
7338 if (HandleDenormalMode)
7339 AttrsToAdd.addDenormalFPEnvAttr(
7340 DenormalFPEnv(DenormalFPMath, DenormalFPMathF32));
7341
7342 if (RemovingAttrs)
7343 F.removeFnAttrs(AttrsToRemove);
7344
7345 if (AddingAttrs)
7346 F.addFnAttrs(AttrsToAdd);
7347}
7348
7349// Check if the function attribute is not present and set it.
7350static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName,
7351 StringRef Value) {
7352 if (!F.hasFnAttribute(FnAttrName))
7353 F.addFnAttr(FnAttrName, Value);
7354}
7355
7356// Check if the function attribute is not present and set it if needed.
7357// If the attribute is "false" then removes it.
7358// If the attribute is "true" resets it to a valueless attribute.
7359static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName) {
7360 if (!F.hasFnAttribute(FnAttrName)) {
7361 if (Set)
7362 F.addFnAttr(FnAttrName);
7363 } else {
7364 auto A = F.getFnAttribute(FnAttrName);
7365 if ("false" == A.getValueAsString())
7366 F.removeFnAttr(FnAttrName);
7367 else if ("true" == A.getValueAsString()) {
7368 F.removeFnAttr(FnAttrName);
7369 F.addFnAttr(FnAttrName);
7370 }
7371 }
7372}
7373
7374void llvm::copyModuleAttrToFunctions(Module &M) {
7375 Triple T(M.getTargetTriple());
7376 if (!T.isThumb() && !T.isARM() && !T.isAArch64())
7377 return;
7378
7379 uint64_t BTEValue = 0;
7380 uint64_t BPPLRValue = 0;
7381 uint64_t GCSValue = 0;
7382 uint64_t SRAValue = 0;
7383 uint64_t SRAALLValue = 0;
7384 uint64_t SRABKeyValue = 0;
7385
7386 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
7387 if (ModFlags) {
7388 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
7389 MDNode *Op = ModFlags->getOperand(I);
7390 if (Op->getNumOperands() != 3)
7391 continue;
7392
7393 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
7394 auto *CI = mdconst::dyn_extract<ConstantInt>(Op->getOperand(2));
7395 if (!ID || !CI)
7396 continue;
7397
7398 StringRef IDStr = ID->getString();
7399 uint64_t *ValPtr = IDStr == "branch-target-enforcement" ? &BTEValue
7400 : IDStr == "branch-protection-pauth-lr" ? &BPPLRValue
7401 : IDStr == "guarded-control-stack" ? &GCSValue
7402 : IDStr == "sign-return-address" ? &SRAValue
7403 : IDStr == "sign-return-address-all" ? &SRAALLValue
7404 : IDStr == "sign-return-address-with-bkey"
7405 ? &SRABKeyValue
7406 : nullptr;
7407 if (!ValPtr)
7408 continue;
7409
7410 *ValPtr = CI->getZExtValue();
7411 if (*ValPtr == 2)
7412 return;
7413 }
7414 }
7415
7416 bool BTE = BTEValue == 1;
7417 bool BPPLR = BPPLRValue == 1;
7418 bool GCS = GCSValue == 1;
7419 bool SRA = SRAValue == 1;
7420
7421 StringRef SignTypeValue = "non-leaf";
7422 if (SRA && SRAALLValue == 1)
7423 SignTypeValue = "all";
7424
7425 StringRef SignKeyValue = "a_key";
7426 if (SRA && SRABKeyValue == 1)
7427 SignKeyValue = "b_key";
7428
7429 for (Function &F : M.getFunctionList()) {
7430 if (F.isDeclaration())
7431 continue;
7432
7433 if (SRA) {
7434 setFunctionAttrIfNotSet(F, "sign-return-address", SignTypeValue);
7435 setFunctionAttrIfNotSet(F, "sign-return-address-key", SignKeyValue);
7436 } else {
7437 if (auto A = F.getFnAttribute("sign-return-address");
7438 A.isValid() && "none" == A.getValueAsString()) {
7439 F.removeFnAttr("sign-return-address");
7440 F.removeFnAttr("sign-return-address-key");
7441 }
7442 }
7443 ConvertFunctionAttr(F, BTE, "branch-target-enforcement");
7444 ConvertFunctionAttr(F, BPPLR, "branch-protection-pauth-lr");
7445 ConvertFunctionAttr(F, GCS, "guarded-control-stack");
7446 }
7447
7448 if (BTE)
7449 M.setModuleFlag(llvm::Module::Min, "branch-target-enforcement", 2);
7450 if (BPPLR)
7451 M.setModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr", 2);
7452 if (GCS)
7453 M.setModuleFlag(llvm::Module::Min, "guarded-control-stack", 2);
7454 if (SRA) {
7455 M.setModuleFlag(llvm::Module::Min, "sign-return-address", 2);
7456 if (SRAALLValue == 1)
7457 M.setModuleFlag(llvm::Module::Min, "sign-return-address-all", 2);
7458 if (SRABKeyValue == 1)
7459 M.setModuleFlag(llvm::Module::Min, "sign-return-address-with-bkey", 2);
7460 }
7461}
7462
7463/// Return the replacement tags if \p T still uses a removed two-operand form.
7464static const BooleanLoopTags *getOldBooleanLoopTags(const MDTuple *T) {
7465 if (T->getNumOperands() != 2 || !mdconst::hasa<ConstantInt>(T->getOperand(1)))
7466 return nullptr;
7467 auto *Tag = dyn_cast_or_null<MDString>(T->getOperand(0));
7468 return Tag ? findBooleanLoopTags(Tag->getString()) : nullptr;
7469}
7470
7471/// Build the single-operand node that replaces a boolean operand: nonzero
7472/// selects the enable tag, zero the disable tag.
7473static Metadata *makeBooleanLoopNode(LLVMContext &C,
7474 const BooleanLoopTags &Tags,
7475 const MDOperand &Op) {
7476 bool Enable = !mdconst::extract<ConstantInt>(Op)->isZero();
7477 return MDTuple::get(C,
7478 {MDString::get(C, Enable ? Tags.Enable : Tags.Disable)});
7479}
7480
7481static bool isOldLoopArgument(Metadata *MD) {
7482 auto *T = dyn_cast_or_null<MDTuple>(MD);
7483 if (!T)
7484 return false;
7485 if (T->getNumOperands() < 1)
7486 return false;
7487 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
7488 if (!S)
7489 return false;
7490 if (S->getString().starts_with("llvm.vectorizer."))
7491 return true;
7492 return getOldBooleanLoopTags(T) != nullptr;
7493}
7494
7495static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
7496 StringRef OldPrefix = "llvm.vectorizer.";
7497 assert(OldTag.starts_with(OldPrefix) && "Expected old prefix");
7498
7499 if (OldTag == "llvm.vectorizer.unroll")
7500 return MDString::get(C, "llvm.loop.interleave.count");
7501
7502 return MDString::get(
7503 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
7504 .str());
7505}
7506
7507static Metadata *upgradeLoopArgument(Metadata *MD) {
7508 auto *T = dyn_cast_or_null<MDTuple>(MD);
7509 if (!T)
7510 return MD;
7511 if (T->getNumOperands() < 1)
7512 return MD;
7513 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
7514 if (!OldTag)
7515 return MD;
7516
7517 LLVMContext &C = T->getContext();
7518
7519 /// Rewrite a removed two-operand boolean form to the single-operand pair.
7520 if (const BooleanLoopTags *Tags = getOldBooleanLoopTags(T))
7521 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7522
7523 if (!OldTag->getString().starts_with("llvm.vectorizer."))
7524 return MD;
7525
7526 // This has an old tag. Upgrade it.
7527 MDString *NewTag = upgradeLoopTag(C, OldTag->getString());
7528
7529 // The legacy !{!"llvm.vectorizer.enable", i1 X} maps onto the single-operand
7530 // vectorize.enable/disable pair, not a two-operand enable node.
7531 if (T->getNumOperands() == 2 && mdconst::hasa<ConstantInt>(T->getOperand(1)))
7532 if (const BooleanLoopTags *Tags = findBooleanLoopTags(NewTag->getString()))
7533 return makeBooleanLoopNode(C, *Tags, T->getOperand(1));
7534
7535 SmallVector<Metadata *, 8> Ops;
7536 Ops.reserve(T->getNumOperands());
7537 Ops.push_back(NewTag);
7538 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
7539 Ops.push_back(T->getOperand(I));
7540
7541 return MDTuple::get(C, Ops);
7542}
7543
7544MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
7545 auto *T = dyn_cast<MDTuple>(&N);
7546 if (!T)
7547 return &N;
7548
7549 if (none_of(T->operands(), isOldLoopArgument))
7550 return &N;
7551
7552 // Fix the removed two-operand boolean nodes in place: the Verifier rejects
7553 // any MDNode carrying those tags with more than one operand, so a leftover
7554 // reference (from the distinct loop-ID) would still trigger a diagnostic.
7555 // In-place mutation is safe on distinct MDNodes.
7556 if (T->isDistinct()) {
7557 for (unsigned I = 0, E = T->getNumOperands(); I < E; ++I) {
7558 auto *OpT = dyn_cast_or_null<MDTuple>(T->getOperand(I));
7559 if (OpT && getOldBooleanLoopTags(OpT))
7560 T->replaceOperandWith(I, upgradeLoopArgument(OpT));
7561 }
7562 if (none_of(T->operands(), isOldLoopArgument))
7563 return &N;
7564 }
7565
7566 // Remaining old arguments (e.g. llvm.vectorizer.*) are handled via a wrapper
7567 // attachment; the original distinct loop-ID is kept as the first operand.
7568 SmallVector<Metadata *, 8> Ops;
7569 Ops.reserve(T->getNumOperands());
7570 for (Metadata *MD : T->operands())
7571 Ops.push_back(upgradeLoopArgument(MD));
7572
7573 return MDTuple::get(T->getContext(), Ops);
7574}
7575
7576std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
7577 Triple T(TT);
7578 // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
7579 // the address space of globals to 1. This does not apply to SPIRV Logical.
7580 if ((T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical())) &&
7581 !DL.contains("-G") && !DL.starts_with("G")) {
7582 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
7583 }
7584
7585 if (T.isLoongArch64() || T.isRISCV64()) {
7586 // Make i32 a native type for 64-bit LoongArch and RISC-V.
7587 auto I = DL.find("-n64-");
7588 if (I != StringRef::npos)
7589 return (DL.take_front(I) + "-n32:64-" + DL.drop_front(I + 5)).str();
7590 return DL.str();
7591 }
7592
7593 // AMDGPU data layout upgrades.
7594 std::string Res = DL.str();
7595 if (T.isAMDGPU()) {
7596 // Define address spaces for constants.
7597 if (!DL.contains("-G") && !DL.starts_with("G"))
7598 Res.append(Res.empty() ? "G1" : "-G1");
7599
7600 // AMDGCN data layout upgrades.
7601 if (T.isAMDGCN()) {
7602
7603 // Add missing non-integral declarations.
7604 // This goes before adding new address spaces to prevent incoherent string
7605 // values.
7606 if (!DL.contains("-ni") && !DL.starts_with("ni"))
7607 Res.append("-ni:7:8:9");
7608 // Update ni:7 to ni:7:8:9.
7609 if (DL.ends_with("ni:7"))
7610 Res.append(":8:9");
7611 if (DL.ends_with("ni:7:8"))
7612 Res.append(":9");
7613
7614 // Add sizing for address spaces 7 and 8 (fat raw buffers and buffer
7615 // resources) An empty data layout has already been upgraded to G1 by now.
7616 if (!DL.contains("-p7") && !DL.starts_with("p7"))
7617 Res.append("-p7:160:256:256:32");
7618 if (!DL.contains("-p8") && !DL.starts_with("p8"))
7619 Res.append("-p8:128:128:128:48");
7620 constexpr StringRef OldP8("-p8:128:128-");
7621 if (DL.contains(OldP8))
7622 Res.replace(Res.find(OldP8), OldP8.size(), "-p8:128:128:128:48-");
7623 if (!DL.contains("-p9") && !DL.starts_with("p9"))
7624 Res.append("-p9:192:256:256:32");
7625 }
7626
7627 // Upgrade the ELF mangling mode.
7628 if (!DL.contains("m:e"))
7629 Res = Res.empty() ? "m:e" : "m:e-" + Res;
7630
7631 return Res;
7632 }
7633
7634 if (T.isSystemZ() && !DL.empty()) {
7635 // Make sure the stack alignment is present.
7636 if (!DL.contains("-S64"))
7637 return "E-S64" + DL.drop_front(1).str();
7638 return DL.str();
7639 }
7640
7641 auto AddPtr32Ptr64AddrSpaces = [&DL, &Res]() {
7642 // If the datalayout matches the expected format, add pointer size address
7643 // spaces to the datalayout.
7644 StringRef AddrSpaces{"-p270:32:32-p271:32:32-p272:64:64"};
7645 if (!DL.contains(AddrSpaces)) {
7646 SmallVector<StringRef, 4> Groups;
7647 Regex R("^([Ee]-m:[a-z](-p:32:32)?)(-.*)$");
7648 if (R.match(Res, &Groups))
7649 Res = (Groups[1] + AddrSpaces + Groups[3]).str();
7650 }
7651 };
7652
7653 // AArch64 data layout upgrades.
7654 if (T.isAArch64()) {
7655 // Add "-Fn32"
7656 if (!DL.empty() && !DL.contains("-Fn32"))
7657 Res.append("-Fn32");
7658 AddPtr32Ptr64AddrSpaces();
7659 return Res;
7660 }
7661
7662 if (T.isSPARC() || (T.isMIPS64() && !DL.contains("m:m")) || T.isPPC64() ||
7663 T.isWasm()) {
7664 // Mips64 with o32 ABI did not add "-i128:128".
7665 // Add "-i128:128"
7666 std::string I64 = "-i64:64";
7667 std::string I128 = "-i128:128";
7668 if (!StringRef(Res).contains(I128)) {
7669 size_t Pos = Res.find(I64);
7670 if (Pos != size_t(-1))
7671 Res.insert(Pos + I64.size(), I128);
7672 }
7673 }
7674
7675 if (T.isPPC() && T.isOSAIX() && !DL.contains("f64:32:64") && !DL.empty()) {
7676 size_t Pos = Res.find("-S128");
7677 if (Pos == StringRef::npos)
7678 Pos = Res.size();
7679 Res.insert(Pos, "-f64:32:64");
7680 }
7681
7682 if (!T.isX86())
7683 return Res;
7684
7685 AddPtr32Ptr64AddrSpaces();
7686
7687 // i128 values need to be 16-byte-aligned. LLVM already called into libgcc
7688 // for i128 operations prior to this being reflected in the data layout, and
7689 // clang mostly produced LLVM IR that already aligned i128 to 16 byte
7690 // boundaries, so although this is a breaking change, the upgrade is expected
7691 // to fix more IR than it breaks.
7692 // Intel MCU is an exception and uses 4-byte-alignment.
7693 if (!T.isOSIAMCU()) {
7694 std::string I128 = "-i128:128";
7695 if (StringRef Ref = Res; !Ref.contains(I128)) {
7696 SmallVector<StringRef, 4> Groups;
7697 Regex R("^(e(-[mpi][^-]*)*)((-[^mpi][^-]*)*)$");
7698 if (R.match(Res, &Groups))
7699 Res = (Groups[1] + I128 + Groups[3]).str();
7700 }
7701 }
7702
7703 // For 32-bit MSVC targets, raise the alignment of f80 values to 16 bytes.
7704 // Raising the alignment is safe because Clang did not produce f80 values in
7705 // the MSVC environment before this upgrade was added.
7706 if (T.isWindowsMSVCEnvironment() && !T.isArch64Bit()) {
7707 StringRef Ref = Res;
7708 auto I = Ref.find("-f80:32-");
7709 if (I != StringRef::npos)
7710 Res = (Ref.take_front(I) + "-f80:128-" + Ref.drop_front(I + 8)).str();
7711 }
7712
7713 return Res;
7714}
7715
7716void llvm::UpgradeAttributes(AttrBuilder &B) {
7717 StringRef FramePointer;
7718 Attribute A = B.getAttribute("no-frame-pointer-elim");
7719 if (A.isValid()) {
7720 // The value can be "true" or "false".
7721 FramePointer = A.getValueAsString() == "true" ? "all" : "none";
7722 B.removeAttribute("no-frame-pointer-elim");
7723 }
7724 if (B.contains("no-frame-pointer-elim-non-leaf")) {
7725 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
7726 if (FramePointer != "all")
7727 FramePointer = "non-leaf";
7728 B.removeAttribute("no-frame-pointer-elim-non-leaf");
7729 }
7730 if (!FramePointer.empty())
7731 B.addAttribute("frame-pointer", FramePointer);
7732
7733 A = B.getAttribute("null-pointer-is-valid");
7734 if (A.isValid()) {
7735 // The value can be "true" or "false".
7736 bool NullPointerIsValid = A.getValueAsString() == "true";
7737 B.removeAttribute("null-pointer-is-valid");
7738 if (NullPointerIsValid)
7739 B.addAttribute(Attribute::NullPointerIsValid);
7740 }
7741
7742 A = B.getAttribute("uniform-work-group-size");
7743 if (A.isValid()) {
7744 StringRef Val = A.getValueAsString();
7745 if (!Val.empty()) {
7746 bool IsTrue = Val == "true";
7747 B.removeAttribute("uniform-work-group-size");
7748 if (IsTrue)
7749 B.addAttribute("uniform-work-group-size");
7750 }
7751 }
7752}
7753
7754void llvm::UpgradeOperandBundles(std::vector<OperandBundleDef> &Bundles) {
7755 // clang.arc.attachedcall bundles are now required to have an operand.
7756 // If they don't, it's okay to drop them entirely: when there is an operand,
7757 // the "attachedcall" is meaningful and required, but without an operand,
7758 // it's just a marker NOP. Dropping it merely prevents an optimization.
7759 erase_if(Bundles, [&](OperandBundleDef &OBD) {
7760 return OBD.getTag() == "clang.arc.attachedcall" &&
7761 OBD.inputs().empty();
7762 });
7763}
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPUAddrSpace.h
AMDGPU address space definition.
Imm
unsigned Imm
Definition AMDGPUDisassembler.cpp:510
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:511
Select
AMDGPU Register Bank Select
Definition AMDGPURegBankSelect.cpp:68
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
Attributes.h
This file contains the simple types necessary to represent the attributes associated with functions a...
upgradeIntrinsicDeclWithDefaultArgs
static bool upgradeIntrinsicDeclWithDefaultArgs(Function *F, Function *&NewFn)
Definition AutoUpgrade.cpp:1534
upgradeX86VPERMT2Intrinsics
static Value * upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
Definition AutoUpgrade.cpp:2483
isLegacyNVPTXBF16IntSignature
static bool isLegacyNVPTXBF16IntSignature(Function *F, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:1391
upgradeLoopArgument
static Metadata * upgradeLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7507
isXYZ
static bool isXYZ(StringRef S)
Definition AutoUpgrade.cpp:6599
upgradeIntrinsicFunction1
static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
Definition AutoUpgrade.cpp:1564
upgradeX86PSLLDQIntrinsics
static Value * upgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2323
shouldUpgradeNVPTXSharedClusterIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1254
upgradeVPIntrinsicCall
static Value * upgradeVPIntrinsicCall(StringRef Name, CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5451
getNVPTXTMAReductionOp
static std::optional< unsigned > getNVPTXTMAReductionOp(StringRef Name)
Definition AutoUpgrade.cpp:1221
shouldUpgradeNVPTXTMAReductionIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics(StringRef Name)
Definition AutoUpgrade.cpp:1234
upgradeRetainReleaseMarker
static bool upgradeRetainReleaseMarker(Module &M)
This checks for objc retain release marker which should be upgraded.
Definition AutoUpgrade.cpp:6705
upgradeX86vpcom
static Value * upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
Definition AutoUpgrade.cpp:2584
upgradeMaskToInt
static Value * upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2810
convertIntrinsicValidType
static bool convertIntrinsicValidType(StringRef Name, const FunctionType *FuncTy)
Definition AutoUpgrade.cpp:1514
upgradeX86Rotate
static Value * upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
Definition AutoUpgrade.cpp:2558
upgradeX86MultiplyAddBytes
static bool upgradeX86MultiplyAddBytes(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:126
getFunctionalIntrinsicIDForVP
static Intrinsic::ID getFunctionalIntrinsicIDForVP(StringRef Name)
Definition AutoUpgrade.cpp:1467
setFunctionAttrIfNotSet
static void setFunctionAttrIfNotSet(Function &F, StringRef FnAttrName, StringRef Value)
Definition AutoUpgrade.cpp:7350
shouldUpgradeNVPTXBF16Intrinsic
static Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1325
upgradeSingleNVVMAnnotation
static bool upgradeSingleNVVMAnnotation(GlobalValue *GV, StringRef K, const Metadata *V)
Definition AutoUpgrade.cpp:6603
unwrapMAVOp
static MDNode * unwrapMAVOp(CallBase *CI, unsigned Op)
Helper to unwrap intrinsic call MetadataAsValue operands.
Definition AutoUpgrade.cpp:5311
upgradeLoopTag
static MDString * upgradeLoopTag(LLVMContext &C, StringRef OldTag)
Definition AutoUpgrade.cpp:7495
getVPIntPredicateFromMD
static ICmpInst::Predicate getVPIntPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5411
upgradeNVVMFnVectorAttr
static void upgradeNVVMFnVectorAttr(const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
Definition AutoUpgrade.cpp:6566
upgradeX86MaskedFPCompare
static bool upgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:113
upgradeX86ALIGNIntrinsics
static Value * upgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
Definition AutoUpgrade.cpp:2438
upgradeAbs
static Value * upgradeAbs(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2695
shouldUpgradeVPIntrinsic
static bool shouldUpgradeVPIntrinsic(StringRef Name)
Definition AutoUpgrade.cpp:1510
emitX86Select
static Value * emitX86Select(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2409
upgradeAArch64IntrinsicCall
static Value * upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:4957
upgradeMaskedMove
static Value * upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI)
Definition AutoUpgrade.cpp:2796
getOldBooleanLoopTags
static const BooleanLoopTags * getOldBooleanLoopTags(const MDTuple *T)
Return the replacement tags if T still uses a removed two-operand form.
Definition AutoUpgrade.cpp:7464
upgradeX86IntrinsicFunction
static bool upgradeX86IntrinsicFunction(Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:551
applyX86MaskOn1BitsVec
static Value * applyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, Value *Mask)
Definition AutoUpgrade.cpp:2735
shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocDeallocIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1305
getModuleFlagNameSafely
static std::optional< StringRef > getModuleFlagNameSafely(const MDNode &Flag)
Definition AutoUpgrade.cpp:6512
consumeNVVMPtrAddrSpace
static bool consumeNVVMPtrAddrSpace(StringRef &Name)
Definition AutoUpgrade.cpp:1424
makeBooleanLoopNode
static Metadata * makeBooleanLoopNode(LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag,...
Definition AutoUpgrade.cpp:7473
shouldUpgradeX86Intrinsic
static bool shouldUpgradeX86Intrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:180
upgradeX86PSRLDQIntrinsics
static Value * upgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift)
Definition AutoUpgrade.cpp:2357
getFunctionalOpcodeForVP
static unsigned getFunctionalOpcodeForVP(StringRef Name)
Definition AutoUpgrade.cpp:1430
shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1280
shouldUpgradeNVPTXTMAG2SIntrinsics
static Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1171
isOldLoopArgument
static bool isOldLoopArgument(Metadata *MD)
Definition AutoUpgrade.cpp:7481
upgradeARMIntrinsicCall
static Value * upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5027
upgradeX86IntrinsicsWith8BitMask
static bool upgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:97
upgradeVectorSplice
static Value * upgradeVectorSplice(CallBase *CI, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5381
upgradeAMDGCNIntrinsicCall
static Value * upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5117
upgradeMaskedLoad
static Value * upgradeMaskedLoad(IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2674
unwrapMAVMetadataOp
static Metadata * unwrapMAVMetadataOp(CallBase *CI, unsigned Op)
Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
Definition AutoUpgrade.cpp:5323
upgradeX86BF16Intrinsic
static bool upgradeX86BF16Intrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:160
upgradeArmOrAarch64IntrinsicFunction
static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, StringRef Name, Function *&NewFn)
Definition AutoUpgrade.cpp:789
upgradeIntrinsicCallWithDefaultArgs
static bool upgradeIntrinsicCallWithDefaultArgs(CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5482
getX86MaskVec
static Value * getX86MaskVec(IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
Definition AutoUpgrade.cpp:2389
emitX86ScalarSelect
static Value * emitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
Definition AutoUpgrade.cpp:2421
upgradeX86ConcatShift
static Value * upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
Definition AutoUpgrade.cpp:2623
shouldUpgradeNVPTXTcgen05MMAIntrinsic
static Intrinsic::ID shouldUpgradeNVPTXTcgen05MMAIntrinsic(Function *F, StringRef Name)
Definition AutoUpgrade.cpp:1412
rename
static void rename(GlobalValue *GV)
Definition AutoUpgrade.cpp:68
upgradePTESTIntrinsic
static bool upgradePTESTIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:81
upgradeX86BF16DPIntrinsic
static bool upgradeX86BF16DPIntrinsic(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:170
DisableAutoUpgradeDebugInfo
static cl::opt< bool > DisableAutoUpgradeDebugInfo("disable-auto-upgrade-debug-info", cl::desc("Disable autoupgrade of debug info"))
upgradeMaskedCompare
static Value * upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
Definition AutoUpgrade.cpp:2757
upgradeX86BinaryIntrinsics
static Value * upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2543
upgradeNVVMIntrinsicCall
static Value * upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3067
upgradeX86MaskedShift
static Value * upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
Definition AutoUpgrade.cpp:2789
upgradeAVX512MaskToSelect
static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
Definition AutoUpgrade.cpp:2819
upgradeDbgIntrinsicToDbgRecord
static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI)
Convert debug intrinsic calls to non-instruction debug records.
Definition AutoUpgrade.cpp:5333
ConvertFunctionAttr
static void ConvertFunctionAttr(Function &F, bool Set, StringRef FnAttrName)
Definition AutoUpgrade.cpp:7359
upgradePMULDQ
static Value * upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
Definition AutoUpgrade.cpp:2705
reportFatalUsageErrorWithCI
static void reportFatalUsageErrorWithCI(StringRef reason, CallBase *CI)
Definition AutoUpgrade.cpp:72
upgradeMaskedStore
static Value * upgradeMaskedStore(IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
Definition AutoUpgrade.cpp:2656
upgradeConvertIntrinsicCall
static Value * upgradeConvertIntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:5394
upgradeX86MultiplyAddWords
static bool upgradeX86MultiplyAddWords(Function *F, Intrinsic::ID IID, Function *&NewFn)
Definition AutoUpgrade.cpp:144
upgradePtrauthInitFiniArrays
static bool upgradePtrauthInitFiniArrays(Module &M)
Definition AutoUpgrade.cpp:6860
upgradeX86IntrinsicCall
static Value * upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
Definition AutoUpgrade.cpp:3265
getVPFPPredicateFromMD
static FCmpInst::Predicate getVPFPPredicateFromMD(const Value *Op)
Definition AutoUpgrade.cpp:5429
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Enable
@ Enable
Definition DwarfDebug.cpp:87
Dwarf.h
This file contains constants used for implementing Dwarf debug support.
Module.h
Module.h This file contains the declarations for the Module class.
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:539
F
#define F(x, y, z)
Definition MD5.cpp:54
I
#define I(x, y, z)
Definition MD5.cpp:57
R2
#define R2(n)
Metadata.h
This file contains the declarations for metadata subclasses.
T
#define T
Definition Mips16ISelLowering.cpp:282
T1
#define T1
Definition Mips16ISelLowering.cpp:281
NVPTXAddrSpace.h
NVPTX address space definition.
High
uint64_t High
Definition NVVMIntrRange.cpp:46
NVVMIntrinsicUtils.h
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
getType
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
Empty
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
Definition TypeHashing.cpp:17
Groups
static const X86InstrFMA3Group Groups[]
Definition X86InstrFMA3Info.cpp:81
RHS
Value * RHS
Definition X86PartialReduction.cpp:83
LHS
Value * LHS
Definition X86PartialReduction.cpp:82
llvm::APInt
Class for arbitrary precision integers.
Definition APInt.h:78
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayType
Class to represent array types.
Definition DerivedTypes.h:458
llvm::ArrayType::get
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
llvm::ArrayType::getElementType
Type * getElementType() const
Definition DerivedTypes.h:471
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition Instructions.h:776
llvm::AtomicRMWInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile RMW or not.
Definition Instructions.h:936
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition Instructions.h:788
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition Instructions.h:792
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition Instructions.h:813
llvm::AtomicRMWInst::USubCond
@ USubCond
Subtract only if no unsigned overflow.
Definition Instructions.h:852
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition Instructions.h:806
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition Instructions.h:796
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition Instructions.h:802
llvm::AtomicRMWInst::USubSat
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition Instructions.h:856
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition Instructions.h:844
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition Instructions.h:804
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition Instructions.h:824
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition Instructions.h:820
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition Instructions.h:848
llvm::AtomicRMWInst::isFloatingPointOperation
bool isFloatingPointOperation() const
Definition Instructions.h:983
llvm::AttributeMask
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
Definition AttributeMask.h:29
llvm::AttributeMask::addAttribute
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition AttributeMask.h:44
llvm::Attribute
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:105
llvm::Attribute::getWithStackAlignment
static LLVM_ABI Attribute getWithStackAlignment(LLVMContext &Context, Align Alignment)
Definition Attributes.cpp:239
llvm::Attribute::get
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition Attributes.cpp:95
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition InstrTypes.h:1181
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition InstrTypes.h:1479
llvm::CallBase::getOperandBundlesAsDefs
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition Instructions.cpp:511
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition InstrTypes.h:1417
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition InstrTypes.h:1475
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition InstrTypes.h:1409
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the attributes for this call.
Definition InstrTypes.h:1496
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition InstrTypes.h:1361
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition InstrTypes.h:1274
llvm::CallBase::getIntrinsicID
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
Definition Instructions.cpp:359
llvm::CallBase::args
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition InstrTypes.h:1352
llvm::CallBase::setCalledOperand
void setCalledOperand(Value *V)
Definition InstrTypes.h:1453
llvm::CallBase::arg_size
unsigned arg_size() const
Definition InstrTypes.h:1359
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the attributes for this call.
Definition InstrTypes.h:1493
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition InstrTypes.h:1456
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition Instructions.h:1597
llvm::CallInst::setTailCallKind
void setTailCallKind(TailCallKind TCK)
Definition Instructions.h:1717
llvm::CastInst::Create
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition Instructions.cpp:3109
llvm::CastInst::castIsValid
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Definition Instructions.cpp:3392
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
llvm::CmpInst::FCMP_OEQ
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
llvm::CmpInst::FCMP_OLT
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
llvm::CmpInst::FCMP_ULE
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
Definition InstrTypes.h:755
llvm::CmpInst::FCMP_OGT
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
Definition InstrTypes.h:744
llvm::CmpInst::FCMP_OGE
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
Definition InstrTypes.h:745
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
llvm::CmpInst::FCMP_ULT
@ FCMP_ULT
1 1 0 0 True if unordered or less than
Definition InstrTypes.h:754
llvm::CmpInst::FCMP_ONE
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
llvm::CmpInst::FCMP_UEQ
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
llvm::CmpInst::FCMP_UGT
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
Definition InstrTypes.h:752
llvm::CmpInst::FCMP_OLE
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
llvm::CmpInst::FCMP_ORD
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition InstrTypes.h:762
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
llvm::CmpInst::FCMP_UNE
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition InstrTypes.h:756
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
llvm::CmpInst::FCMP_UGE
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
Definition InstrTypes.h:753
llvm::CmpInst::FCMP_UNO
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
llvm::ConstantAggregateZero::get
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
Definition Constants.cpp:1810
llvm::ConstantArray::get
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1432
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition Metadata.h:537
llvm::ConstantExpr::getIntToPtr
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2470
llvm::ConstantExpr::getPointerCast
static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition Constants.cpp:2402
llvm::ConstantExpr::getPtrToInt
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition Constants.cpp:2456
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition Constants.h:87
llvm::ConstantInt::isZero
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
llvm::ConstantPointerNull::get
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition Constants.cpp:1976
llvm::ConstantStruct::get
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition Constants.cpp:1499
llvm::ConstantStruct::getType
StructType * getType() const
Specialization - reduce amount of casting.
Definition Constants.h:661
llvm::ConstantTokenNone::get
static LLVM_ABI ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition Constants.cpp:1659
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::Constant::getAllOnesValue
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
Definition Constants.cpp:427
llvm::Constant::getNullValue
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition Constants.cpp:367
llvm::DIExpression
DWARF expression.
Definition DebugInfoMetadata.h:3474
llvm::DIExpression::append
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
Definition DebugInfoMetadata.cpp:2335
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord
static LLVM_ABI DbgLabelRecord * createUnresolvedDbgLabelRecord(MDNode *Label)
For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved MDNodes.
Definition DebugProgramInstruction.cpp:159
llvm::DbgRecord
Base class for non-instruction debug metadata records that have positions within IR.
Definition DebugProgramInstruction.h:139
llvm::DbgRecord::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Definition DebugProgramInstruction.h:217
llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord
static LLVM_ABI DbgVariableRecord * createUnresolvedDbgVariableRecord(LocationType Type, Metadata *Val, MDNode *Variable, MDNode *Expression, MDNode *AssignID, Metadata *Address, MDNode *AddressExpression)
Used to create DbgVariableRecords during parsing, where some metadata references may still be unresol...
Definition DebugProgramInstruction.cpp:172
llvm::DiagnosticInfoDebugMetadataVersion
Diagnostic information for debug metadata version reporting.
Definition DiagnosticInfo.h:214
llvm::DiagnosticInfoIgnoringInvalidDebugMetadata
Diagnostic information for stripping invalid debug metadata.
Definition DiagnosticInfo.h:242
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
llvm::FastMathFlags::setApproxFunc
void setApproxFunc(bool B=true)
Definition FMF.h:93
llvm::FixedVectorType::get
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:867
llvm::FunctionType
Class to represent function types.
Definition DerivedTypes.h:165
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition DerivedTypes.h:197
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition DerivedTypes.h:186
llvm::FunctionType::get
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition Function.h:169
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
llvm::Function::getIntrinsicID
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
llvm::Function::getFunction
const Function & getFunction() const
Definition Function.h:167
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition Function.cpp:451
llvm::Function::arg_size
size_t arg_size() const
Definition Function.h:886
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition Function.h:217
llvm::Function::getArg
Argument * getArg(unsigned i) const
Definition Function.h:871
llvm::GlobalValue::getGUIDAssumingExternalLinkage
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition GlobalValue.h:556
llvm::GlobalValue::GUID
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition GlobalValue.h:585
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition GlobalValue.h:577
llvm::GlobalValue::getValueType
Type * getValueType() const
Definition GlobalValue.h:302
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition GlobalVariable.h:155
llvm::GlobalVariable::hasInitializer
bool hasInitializer() const
Definitions have initializers, declarations don't.
Definition GlobalVariable.h:111
llvm::IRBuilderBase::getPtrTy
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Definition IRBuilder.h:577
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2910
llvm::InstVisitor
Base class for instruction visitors.
Definition InstVisitor.h:78
llvm::Instruction::isCast
bool isCast() const
Definition Instruction.h:321
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition Instruction.h:519
llvm::Instruction::getModule
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition Instruction.cpp:83
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition Instruction.h:317
llvm::Instruction::eraseFromParent
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition Instruction.cpp:109
llvm::Instruction::setMetadata
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition Metadata.cpp:1809
llvm::Instruction::getFastMathFlags
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition Instruction.cpp:704
llvm::Instruction::isUnaryOp
bool isUnaryOp() const
Definition Instruction.h:316
llvm::Instruction::copyMetadata
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition Instruction.cpp:1526
llvm::Instruction::getDataLayout
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition Instruction.cpp:91
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
llvm::LLVMContext::getOrInsertSyncScopeID
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
Definition LLVMContext.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MDBuilder::createRange
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
Definition MDBuilder.cpp:96
llvm::MDNode
Metadata node.
Definition Metadata.h:1069
llvm::MDNode::getOperand
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1426
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1567
llvm::MDNode::getNumOperands
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1432
llvm::MDNode::getContext
LLVMContext & getContext() const
Definition Metadata.h:1233
llvm::MDOperand
Tracking metadata reference owned by Metadata.
Definition Metadata.h:891
llvm::MDString
A single uniqued string.
Definition Metadata.h:722
llvm::MDString::getString
LLVM_ABI StringRef getString() const
Definition Metadata.cpp:633
llvm::MDString::get
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:615
llvm::MDTuple
Tuple of metadata.
Definition Metadata.h:1484
llvm::MDTuple::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1513
llvm::MetadataAsValue
Metadata wrapper in the Value hierarchy.
Definition Metadata.h:184
llvm::MetadataAsValue::get
static LLVM_ABI MetadataAsValue * get(LLVMContext &Context, Metadata *MD)
Definition Metadata.cpp:111
llvm::Metadata
Root of the metadata hierarchy.
Definition Metadata.h:64
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
llvm::Module::ModFlagBehavior
ModFlagBehavior
This enumeration defines the supported behaviors of module flags.
Definition Module.h:118
llvm::Module::Override
@ Override
Uses the specified value, regardless of the behavior or value of the other module.
Definition Module.h:139
llvm::Module::Error
@ Error
Emits an error if two values disagree, otherwise the resulting value is that of the operands.
Definition Module.h:121
llvm::Module::Min
@ Min
Takes the min of the two values, which are required to be integers.
Definition Module.h:153
llvm::Module::Max
@ Max
Takes the max of the two values, which are required to be integers.
Definition Module.h:150
llvm::NamedMDNode
A tuple of MDNodes.
Definition Metadata.h:1755
llvm::NamedMDNode::setOperand
LLVM_ABI void setOperand(unsigned I, MDNode *New)
Definition Metadata.cpp:1593
llvm::NamedMDNode::getOperand
LLVM_ABI MDNode * getOperand(unsigned i) const
Definition Metadata.cpp:1585
llvm::NamedMDNode::getNumOperands
LLVM_ABI unsigned getNumOperands() const
Definition Metadata.cpp:1581
llvm::NamedMDNode::clearOperands
LLVM_ABI void clearOperands()
Drop all references to this node's operands.
Definition Metadata.cpp:1600
llvm::NamedMDNode::operands
iterator_range< op_iterator > operands()
Definition Metadata.h:1851
llvm::NamedMDNode::addOperand
LLVM_ABI void addOperand(MDNode *M)
Definition Metadata.cpp:1591
llvm::OperandBundleDefT::inputs
ArrayRef< InputTy > inputs() const
Definition InstrTypes.h:1149
llvm::OperandBundleDefT::getTag
StringRef getTag() const
Definition InstrTypes.h:1157
llvm::PoisonValue::get
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constants.cpp:2034
llvm::Regex::match
LLVM_ABI bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition Regex.cpp:83
llvm::ScalableVectorType::get
static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)
Definition Type.cpp:889
llvm::ShuffleVectorInst::getShuffleMask
ArrayRef< int > getShuffleMask() const
Definition Instructions.h:2097
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallString
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:26
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:993
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition SmallVector.h:718
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition SmallVector.h:1252
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringLiteral
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
llvm::StringRef::npos
static constexpr size_t npos
Definition StringRef.h:58
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
llvm::StringRef::empty
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
llvm::StringRef::drop_front
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
Definition StringRef.h:635
llvm::StringRef::size
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
llvm::StringRef::trim
StringRef trim(char Char) const
Return string with consecutive Char characters starting from the left and right removed.
Definition StringRef.h:850
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition StringSwitch.h:47
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition StringSwitch.h:69
llvm::StringSwitch::StartsWith
StringSwitch & StartsWith(StringLiteral S, T Value)
Definition StringSwitch.h:81
llvm::StringSwitch::Cases
StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)
Definition StringSwitch.h:88
llvm::StructType
Class to represent struct types.
Definition DerivedTypes.h:278
llvm::StructType::get
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:477
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition DerivedTypes.h:428
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition DerivedTypes.h:429
llvm::TimeTraceScope
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
Definition TimeProfiler.h:182
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
llvm::Type::getInt64Ty
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:310
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:288
llvm::Type::getInt32Ty
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:309
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
llvm::Type::isBFloatTy
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
llvm::Type::getPointerAddressSpace
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition DerivedTypes.h:823
llvm::Type::getInt8Ty
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:307
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:368
llvm::Type::getPrimitiveSizeInBits
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:197
llvm::Type::getInt16Ty
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:308
llvm::Type::getScalarSizeInBits
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:232
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:285
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:257
llvm::Type::isFPOrFPVectorTy
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:227
llvm::Type::getFloatTy
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:286
llvm::Type::getBFloatTy
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:285
llvm::Type::getHalfTy
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:284
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition User.h:207
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition User.h:229
llvm::Value
LLVM Value Representation.
Definition Value.h:75
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:255
llvm::Value::print
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition AsmWriter.cpp:5333
llvm::Value::setName
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:394
llvm::Value::replaceAllUsesWith
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:258
llvm::Value::users
iterator_range< user_iterator > users()
Definition Value.h:426
llvm::Value::stripPointerCasts
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition Value.cpp:713
llvm::Value::use_empty
bool use_empty() const
Definition Value.h:346
llvm::Value::hasName
bool hasName() const
Definition Value.h:261
llvm::Value::getName
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
llvm::Value::takeName
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
llvm::VectorType
Base class of all SIMD vector types.
Definition DerivedTypes.h:490
llvm::VectorType::getInteger
static VectorType * getInteger(VectorType *VTy)
This static method gets a VectorType with the same number of elements as the input type,...
Definition DerivedTypes.h:541
llvm::VectorType::get
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition ilist_node.h:34
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition raw_ostream.h:694
llvm::raw_svector_ostream::str
StringRef str() const
Return a StringRef for the vector contents.
Definition raw_ostream.h:723
uint32_t
uint8_t
Call
CallInst * Call
Definition ObjCARCOpts.cpp:2458
Changed
Changed
Definition ObjCARCOpts.cpp:2468
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AMDGPUAS::LOCAL_ADDRESS
@ LOCAL_ADDRESS
Address space for local memory.
Definition AMDGPUAddrSpace.h:36
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:38
llvm::ARM_MB::ST
@ ST
Definition ARMBaseInfo.h:73
llvm::CallingConv::PTX_Kernel
@ PTX_Kernel
Call to a PTX kernel. Passes all arguments in parameter space.
Definition CallingConv.h:125
llvm::FloatABI::parseABIType
std::optional< ABIType > parseABIType(StringRef S)
Parse the string spelling used by the "float-abi" IR module flag into an ABIType.
Definition CodeGen.h:117
llvm::Intrinsic::remangleIntrinsicFunction
LLVM_ABI std::optional< Function * > remangleIntrinsicFunction(Function *F)
Definition Intrinsics.cpp:1423
llvm::Intrinsic::getOrInsertDeclaration
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
Definition Intrinsics.cpp:801
llvm::Intrinsic::lookupIntrinsicID
LLVM_ABI ID lookupIntrinsicID(StringRef Name)
This does the actual lookup of an intrinsic ID which matches the given function name.
Definition Intrinsics.cpp:757
llvm::Intrinsic::getAttributes
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
llvm::Intrinsic::isOverloaded
LLVM_ABI bool isOverloaded(ID id)
Returns true if the intrinsic can be overloaded.
Definition Intrinsics.cpp:649
llvm::Intrinsic::getType
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
Definition Intrinsics.cpp:636
llvm::Intrinsic::isSignatureValid
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
Definition Intrinsics.cpp:1404
llvm::Intrinsic::hasStructReturnType
LLVM_ABI bool hasStructReturnType(ID id)
Returns true if id has a struct return type.
Definition Intrinsics.cpp:1397
llvm::Intrinsic::getAllDefaultArgValues
LLVM_ABI std::pair< unsigned, ArrayRef< uint64_t > > getAllDefaultArgValues(ID IID)
Returns the first default argument index and an ArrayRef of all default values for the trailing param...
llvm::NVVMAttr::GridConstant
constexpr StringLiteral GridConstant("nvvm.grid_constant")
llvm::NVVMAttr::MaxNTID
constexpr StringLiteral MaxNTID("nvvm.maxntid")
llvm::NVVMAttr::MaxNReg
constexpr StringLiteral MaxNReg("nvvm.maxnreg")
llvm::NVVMAttr::MinCTASm
constexpr StringLiteral MinCTASm("nvvm.minctasm")
llvm::NVVMAttr::ReqNTID
constexpr StringLiteral ReqNTID("nvvm.reqntid")
llvm::NVVMAttr::MaxClusterRank
constexpr StringLiteral MaxClusterRank("nvvm.maxclusterrank")
llvm::NVVMAttr::ClusterDim
constexpr StringLiteral ClusterDim("nvvm.cluster_dim")
llvm::mdconst::dyn_extract_or_null
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
Definition Metadata.h:709
llvm::mdconst::hasa
std::enable_if_t< detail::IsValidPointer< X, Y >::value, bool > hasa(Y &&MD)
Check whether Metadata has a Value.
Definition Metadata.h:651
llvm::mdconst::dyn_extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:696
llvm::mdconst::extract
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:668
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
Definition MicrosoftDemangle.h:132
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::Length
@ Length
Definition DWP.cpp:577
llvm::UpgradeIntrinsicCall
LLVM_ABI void UpgradeIntrinsicCall(CallBase *CB, Function *NewFn)
This is the complement to the above, replacing a specific call to an intrinsic function with a call t...
Definition AutoUpgrade.cpp:5537
llvm::UpgradeSectionAttributes
LLVM_ABI void UpgradeSectionAttributes(Module &M)
Definition AutoUpgrade.cpp:7175
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1669
llvm::UpgradeInlineAsmString
LLVM_ABI void UpgradeInlineAsmString(std::string *AsmStr)
Upgrade comment in call to inline asm that represents an objc retain release marker.
Definition AutoUpgrade.cpp:3058
llvm::isValidAtomicOrdering
bool isValidAtomicOrdering(Int I)
Definition AtomicOrdering.h:75
llvm::dyn_cast
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
llvm::VectorInstrContext::Load
@ Load
The value being inserted comes from a load (InsertElement only).
Definition TargetTransformInfo.h:191
llvm::getLongDoubleFormatName
StringRef getLongDoubleFormatName(LongDoubleFormat Format)
Returns the IR floating-point type name for a LongDoubleFormat.
Definition CodeGen.h:76
llvm::LongDoubleFormat
LongDoubleFormat
The floating-point format used for the target's "long double" type.
Definition CodeGen.h:67
llvm::UpgradeIntrinsicFunction
LLVM_ABI bool UpgradeIntrinsicFunction(Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords=true)
This is a more granular function that simply checks an intrinsic function for upgrading,...
Definition AutoUpgrade.cpp:2271
llvm::upgradeInstructionLoopAttachment
LLVM_ABI MDNode * upgradeInstructionLoopAttachment(MDNode &N)
Upgrade the loop attachment metadata node.
Definition AutoUpgrade.cpp:7544
llvm::dyn_cast_if_present
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
llvm::UpgradeAttributes
LLVM_ABI void UpgradeAttributes(AttrBuilder &B)
Upgrade attributes that changed format or kind.
Definition AutoUpgrade.cpp:7716
llvm::UpgradeCallsToIntrinsic
LLVM_ABI void UpgradeCallsToIntrinsic(Function *F)
This is an auto-upgrade hook for any old intrinsic function syntaxes which need to have both the func...
Definition AutoUpgrade.cpp:6425
llvm::UpgradeNVVMAnnotations
LLVM_ABI void UpgradeNVVMAnnotations(Module &M)
Convert legacy nvvm.annotations metadata to appropriate function attributes.
Definition AutoUpgrade.cpp:6664
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:633
llvm::UpgradeModuleFlags
LLVM_ABI bool UpgradeModuleFlags(Module &M)
This checks for module flags which should be upgraded.
Definition AutoUpgrade.cpp:6959
llvm::utostr
std::string utostr(uint64_t X, bool isNeg=false)
Definition StringExtras.h:322
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
llvm::UpgradeCFIFunctionsMetadata
LLVM_ABI bool UpgradeCFIFunctionsMetadata(Module &M)
Upgrade the cfi.functions metadata node by calculating and inserting the GUID for each function entry...
Definition AutoUpgrade.cpp:7131
llvm::copyModuleAttrToFunctions
LLVM_ABI void copyModuleAttrToFunctions(Module &M)
Copies module attributes to the functions in the module.
Definition AutoUpgrade.cpp:7374
llvm::UpgradeOperandBundles
LLVM_ABI void UpgradeOperandBundles(std::vector< OperandBundleDef > &OperandBundles)
Upgrade operand bundles (without knowing about their user instruction).
Definition AutoUpgrade.cpp:7754
llvm::UpgradeBitCastExpr
LLVM_ABI Constant * UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy)
This is an auto-upgrade for bitcast constant expression between pointers with different address space...
Definition AutoUpgrade.cpp:6492
llvm::dyn_cast_or_null
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::UpgradeDataLayoutString
LLVM_ABI std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple)
Upgrade the datalayout string by adding a section for address space pointers.
Definition AutoUpgrade.cpp:7576
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1753
llvm::report_fatal_error
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
llvm::isa
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
llvm::UpgradeGlobalVariable
LLVM_ABI GlobalVariable * UpgradeGlobalVariable(GlobalVariable *GV)
This checks for global variables which should be upgraded.
Definition AutoUpgrade.cpp:2290
llvm::errs
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition raw_ostream.cpp:890
llvm::StripDebugInfo
LLVM_ABI bool StripDebugInfo(Module &M)
Strip debug info in the module if it exists.
Definition DebugInfo.cpp:711
llvm::drop_end
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:322
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition AtomicOrdering.h:56
llvm::ModRefInfo::Ref
@ Ref
The access may reference the value stored in memory.
Definition ModRef.h:32
llvm::join
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
Definition StringExtras.h:492
llvm::findBooleanLoopTags
const BooleanLoopTags * findBooleanLoopTags(StringRef Name)
Return the replacement tags for the enable tag Name, or nullptr.
Definition AutoUpgrade.h:129
llvm::OperandBundleDef
OperandBundleDefT< Value * > OperandBundleDef
Definition AutoUpgrade.h:34
llvm::UpgradeBitCastInst
LLVM_ABI Instruction * UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, Instruction *&Temp)
This is an auto-upgrade for bitcast between pointers with different address spaces: the instruction i...
Definition AutoUpgrade.cpp:6470
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::RoundingMode::Dynamic
@ Dynamic
Denotes mode unknown at compile time.
Definition FloatingPointMode.h:47
llvm::ArrayRef
ArrayRef(const T &OneElt) -> ArrayRef< T >
llvm::parseDenormalFPAttribute
DenormalMode parseDenormalFPAttribute(StringRef Str)
Returns the denormal mode to use for inputs and outputs.
Definition FloatingPointMode.h:254
llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1772
llvm::erase_if
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2192
llvm::UpgradeDebugInfo
LLVM_ABI bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Definition AutoUpgrade.cpp:6522
llvm::UpgradeFunctionAttributes
LLVM_ABI void UpgradeFunctionAttributes(Function &F)
Correct any IR that is relying on old function attribute behavior.
Definition AutoUpgrade.cpp:7248
llvm::UpgradeTBAANode
LLVM_ABI MDNode * UpgradeTBAANode(MDNode &TBAANode)
If the given TBAA tag uses the scalar TBAA format, create a new node corresponding to the upgrade to ...
Definition AutoUpgrade.cpp:6444
llvm::UpgradeARCRuntime
LLVM_ABI void UpgradeARCRuntime(Module &M)
Convert calls to ARC runtime functions to intrinsic calls and upgrade the old retain release marker t...
Definition AutoUpgrade.cpp:6729
llvm::DEBUG_METADATA_VERSION
@ DEBUG_METADATA_VERSION
Definition Metadata.h:54
llvm::ValueUniformity::Default
@ Default
The result value is uniform if and only if all operands are uniform.
Definition Uniformity.h:20
llvm::verifyModule
LLVM_ABI bool verifyModule(const Module &M, raw_ostream *OS=nullptr, bool *BrokenDebugInfo=nullptr)
Check a module for errors.
Definition Verifier.cpp:7811
llvm::reportFatalUsageError
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
N
#define N
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
llvm::BooleanLoopTags
Single-operand tags replacing a removed two-operand form !
Definition AutoUpgrade.h:115
llvm::BooleanLoopTags::Disable
StringLiteral Disable
Definition AutoUpgrade.h:117
llvm::BooleanLoopTags::Enable
StringLiteral Enable
Definition AutoUpgrade.h:116
llvm::DenormalFPEnv
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Definition FloatingPointMode.h:279
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getInvalid
static constexpr DenormalMode getInvalid()
Definition FloatingPointMode.h:106
llvm::DenormalMode::isValid
constexpr bool isValid() const
Definition FloatingPointMode.h:156
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:106
Generated on for LLVM by doxygen 1.14.0