Bug Summary

File:llvm/include/llvm/IR/IRBuilder.h
Warning:line 2551, column 23
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name X86LowerAMXType.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/X86 -I /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Target/X86 -I include -I /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-01-16-232930-107970-1 -x c++ /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Target/X86/X86LowerAMXType.cpp

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Target/X86/X86LowerAMXType.cpp

1//===- Target/X86/X86LowerAMXType.cpp - -------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to transform <256 x i32> load/store
10/// <256 x i32> is bitcasted to x86_amx on X86, and AMX instruction set only
11/// provides simple operation on x86_amx. The basic elementwise operation
12/// is not supported by AMX. Since x86_amx is bitcasted from vector <256 x i32>
13/// and only AMX intrinsics can operate on the type, we need transform
14/// load/store <256 x i32> instruction to AMX load/store. If the bitcast can
15/// not be combined with load/store, we transform the bitcast to amx load/store
16/// and <256 x i32> store/load.
17///
18/// If Front End not use O0 but the Mid/Back end use O0, (e.g. "Clang -O2 -S
19/// -emit-llvm t.c" + "llc t.ll") we should make sure the amx data is volatile,
20/// because that is necessary for AMX fast register allocation. (In Fast
21/// registera allocation, register will be allocated before spill/reload, so
22/// there is no additional register for amx to identify the step in spill.)
23/// The volatileTileData() will handle this case.
24/// e.g.
25/// ----------------------------------------------------------
26/// | def %td = ... |
27/// | ... |
28/// | "use %td" |
29/// ----------------------------------------------------------
30/// will transfer to -->
31/// ----------------------------------------------------------
32/// | def %td = ... |
33/// | call void @llvm.x86.tilestored64.internal(mem, %td) |
34/// | ... |
35/// | %td2 = call x86_amx @llvm.x86.tileloadd64.internal(mem)|
36/// | "use %td2" |
37/// ----------------------------------------------------------
38//
39//===----------------------------------------------------------------------===//
40//
41#include "X86.h"
42#include "llvm/ADT/PostOrderIterator.h"
43#include "llvm/ADT/SetVector.h"
44#include "llvm/ADT/SmallSet.h"
45#include "llvm/Analysis/OptimizationRemarkEmitter.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/CodeGen/Passes.h"
49#include "llvm/CodeGen/TargetPassConfig.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/IR/DataLayout.h"
52#include "llvm/IR/Function.h"
53#include "llvm/IR/IRBuilder.h"
54#include "llvm/IR/Instructions.h"
55#include "llvm/IR/IntrinsicInst.h"
56#include "llvm/IR/IntrinsicsX86.h"
57#include "llvm/IR/PatternMatch.h"
58#include "llvm/InitializePasses.h"
59#include "llvm/Pass.h"
60#include "llvm/Target/TargetMachine.h"
61#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
62#include "llvm/Transforms/Utils/Local.h"
63
64using namespace llvm;
65using namespace PatternMatch;
66
67#define DEBUG_TYPE"lower-amx-type" "lower-amx-type"
68
69static bool isAMXCast(Instruction *II) {
70 return match(II,
71 m_Intrinsic<Intrinsic::x86_cast_vector_to_tile>(m_Value())) ||
72 match(II, m_Intrinsic<Intrinsic::x86_cast_tile_to_vector>(m_Value()));
73}
74
75static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB,
76 Type *Ty) {
77 Function &F = *BB->getParent();
78 Module *M = BB->getModule();
79 const DataLayout &DL = M->getDataLayout();
80
81 LLVMContext &Ctx = Builder.getContext();
82 auto AllocaAlignment = DL.getPrefTypeAlign(Type::getX86_AMXTy(Ctx));
83 unsigned AllocaAS = DL.getAllocaAddrSpace();
84 AllocaInst *AllocaRes =
85 new AllocaInst(Ty, AllocaAS, "", &F.getEntryBlock().front());
86 AllocaRes->setAlignment(AllocaAlignment);
87 return AllocaRes;
88}
89
90static Instruction *getFirstNonAllocaInTheEntryBlock(Function &F) {
91 for (Instruction &I : F.getEntryBlock())
92 if (!isa<AllocaInst>(&I))
93 return &I;
94 llvm_unreachable("No terminator in the entry block!")::llvm::llvm_unreachable_internal("No terminator in the entry block!"
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 94)
;
95}
96
97static std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
98 IRBuilder<> Builder(II);
99 Value *Row = nullptr, *Col = nullptr;
100 switch (II->getIntrinsicID()) {
101 default:
102 llvm_unreachable("Expect amx intrinsics")::llvm::llvm_unreachable_internal("Expect amx intrinsics", "llvm/lib/Target/X86/X86LowerAMXType.cpp"
, 102)
;
103 case Intrinsic::x86_tileloadd64_internal:
104 case Intrinsic::x86_tileloaddt164_internal:
105 case Intrinsic::x86_tilestored64_internal: {
106 Row = II->getArgOperand(0);
107 Col = II->getArgOperand(1);
108 break;
109 }
110 // a * b + c
111 // The shape depends on which operand.
112 case Intrinsic::x86_tdpbssd_internal:
113 case Intrinsic::x86_tdpbsud_internal:
114 case Intrinsic::x86_tdpbusd_internal:
115 case Intrinsic::x86_tdpbuud_internal:
116 case Intrinsic::x86_tdpbf16ps_internal: {
117 switch (OpNo) {
118 case 3:
119 Row = II->getArgOperand(0);
120 Col = II->getArgOperand(1);
121 break;
122 case 4:
123 Row = II->getArgOperand(0);
124 Col = II->getArgOperand(2);
125 break;
126 case 5:
127 if (isa<ConstantInt>(II->getArgOperand(2)))
128 Row = Builder.getInt16(
129 (cast<ConstantInt>(II->getOperand(2))->getSExtValue()) / 4);
130 else if (isa<Instruction>(II->getArgOperand(2))) {
131 // When it is not a const value and it is not a function argument, we
132 // create Row after the definition of II->getOperand(2) instead of
133 // before II. For example, II is %118, we try to getshape for %117:
134 // %117 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x
135 // i32> %115).
136 // %118 = call x86_amx @llvm.x86.tdpbf16ps.internal(i16
137 // %104, i16 %105, i16 %106, x86_amx %110, x86_amx %114, x86_amx
138 // %117).
139 // If we create %row = udiv i16 %106, 4 before %118(aka. II), then its
140 // definition is after its user(new tileload for %117).
141 // So, the best choice is to create %row right after the definition of
142 // %106.
143 Builder.SetInsertPoint(cast<Instruction>(II->getOperand(2)));
144 Row = Builder.CreateUDiv(II->getOperand(2), Builder.getInt16(4));
145 cast<Instruction>(Row)->moveAfter(cast<Instruction>(II->getOperand(2)));
146 } else {
147 // When it is not a const value and it is a function argument, we create
148 // Row at the entry bb.
149 IRBuilder<> NewBuilder(
150 getFirstNonAllocaInTheEntryBlock(*II->getFunction()));
151 Row = NewBuilder.CreateUDiv(II->getOperand(2), NewBuilder.getInt16(4));
152 }
153 Col = II->getArgOperand(1);
154 break;
155 }
156 break;
157 }
158 }
159
160 return std::make_pair(Row, Col);
161}
162
163namespace {
164class X86LowerAMXType {
165 Function &Func;
166
167 // In AMX intrinsics we let Shape = {Row, Col}, but the
168 // RealCol = Col / ElementSize. We may use the RealCol
169 // as a new Row for other new created AMX intrinsics.
170 std::map<Value *, Value *> Col2Row;
171
172public:
173 X86LowerAMXType(Function &F) : Func(F) {}
174 bool visit();
175 void combineLoadBitcast(LoadInst *LD, BitCastInst *Bitcast);
176 void combineBitcastStore(BitCastInst *Bitcast, StoreInst *ST);
177 bool transformBitcast(BitCastInst *Bitcast);
178};
179
180// %src = load <256 x i32>, <256 x i32>* %addr, align 64
181// %2 = bitcast <256 x i32> %src to x86_amx
182// -->
183// %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
184// i8* %addr, i64 %stride64)
185void X86LowerAMXType::combineLoadBitcast(LoadInst *LD, BitCastInst *Bitcast) {
186 Value *Row = nullptr, *Col = nullptr;
187 Use &U = *(Bitcast->use_begin());
188 unsigned OpNo = U.getOperandNo();
189 auto *II = cast<IntrinsicInst>(U.getUser());
190 std::tie(Row, Col) = getShape(II, OpNo);
191 IRBuilder<> Builder(Bitcast);
192 // Use the maximun column as stride.
193 Value *Stride = Builder.getInt64(64);
194 Value *I8Ptr =
195 Builder.CreateBitCast(LD->getOperand(0), Builder.getInt8PtrTy());
196 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
197
198 Value *NewInst =
199 Builder.CreateIntrinsic(Intrinsic::x86_tileloadd64_internal, None, Args);
200 Bitcast->replaceAllUsesWith(NewInst);
201}
202
203// %src = call x86_amx @llvm.x86.tileloadd64.internal(%row, %col, %addr,
204// %stride);
205// %13 = bitcast x86_amx %src to <256 x i32>
206// store <256 x i32> %13, <256 x i32>* %addr, align 64
207// -->
208// call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
209// %stride64, %13)
210void X86LowerAMXType::combineBitcastStore(BitCastInst *Bitcast, StoreInst *ST) {
211
212 Value *Tile = Bitcast->getOperand(0);
213 auto *II = cast<IntrinsicInst>(Tile);
214 // Tile is output from AMX intrinsic. The first operand of the
215 // intrinsic is row, the second operand of the intrinsic is column.
216 Value *Row = II->getOperand(0);
217 Value *Col = II->getOperand(1);
218 IRBuilder<> Builder(ST);
219 // Use the maximum column as stride. It must be the same with load
220 // stride.
221 Value *Stride = Builder.getInt64(64);
222 Value *I8Ptr =
223 Builder.CreateBitCast(ST->getOperand(1), Builder.getInt8PtrTy());
224 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Tile};
225 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
226 if (Bitcast->hasOneUse())
227 return;
228 // %13 = bitcast x86_amx %src to <256 x i32>
229 // store <256 x i32> %13, <256 x i32>* %addr, align 64
230 // %add = <256 x i32> %13, <256 x i32> %src2
231 // -->
232 // %13 = bitcast x86_amx %src to <256 x i32>
233 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
234 // %stride64, %13)
235 // %14 = load <256 x i32>, %addr
236 // %add = <256 x i32> %14, <256 x i32> %src2
237 Value *Vec = Builder.CreateLoad(Bitcast->getType(), ST->getOperand(1));
238 Bitcast->replaceAllUsesWith(Vec);
239}
240
241// transform bitcast to <store, load> instructions.
242bool X86LowerAMXType::transformBitcast(BitCastInst *Bitcast) {
243 IRBuilder<> Builder(Bitcast);
244 AllocaInst *AllocaAddr;
245 Value *I8Ptr, *Stride;
246 auto *Src = Bitcast->getOperand(0);
247
248 auto Prepare = [&](Type *MemTy) {
249 AllocaAddr = createAllocaInstAtEntry(Builder, Bitcast->getParent(), MemTy);
250 I8Ptr = Builder.CreateBitCast(AllocaAddr, Builder.getInt8PtrTy());
251 Stride = Builder.getInt64(64);
252 };
253
254 if (Bitcast->getType()->isX86_AMXTy()) {
255 // %2 = bitcast <256 x i32> %src to x86_amx
256 // -->
257 // %addr = alloca <256 x i32>, align 64
258 // store <256 x i32> %src, <256 x i32>* %addr, align 64
259 // %addr2 = bitcast <256 x i32>* to i8*
260 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
261 // i8* %addr2,
262 // i64 64)
263 Use &U = *(Bitcast->use_begin());
264 unsigned OpNo = U.getOperandNo();
265 auto *II = dyn_cast<IntrinsicInst>(U.getUser());
266 if (!II)
267 return false; // May be bitcast from x86amx to <256 x i32>.
268 Prepare(Bitcast->getOperand(0)->getType());
269 Builder.CreateStore(Src, AllocaAddr);
270 // TODO we can pick an constant operand for the shape.
271 Value *Row = nullptr, *Col = nullptr;
272 std::tie(Row, Col) = getShape(II, OpNo);
273 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
274 Value *NewInst = Builder.CreateIntrinsic(
275 Intrinsic::x86_tileloadd64_internal, None, Args);
276 Bitcast->replaceAllUsesWith(NewInst);
277 } else {
278 // %2 = bitcast x86_amx %src to <256 x i32>
279 // -->
280 // %addr = alloca <256 x i32>, align 64
281 // %addr2 = bitcast <256 x i32>* to i8*
282 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col,
283 // i8* %addr2, i64 %stride)
284 // %2 = load <256 x i32>, <256 x i32>* %addr, align 64
285 auto *II = dyn_cast<IntrinsicInst>(Src);
286 if (!II)
287 return false; // May be bitcast from <256 x i32> to x86amx.
288 Prepare(Bitcast->getType());
289 Value *Row = II->getOperand(0);
290 Value *Col = II->getOperand(1);
291 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Src};
292 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
293 Value *NewInst = Builder.CreateLoad(Bitcast->getType(), AllocaAddr);
294 Bitcast->replaceAllUsesWith(NewInst);
295 }
296
297 return true;
298}
299
300bool X86LowerAMXType::visit() {
301 SmallVector<Instruction *, 8> DeadInsts;
302 Col2Row.clear();
303
304 for (BasicBlock *BB : post_order(&Func)) {
305 for (Instruction &Inst : llvm::make_early_inc_range(llvm::reverse(*BB))) {
306 auto *Bitcast = dyn_cast<BitCastInst>(&Inst);
307 if (!Bitcast)
308 continue;
309
310 Value *Src = Bitcast->getOperand(0);
311 if (Bitcast->getType()->isX86_AMXTy()) {
312 if (Bitcast->user_empty()) {
313 DeadInsts.push_back(Bitcast);
314 continue;
315 }
316 LoadInst *LD = dyn_cast<LoadInst>(Src);
317 if (!LD) {
318 if (transformBitcast(Bitcast))
319 DeadInsts.push_back(Bitcast);
320 continue;
321 }
322 // If load has mutli-user, duplicate a vector load.
323 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
324 // %2 = bitcast <256 x i32> %src to x86_amx
325 // %add = add <256 x i32> %src, <256 x i32> %src2
326 // -->
327 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
328 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
329 // i8* %addr, i64 %stride64)
330 // %add = add <256 x i32> %src, <256 x i32> %src2
331
332 // If load has one user, the load will be eliminated in DAG ISel.
333 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
334 // %2 = bitcast <256 x i32> %src to x86_amx
335 // -->
336 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
337 // i8* %addr, i64 %stride64)
338 combineLoadBitcast(LD, Bitcast);
339 DeadInsts.push_back(Bitcast);
340 if (LD->hasOneUse())
341 DeadInsts.push_back(LD);
342 } else if (Src->getType()->isX86_AMXTy()) {
343 if (Bitcast->user_empty()) {
344 DeadInsts.push_back(Bitcast);
345 continue;
346 }
347 StoreInst *ST = nullptr;
348 for (Use &U : Bitcast->uses()) {
349 ST = dyn_cast<StoreInst>(U.getUser());
350 if (ST)
351 break;
352 }
353 if (!ST) {
354 if (transformBitcast(Bitcast))
355 DeadInsts.push_back(Bitcast);
356 continue;
357 }
358 // If bitcast (%13) has one use, combine bitcast and store to amx store.
359 // %src = call x86_amx @llvm.x86.tileloadd64.internal(%row, %col, %addr,
360 // %stride);
361 // %13 = bitcast x86_amx %src to <256 x i32>
362 // store <256 x i32> %13, <256 x i32>* %addr, align 64
363 // -->
364 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
365 // %stride64, %13)
366 //
367 // If bitcast (%13) has multi-use, transform as below.
368 // %13 = bitcast x86_amx %src to <256 x i32>
369 // store <256 x i32> %13, <256 x i32>* %addr, align 64
370 // %add = <256 x i32> %13, <256 x i32> %src2
371 // -->
372 // %13 = bitcast x86_amx %src to <256 x i32>
373 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
374 // %stride64, %13)
375 // %14 = load <256 x i32>, %addr
376 // %add = <256 x i32> %14, <256 x i32> %src2
377 //
378 combineBitcastStore(Bitcast, ST);
379 // Delete user first.
380 DeadInsts.push_back(ST);
381 DeadInsts.push_back(Bitcast);
382 }
383 }
384 }
385
386 bool C = !DeadInsts.empty();
387
388 for (auto *Inst : DeadInsts)
389 Inst->eraseFromParent();
390
391 return C;
392}
393} // anonymous namespace
394
395static Value *getAllocaPos(BasicBlock *BB) {
396 Module *M = BB->getModule();
397 Function *F = BB->getParent();
398 IRBuilder<> Builder(&F->getEntryBlock().front());
399 const DataLayout &DL = M->getDataLayout();
400 unsigned AllocaAS = DL.getAllocaAddrSpace();
401 Type *V256I32Ty = VectorType::get(Builder.getInt32Ty(), 256, false);
402 AllocaInst *AllocaRes =
403 new AllocaInst(V256I32Ty, AllocaAS, "", &F->getEntryBlock().front());
404 BasicBlock::iterator Iter = AllocaRes->getIterator();
405 ++Iter;
406 Builder.SetInsertPoint(&*Iter);
407 Value *I8Ptr = Builder.CreateBitCast(AllocaRes, Builder.getInt8PtrTy());
408 return I8Ptr;
409}
410
411static Instruction *createTileStore(Instruction *TileDef, Value *Ptr) {
412 assert(TileDef->getType()->isX86_AMXTy() && "Not define tile!")(static_cast <bool> (TileDef->getType()->isX86_AMXTy
() && "Not define tile!") ? void (0) : __assert_fail (
"TileDef->getType()->isX86_AMXTy() && \"Not define tile!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 412, __extension__
__PRETTY_FUNCTION__))
;
413 auto *II = cast<IntrinsicInst>(TileDef);
414 assert(II && "Not tile intrinsic!")(static_cast <bool> (II && "Not tile intrinsic!"
) ? void (0) : __assert_fail ("II && \"Not tile intrinsic!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 414, __extension__
__PRETTY_FUNCTION__))
;
415 Value *Row = II->getOperand(0);
416 Value *Col = II->getOperand(1);
417
418 BasicBlock *BB = TileDef->getParent();
419 BasicBlock::iterator Iter = TileDef->getIterator();
420 IRBuilder<> Builder(BB, ++Iter);
421 Value *Stride = Builder.getInt64(64);
422 std::array<Value *, 5> Args = {Row, Col, Ptr, Stride, TileDef};
423
424 Instruction *TileStore =
425 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
426 return TileStore;
427}
428
429static void replaceWithTileLoad(Use &U, Value *Ptr, bool IsPHI = false) {
430 Value *V = U.get();
431 assert(V->getType()->isX86_AMXTy() && "Not define tile!")(static_cast <bool> (V->getType()->isX86_AMXTy() &&
"Not define tile!") ? void (0) : __assert_fail ("V->getType()->isX86_AMXTy() && \"Not define tile!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 431, __extension__
__PRETTY_FUNCTION__))
;
14
'?' condition is true
432
433 // Get tile shape.
434 IntrinsicInst *II = nullptr;
435 if (IsPHI
14.1
'IsPHI' is true
14.1
'IsPHI' is true
) {
15
Taking true branch
436 Value *PhiOp = dyn_cast<PHINode>(V)->getIncomingValue(0);
16
Assuming 'V' is a 'PHINode'
437 II = cast<IntrinsicInst>(PhiOp);
17
'PhiOp' is a 'IntrinsicInst'
438 } else {
439 II = cast<IntrinsicInst>(V);
440 }
441 Value *Row = II->getOperand(0);
442 Value *Col = II->getOperand(1);
443
444 Instruction *UserI = dyn_cast<Instruction>(U.getUser());
18
Assuming the object is not a 'Instruction'
19
'UserI' initialized to a null pointer value
445 IRBuilder<> Builder(UserI);
20
Passing null pointer value via 1st parameter 'IP'
21
Calling constructor for 'IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>'
446 Value *Stride = Builder.getInt64(64);
447 std::array<Value *, 4> Args = {Row, Col, Ptr, Stride};
448
449 Value *TileLoad =
450 Builder.CreateIntrinsic(Intrinsic::x86_tileloadd64_internal, None, Args);
451 UserI->replaceUsesOfWith(V, TileLoad);
452}
453
454static bool isIncomingOfPHI(Instruction *I) {
455 for (Use &U : I->uses()) {
456 User *V = U.getUser();
457 if (isa<PHINode>(V))
458 return true;
459 }
460 return false;
461}
462
463// Let all AMX tile data become volatile data, shorten the life range
464// of each tile register before fast register allocation.
465namespace {
466class X86VolatileTileData {
467 Function &F;
468
469public:
470 X86VolatileTileData(Function &Func) : F(Func) {}
471 Value *updatePhiIncomings(BasicBlock *BB,
472 SmallVector<Instruction *, 2> &Incomings);
473 void replacePhiDefWithLoad(Instruction *PHI, Value *StorePtr);
474 bool volatileTileData();
475 void volatileTilePHI(PHINode *Inst);
476 void volatileTileNonPHI(Instruction *I);
477};
478
479Value *X86VolatileTileData::updatePhiIncomings(
480 BasicBlock *BB, SmallVector<Instruction *, 2> &Incomings) {
481 Value *I8Ptr = getAllocaPos(BB);
482
483 for (auto *I : Incomings) {
484 User *Store = createTileStore(I, I8Ptr);
485
486 // All its uses (except phi) should load from stored mem.
487 for (Use &U : I->uses()) {
488 User *V = U.getUser();
489 if (isa<PHINode>(V) || V == Store)
490 continue;
491 replaceWithTileLoad(U, I8Ptr);
492 }
493 }
494 return I8Ptr;
495}
496
497void X86VolatileTileData::replacePhiDefWithLoad(Instruction *PHI,
498 Value *StorePtr) {
499 for (Use &U : PHI->uses())
500 replaceWithTileLoad(U, StorePtr, true);
13
Calling 'replaceWithTileLoad'
501 PHI->eraseFromParent();
502}
503
504// Smilar with volatileTileNonPHI, this function only handle PHI Nodes
505// and their related AMX intrinsics.
506// 1) PHI Def should change to tileload.
507// 2) PHI Incoming Values should tilestored in just after their def.
508// 3) The mem of these tileload and tilestores should be same.
509// e.g.
510// ------------------------------------------------------
511// bb_dom:
512// ...
513// br i1 %bool.cond, label %if.else, label %if.then
514//
515// if.then:
516// def %t0 = ...
517// ...
518// use %t0
519// ...
520// br label %if.end
521//
522// if.else:
523// def %t1 = ...
524// br label %if.end
525//
526// if.end:
527// %td = phi x86_amx [ %t1, %if.else ], [ %t0, %if.then ]
528// ...
529// use %td
530// ------------------------------------------------------
531// -->
532// ------------------------------------------------------
533// bb_entry:
534// %mem = alloca <256 x i32>, align 1024 *
535// ...
536// bb_dom:
537// ...
538// br i1 %bool.cond, label %if.else, label %if.then
539//
540// if.then:
541// def %t0 = ...
542// call void @llvm.x86.tilestored64.internal(mem, %t0) *
543// ...
544// %t0` = call x86_amx @llvm.x86.tileloadd64.internal(mem)*
545// use %t0` *
546// ...
547// br label %if.end
548//
549// if.else:
550// def %t1 = ...
551// call void @llvm.x86.tilestored64.internal(mem, %t1) *
552// br label %if.end
553//
554// if.end:
555// ...
556// %td = call x86_amx @llvm.x86.tileloadd64.internal(mem) *
557// use %td
558// ------------------------------------------------------
559void X86VolatileTileData::volatileTilePHI(PHINode *PHI) {
560 BasicBlock *BB = PHI->getParent();
561 SmallVector<Instruction *, 2> Incomings;
562
563 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) {
10
Assuming 'I' is equal to 'E'
11
Loop condition is false. Execution continues on line 570
564 Value *Op = PHI->getIncomingValue(I);
565 Instruction *Inst = dyn_cast<Instruction>(Op);
566 assert(Inst && "We shouldn't fold AMX instrution!")(static_cast <bool> (Inst && "We shouldn't fold AMX instrution!"
) ? void (0) : __assert_fail ("Inst && \"We shouldn't fold AMX instrution!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 566, __extension__
__PRETTY_FUNCTION__))
;
567 Incomings.push_back(Inst);
568 }
569
570 Value *StorePtr = updatePhiIncomings(BB, Incomings);
571 replacePhiDefWithLoad(PHI, StorePtr);
12
Calling 'X86VolatileTileData::replacePhiDefWithLoad'
572}
573
574// Store the defined tile and load it before use.
575// All its users are not PHI.
576// e.g.
577// ------------------------------------------------------
578// def %td = ...
579// ...
580// "use %td"
581// ------------------------------------------------------
582// -->
583// ------------------------------------------------------
584// def %td = ...
585// call void @llvm.x86.tilestored64.internal(mem, %td)
586// ...
587// %td2 = call x86_amx @llvm.x86.tileloadd64.internal(mem)
588// "use %td2"
589// ------------------------------------------------------
590void X86VolatileTileData::volatileTileNonPHI(Instruction *I) {
591 BasicBlock *BB = I->getParent();
592 Value *I8Ptr = getAllocaPos(BB);
593 User *Store = createTileStore(I, I8Ptr);
594
595 // All its uses should load from stored mem.
596 for (Use &U : I->uses()) {
597 User *V = U.getUser();
598 assert(!isa<PHINode>(V) && "PHI Nodes should be excluded!")(static_cast <bool> (!isa<PHINode>(V) && "PHI Nodes should be excluded!"
) ? void (0) : __assert_fail ("!isa<PHINode>(V) && \"PHI Nodes should be excluded!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 598, __extension__
__PRETTY_FUNCTION__))
;
599 if (V != Store)
600 replaceWithTileLoad(U, I8Ptr);
601 }
602}
603
604// Volatile Tile Model:
605// 1) All the uses of tile data comes from tileload in time.
606// 2) All the defs of tile data tilestore into mem immediately.
607// For example:
608// --------------------------------------------------------------------------
609// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...) key
610// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)
611// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...) amx
612// %td = tail call x86_amx @llvm.x86.tdpbssd.internal(m, n, k, t1, t2, t3)
613// call void @llvm.x86.tilestored64.internal(... td) area
614// --------------------------------------------------------------------------
615// 3) No terminator, call or other amx instructions in the key amx area.
616bool X86VolatileTileData::volatileTileData() {
617 bool Changed = false;
618 for (BasicBlock &BB : F) {
619 SmallVector<Instruction *, 2> PHIInsts;
620 SmallVector<Instruction *, 8> AMXDefInsts;
621
622 for (Instruction &I : BB) {
623 if (!I.getType()->isX86_AMXTy())
624 continue;
625 if (isa<PHINode>(&I))
626 PHIInsts.push_back(&I);
627 else
628 AMXDefInsts.push_back(&I);
629 }
630
631 // First we "volatile" the non-phi related amx intrinsics.
632 for (Instruction *I : AMXDefInsts) {
6
Assuming '__begin2' is equal to '__end2'
633 if (isIncomingOfPHI(I))
634 continue;
635 volatileTileNonPHI(I);
636 Changed = true;
637 }
638
639 for (Instruction *I : PHIInsts) {
7
Assuming '__begin2' is not equal to '__end2'
640 volatileTilePHI(dyn_cast<PHINode>(I));
8
Assuming 'I' is a 'PHINode'
9
Calling 'X86VolatileTileData::volatileTilePHI'
641 Changed = true;
642 }
643 }
644 return Changed;
645}
646
647} // anonymous namespace
648
649namespace {
650
651class X86LowerAMXCast {
652 Function &Func;
653
654public:
655 X86LowerAMXCast(Function &F) : Func(F) {}
656 bool combineAMXcast(TargetLibraryInfo *TLI);
657 bool transformAMXCast(IntrinsicInst *AMXCast);
658 bool transformAllAMXCast();
659 bool optimizeAMXCastFromPhi(IntrinsicInst *CI, PHINode *PN,
660 SmallSetVector<Instruction *, 16> &DeadInst);
661};
662
663static bool DCEInstruction(Instruction *I,
664 SmallSetVector<Instruction *, 16> &WorkList,
665 const TargetLibraryInfo *TLI) {
666 if (isInstructionTriviallyDead(I, TLI)) {
667 salvageDebugInfo(*I);
668 salvageKnowledge(I);
669
670 // Null out all of the instruction's operands to see if any operand becomes
671 // dead as we go.
672 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
673 Value *OpV = I->getOperand(i);
674 I->setOperand(i, nullptr);
675
676 if (!OpV->use_empty() || I == OpV)
677 continue;
678
679 // If the operand is an instruction that became dead as we nulled out the
680 // operand, and if it is 'trivially' dead, delete it in a future loop
681 // iteration.
682 if (Instruction *OpI = dyn_cast<Instruction>(OpV)) {
683 if (isInstructionTriviallyDead(OpI, TLI)) {
684 WorkList.insert(OpI);
685 }
686 }
687 }
688 I->eraseFromParent();
689 return true;
690 }
691 return false;
692}
693
694/// This function handles following case
695///
696/// A -> B amxcast
697/// PHI
698/// B -> A amxcast
699///
700/// All the related PHI nodes can be replaced by new PHI nodes with type A.
701/// The uses of \p CI can be changed to the new PHI node corresponding to \p PN.
702bool X86LowerAMXCast::optimizeAMXCastFromPhi(
703 IntrinsicInst *CI, PHINode *PN,
704 SmallSetVector<Instruction *, 16> &DeadInst) {
705 IRBuilder<> Builder(CI);
706 Value *Src = CI->getOperand(0);
707 Type *SrcTy = Src->getType(); // Type B
708 Type *DestTy = CI->getType(); // Type A
709
710 SmallVector<PHINode *, 4> PhiWorklist;
711 SmallSetVector<PHINode *, 4> OldPhiNodes;
712
713 // Find all of the A->B casts and PHI nodes.
714 // We need to inspect all related PHI nodes, but PHIs can be cyclic, so
715 // OldPhiNodes is used to track all known PHI nodes, before adding a new
716 // PHI to PhiWorklist, it is checked against and added to OldPhiNodes first.
717 PhiWorklist.push_back(PN);
718 OldPhiNodes.insert(PN);
719 while (!PhiWorklist.empty()) {
720 auto *OldPN = PhiWorklist.pop_back_val();
721 for (Value *IncValue : OldPN->incoming_values()) {
722 // TODO: currently, We ignore cases where it is a const. In the future, we
723 // might support const.
724 if (isa<Constant>(IncValue))
725 return false;
726
727 if (auto *PNode = dyn_cast<PHINode>(IncValue)) {
728 if (OldPhiNodes.insert(PNode))
729 PhiWorklist.push_back(PNode);
730 continue;
731 }
732 Instruction *ACI = dyn_cast<Instruction>(IncValue);
733 if (ACI && isAMXCast(ACI)) {
734 // Verify it's a A->B cast.
735 Type *TyA = ACI->getOperand(0)->getType();
736 Type *TyB = ACI->getType();
737 if (TyA != DestTy || TyB != SrcTy)
738 return false;
739 continue;
740 }
741 return false;
742 }
743 }
744
745 // Check that each user of each old PHI node is something that we can
746 // rewrite, so that all of the old PHI nodes can be cleaned up afterwards.
747 for (auto *OldPN : OldPhiNodes) {
748 for (User *V : OldPN->users()) {
749 Instruction *ACI = dyn_cast<Instruction>(V);
750 if (ACI && isAMXCast(ACI)) {
751 // Verify it's a B->A cast.
752 Type *TyB = ACI->getOperand(0)->getType();
753 Type *TyA = ACI->getType();
754 if (TyA != DestTy || TyB != SrcTy)
755 return false;
756 } else if (auto *PHI = dyn_cast<PHINode>(V)) {
757 // As long as the user is another old PHI node, then even if we don't
758 // rewrite it, the PHI web we're considering won't have any users
759 // outside itself, so it'll be dead.
760 // example:
761 // bb.0:
762 // %0 = amxcast ...
763 // bb.1:
764 // %1 = amxcast ...
765 // bb.2:
766 // %goodphi = phi %0, %1
767 // %3 = amxcast %goodphi
768 // bb.3:
769 // %goodphi2 = phi %0, %goodphi
770 // %4 = amxcast %goodphi2
771 // When optimizeAMXCastFromPhi process %3 and %goodphi, %goodphi2 is
772 // outside the phi-web, so the combination stop When
773 // optimizeAMXCastFromPhi process %4 and %goodphi2, the optimization
774 // will be done.
775 if (OldPhiNodes.count(PHI) == 0)
776 return false;
777 } else
778 return false;
779 }
780 }
781
782 // For each old PHI node, create a corresponding new PHI node with a type A.
783 SmallDenseMap<PHINode *, PHINode *> NewPNodes;
784 for (auto *OldPN : OldPhiNodes) {
785 Builder.SetInsertPoint(OldPN);
786 PHINode *NewPN = Builder.CreatePHI(DestTy, OldPN->getNumOperands());
787 NewPNodes[OldPN] = NewPN;
788 }
789
790 // Fill in the operands of new PHI nodes.
791 for (auto *OldPN : OldPhiNodes) {
792 PHINode *NewPN = NewPNodes[OldPN];
793 for (unsigned j = 0, e = OldPN->getNumOperands(); j != e; ++j) {
794 Value *V = OldPN->getOperand(j);
795 Value *NewV = nullptr;
796 Instruction *ACI = dyn_cast<Instruction>(V);
797 // There should not be a AMXcast from a const.
798 if (ACI && isAMXCast(ACI))
799 NewV = ACI->getOperand(0);
800 else if (auto *PrevPN = dyn_cast<PHINode>(V))
801 NewV = NewPNodes[PrevPN];
802 assert(NewV)(static_cast <bool> (NewV) ? void (0) : __assert_fail (
"NewV", "llvm/lib/Target/X86/X86LowerAMXType.cpp", 802, __extension__
__PRETTY_FUNCTION__))
;
803 NewPN->addIncoming(NewV, OldPN->getIncomingBlock(j));
804 }
805 }
806
807 // Traverse all accumulated PHI nodes and process its users,
808 // which are Stores and BitcCasts. Without this processing
809 // NewPHI nodes could be replicated and could lead to extra
810 // moves generated after DeSSA.
811 // If there is a store with type B, change it to type A.
812
813 // Replace users of BitCast B->A with NewPHI. These will help
814 // later to get rid of a closure formed by OldPHI nodes.
815 for (auto *OldPN : OldPhiNodes) {
816 PHINode *NewPN = NewPNodes[OldPN];
817 for (User *V : make_early_inc_range(OldPN->users())) {
818 Instruction *ACI = dyn_cast<Instruction>(V);
819 if (ACI && isAMXCast(ACI)) {
820 Type *TyB = ACI->getOperand(0)->getType();
821 Type *TyA = ACI->getType();
822 assert(TyA == DestTy && TyB == SrcTy)(static_cast <bool> (TyA == DestTy && TyB == SrcTy
) ? void (0) : __assert_fail ("TyA == DestTy && TyB == SrcTy"
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 822, __extension__
__PRETTY_FUNCTION__))
;
823 (void)TyA;
824 (void)TyB;
825 ACI->replaceAllUsesWith(NewPN);
826 DeadInst.insert(ACI);
827 } else if (auto *PHI = dyn_cast<PHINode>(V)) {
828 // We don't need to push PHINode into DeadInst since they are operands
829 // of rootPN DCE can safely delete rootPN's operands if rootPN is dead.
830 assert(OldPhiNodes.contains(PHI))(static_cast <bool> (OldPhiNodes.contains(PHI)) ? void (
0) : __assert_fail ("OldPhiNodes.contains(PHI)", "llvm/lib/Target/X86/X86LowerAMXType.cpp"
, 830, __extension__ __PRETTY_FUNCTION__))
;
831 (void)PHI;
832 } else
833 llvm_unreachable("all uses should be handled")::llvm::llvm_unreachable_internal("all uses should be handled"
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 833)
;
834 }
835 }
836 return true;
837}
838
839bool X86LowerAMXCast::combineAMXcast(TargetLibraryInfo *TLI) {
840 bool Change = false;
841 // Collect tile cast instruction.
842 SmallVector<Instruction *, 8> Vec2TileInsts;
843 SmallVector<Instruction *, 8> Tile2VecInsts;
844 SmallVector<Instruction *, 8> PhiCastWorkList;
845 SmallSetVector<Instruction *, 16> DeadInst;
846 for (BasicBlock &BB : Func) {
847 for (Instruction &I : BB) {
848 Value *Vec;
849 if (match(&I,
850 m_Intrinsic<Intrinsic::x86_cast_vector_to_tile>(m_Value(Vec))))
851 Vec2TileInsts.push_back(&I);
852 else if (match(&I, m_Intrinsic<Intrinsic::x86_cast_tile_to_vector>(
853 m_Value(Vec))))
854 Tile2VecInsts.push_back(&I);
855 }
856 }
857
858 auto Convert = [&](SmallVectorImpl<Instruction *> &Insts, Intrinsic::ID IID) {
859 for (auto *Inst : Insts) {
860 for (User *U : Inst->users()) {
861 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
862 if (!II || II->getIntrinsicID() != IID)
863 continue;
864 // T1 = vec2tile V0
865 // V2 = tile2vec T1
866 // V3 = OP V2
867 // -->
868 // T1 = vec2tile V0
869 // V2 = tile2vec T1
870 // V3 = OP V0
871 II->replaceAllUsesWith(Inst->getOperand(0));
872 Change = true;
873 }
874 }
875 };
876
877 Convert(Vec2TileInsts, Intrinsic::x86_cast_tile_to_vector);
878 Convert(Tile2VecInsts, Intrinsic::x86_cast_vector_to_tile);
879
880 auto EraseInst = [&](SmallVectorImpl<Instruction *> &Insts) {
881 for (auto *Inst : Insts) {
882 if (Inst->use_empty()) {
883 Inst->eraseFromParent();
884 Change = true;
885 }
886 }
887 };
888
889 EraseInst(Vec2TileInsts);
890 EraseInst(Tile2VecInsts);
891
892 // Handle the A->B->A cast, and there is an intervening PHI node.
893 for (BasicBlock &BB : Func) {
894 for (Instruction &I : BB) {
895 if (isAMXCast(&I)) {
896 if (isa<PHINode>(I.getOperand(0)))
897 PhiCastWorkList.push_back(&I);
898 }
899 }
900 }
901 for (auto *I : PhiCastWorkList) {
902 // We skip the dead Amxcast.
903 if (DeadInst.contains(I))
904 continue;
905 PHINode *PN = cast<PHINode>(I->getOperand(0));
906 if (optimizeAMXCastFromPhi(cast<IntrinsicInst>(I), PN, DeadInst)) {
907 DeadInst.insert(PN);
908 Change = true;
909 }
910 }
911
912 // Since we create new phi and merge AMXCast, some old phis and AMXCast might
913 // have no uses. We do some DeadCodeElimination for them.
914 while (!DeadInst.empty()) {
915 Instruction *I = DeadInst.pop_back_val();
916 Change |= DCEInstruction(I, DeadInst, TLI);
917 }
918 return Change;
919}
920
921// There might be remaining AMXcast after combineAMXcast and they should be
922// handled elegantly.
923bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
924 IRBuilder<> Builder(AMXCast);
925 AllocaInst *AllocaAddr;
926 Value *I8Ptr, *Stride;
927 auto *Src = AMXCast->getOperand(0);
928
929 auto Prepare = [&](Type *MemTy) {
930 AllocaAddr = createAllocaInstAtEntry(Builder, AMXCast->getParent(), MemTy);
931 I8Ptr = Builder.CreateBitCast(AllocaAddr, Builder.getInt8PtrTy());
932 Stride = Builder.getInt64(64);
933 };
934
935 if (AMXCast->getType()->isX86_AMXTy()) {
936 // %2 = amxcast <225 x i32> %src to x86_amx
937 // call void @llvm.x86.tilestored64.internal(i16 15, i16 60,
938 // i8* %addr3, i64 60, x86_amx %2)
939 // -->
940 // %addr = alloca <225 x i32>, align 64
941 // store <225 x i32> %src, <225 x i32>* %addr, align 64
942 // %addr2 = bitcast <225 x i32>* %addr to i8*
943 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 15, i16 60,
944 // i8* %addr2,
945 // i64 60)
946 // call void @llvm.x86.tilestored64.internal(i16 15, i16 60,
947 // i8* %addr3, i64 60, x86_amx %2)
948 Use &U = *(AMXCast->use_begin());
949 unsigned OpNo = U.getOperandNo();
950 auto *II = dyn_cast<IntrinsicInst>(U.getUser());
951 if (!II)
952 return false; // May be bitcast from x86amx to <256 x i32>.
953 Prepare(AMXCast->getOperand(0)->getType());
954 Builder.CreateStore(Src, AllocaAddr);
955 // TODO we can pick an constant operand for the shape.
956 Value *Row = nullptr, *Col = nullptr;
957 std::tie(Row, Col) = getShape(II, OpNo);
958 std::array<Value *, 4> Args = {
959 Row, Col, I8Ptr, Builder.CreateSExt(Col, Builder.getInt64Ty())};
960 Value *NewInst = Builder.CreateIntrinsic(
961 Intrinsic::x86_tileloadd64_internal, None, Args);
962 AMXCast->replaceAllUsesWith(NewInst);
963 AMXCast->eraseFromParent();
964 } else {
965 // %2 = amxcast x86_amx %src to <225 x i32>
966 // -->
967 // %addr = alloca <225 x i32>, align 64
968 // %addr2 = bitcast <225 x i32>* to i8*
969 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col,
970 // i8* %addr2, i64 %stride)
971 // %2 = load <225 x i32>, <225 x i32>* %addr, align 64
972 auto *II = dyn_cast<IntrinsicInst>(Src);
973 if (!II)
974 return false; // May be bitcast from <256 x i32> to x86amx.
975 Prepare(AMXCast->getType());
976 Value *Row = II->getOperand(0);
977 Value *Col = II->getOperand(1);
978 std::array<Value *, 5> Args = {
979 Row, Col, I8Ptr, Builder.CreateSExt(Col, Builder.getInt64Ty()), Src};
980 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
981 Value *NewInst = Builder.CreateLoad(AMXCast->getType(), AllocaAddr);
982 AMXCast->replaceAllUsesWith(NewInst);
983 AMXCast->eraseFromParent();
984 }
985
986 return true;
987}
988
989bool X86LowerAMXCast::transformAllAMXCast() {
990 bool Change = false;
991 // Collect tile cast instruction.
992 SmallVector<Instruction *, 8> WorkLists;
993 for (BasicBlock &BB : Func) {
994 for (Instruction &I : BB) {
995 if (isAMXCast(&I))
996 WorkLists.push_back(&I);
997 }
998 }
999
1000 for (auto *Inst : WorkLists) {
1001 Change |= transformAMXCast(cast<IntrinsicInst>(Inst));
1002 }
1003
1004 return Change;
1005}
1006
1007} // anonymous namespace
1008
1009namespace {
1010
1011class X86LowerAMXTypeLegacyPass : public FunctionPass {
1012public:
1013 static char ID;
1014
1015 X86LowerAMXTypeLegacyPass() : FunctionPass(ID) {
1016 initializeX86LowerAMXTypeLegacyPassPass(*PassRegistry::getPassRegistry());
1017 }
1018
1019 bool runOnFunction(Function &F) override {
1020 bool C = false;
1021 TargetMachine *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
1022 TargetLibraryInfo *TLI =
1023 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
1024 X86LowerAMXCast LAC(F);
1025 C |= LAC.combineAMXcast(TLI);
1026 // There might be remaining AMXcast after combineAMXcast and they should be
1027 // handled elegantly.
1028 C |= LAC.transformAllAMXCast();
1029
1030 X86LowerAMXType LAT(F);
1031 C |= LAT.visit();
1032
1033 // Prepare for fast register allocation at O0.
1034 // Todo: May better check the volatile model of AMX code, not just
1035 // by checking Attribute::OptimizeNone and CodeGenOpt::None.
1036 if (TM->getOptLevel() == CodeGenOpt::None) {
1
Assuming the condition is true
2
Taking true branch
1037 // If Front End not use O0 but the Mid/Back end use O0, (e.g.
1038 // "Clang -O2 -S -emit-llvm t.c" + "llc t.ll") we should make
1039 // sure the amx data is volatile, that is nessary for AMX fast
1040 // register allocation.
1041 if (!F.hasFnAttribute(Attribute::OptimizeNone)) {
3
Assuming the condition is true
4
Taking true branch
1042 X86VolatileTileData VTD(F);
1043 C = VTD.volatileTileData() || C;
5
Calling 'X86VolatileTileData::volatileTileData'
1044 }
1045 }
1046
1047 return C;
1048 }
1049
1050 void getAnalysisUsage(AnalysisUsage &AU) const override {
1051 AU.setPreservesCFG();
1052 AU.addRequired<TargetPassConfig>();
1053 AU.addRequired<TargetLibraryInfoWrapperPass>();
1054 }
1055};
1056
1057} // anonymous namespace
1058
1059static const char PassName[] = "Lower AMX type for load/store";
1060char X86LowerAMXTypeLegacyPass::ID = 0;
1061INITIALIZE_PASS_BEGIN(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,static void *initializeX86LowerAMXTypeLegacyPassPassOnce(PassRegistry
&Registry) {
1062 false)static void *initializeX86LowerAMXTypeLegacyPassPassOnce(PassRegistry
&Registry) {
1063INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)initializeTargetPassConfigPass(Registry);
1064INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
1065INITIALIZE_PASS_END(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,PassInfo *PI = new PassInfo( PassName, "lower-amx-type", &
X86LowerAMXTypeLegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<X86LowerAMXTypeLegacyPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeX86LowerAMXTypeLegacyPassPassFlag
; void llvm::initializeX86LowerAMXTypeLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeX86LowerAMXTypeLegacyPassPassFlag
, initializeX86LowerAMXTypeLegacyPassPassOnce, std::ref(Registry
)); }
1066 false)PassInfo *PI = new PassInfo( PassName, "lower-amx-type", &
X86LowerAMXTypeLegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<X86LowerAMXTypeLegacyPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeX86LowerAMXTypeLegacyPassPassFlag
; void llvm::initializeX86LowerAMXTypeLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeX86LowerAMXTypeLegacyPassPassFlag
, initializeX86LowerAMXTypeLegacyPassPassOnce, std::ref(Registry
)); }
1067
1068FunctionPass *llvm::createX86LowerAMXTypePass() {
1069 return new X86LowerAMXTypeLegacyPass();
1070}

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/DebugLoc.h"
30#include "llvm/IR/DerivedTypes.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstddef>
48#include <cstdint>
49#include <functional>
50#include <utility>
51
52namespace llvm {
53
54class APInt;
55class MDNode;
56class Use;
57
58/// This provides the default implementation of the IRBuilder
59/// 'InsertHelper' method that is called whenever an instruction is created by
60/// IRBuilder and needs to be inserted.
61///
62/// By default, this inserts the instruction at the insertion point.
63class IRBuilderDefaultInserter {
64public:
65 virtual ~IRBuilderDefaultInserter();
66
67 virtual void InsertHelper(Instruction *I, const Twine &Name,
68 BasicBlock *BB,
69 BasicBlock::iterator InsertPt) const {
70 if (BB) BB->getInstList().insert(InsertPt, I);
71 I->setName(Name);
72 }
73};
74
75/// Provides an 'InsertHelper' that calls a user-provided callback after
76/// performing the default insertion.
77class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
78 std::function<void(Instruction *)> Callback;
79
80public:
81 virtual ~IRBuilderCallbackInserter();
82
83 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
84 : Callback(std::move(Callback)) {}
85
86 void InsertHelper(Instruction *I, const Twine &Name,
87 BasicBlock *BB,
88 BasicBlock::iterator InsertPt) const override {
89 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
90 Callback(I);
91 }
92};
93
94/// Common base class shared among various IRBuilders.
95class IRBuilderBase {
96 /// Pairs of (metadata kind, MDNode *) that should be added to all newly
97 /// created instructions, like !dbg metadata.
98 SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
99
100 /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
101 /// null. If \p MD is null, remove the entry with \p Kind.
102 void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
103 if (!MD) {
104 erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
105 return KV.first == Kind;
106 });
107 return;
108 }
109
110 for (auto &KV : MetadataToCopy)
111 if (KV.first == Kind) {
112 KV.second = MD;
113 return;
114 }
115
116 MetadataToCopy.emplace_back(Kind, MD);
117 }
118
119protected:
120 BasicBlock *BB;
121 BasicBlock::iterator InsertPt;
122 LLVMContext &Context;
123 const IRBuilderFolder &Folder;
124 const IRBuilderDefaultInserter &Inserter;
125
126 MDNode *DefaultFPMathTag;
127 FastMathFlags FMF;
128
129 bool IsFPConstrained;
130 fp::ExceptionBehavior DefaultConstrainedExcept;
131 RoundingMode DefaultConstrainedRounding;
132
133 ArrayRef<OperandBundleDef> DefaultOperandBundles;
134
135public:
136 IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
137 const IRBuilderDefaultInserter &Inserter,
138 MDNode *FPMathTag, ArrayRef<OperandBundleDef> OpBundles)
139 : Context(context), Folder(Folder), Inserter(Inserter),
140 DefaultFPMathTag(FPMathTag), IsFPConstrained(false),
141 DefaultConstrainedExcept(fp::ebStrict),
142 DefaultConstrainedRounding(RoundingMode::Dynamic),
143 DefaultOperandBundles(OpBundles) {
144 ClearInsertionPoint();
145 }
146
147 /// Insert and return the specified instruction.
148 template<typename InstTy>
149 InstTy *Insert(InstTy *I, const Twine &Name = "") const {
150 Inserter.InsertHelper(I, Name, BB, InsertPt);
151 AddMetadataToInst(I);
152 return I;
153 }
154
155 /// No-op overload to handle constants.
156 Constant *Insert(Constant *C, const Twine& = "") const {
157 return C;
158 }
159
160 Value *Insert(Value *V, const Twine &Name = "") const {
161 if (Instruction *I = dyn_cast<Instruction>(V))
162 return Insert(I, Name);
163 assert(isa<Constant>(V))(static_cast <bool> (isa<Constant>(V)) ? void (0)
: __assert_fail ("isa<Constant>(V)", "llvm/include/llvm/IR/IRBuilder.h"
, 163, __extension__ __PRETTY_FUNCTION__))
;
164 return V;
165 }
166
167 //===--------------------------------------------------------------------===//
168 // Builder configuration methods
169 //===--------------------------------------------------------------------===//
170
171 /// Clear the insertion point: created instructions will not be
172 /// inserted into a block.
173 void ClearInsertionPoint() {
174 BB = nullptr;
175 InsertPt = BasicBlock::iterator();
176 }
177
178 BasicBlock *GetInsertBlock() const { return BB; }
179 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
180 LLVMContext &getContext() const { return Context; }
181
182 /// This specifies that created instructions should be appended to the
183 /// end of the specified block.
184 void SetInsertPoint(BasicBlock *TheBB) {
185 BB = TheBB;
186 InsertPt = BB->end();
187 }
188
189 /// This specifies that created instructions should be inserted before
190 /// the specified instruction.
191 void SetInsertPoint(Instruction *I) {
192 BB = I->getParent();
193 InsertPt = I->getIterator();
194 assert(InsertPt != BB->end() && "Can't read debug loc from end()")(static_cast <bool> (InsertPt != BB->end() &&
"Can't read debug loc from end()") ? void (0) : __assert_fail
("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "llvm/include/llvm/IR/IRBuilder.h", 194, __extension__ __PRETTY_FUNCTION__
))
;
195 SetCurrentDebugLocation(I->getDebugLoc());
196 }
197
198 /// This specifies that created instructions should be inserted at the
199 /// specified point.
200 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
201 BB = TheBB;
202 InsertPt = IP;
203 if (IP != TheBB->end())
204 SetCurrentDebugLocation(IP->getDebugLoc());
205 }
206
207 /// Set location information used by debugging information.
208 void SetCurrentDebugLocation(DebugLoc L) {
209 AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
210 }
211
212 /// Collect metadata with IDs \p MetadataKinds from \p Src which should be
213 /// added to all created instructions. Entries present in MedataDataToCopy but
214 /// not on \p Src will be dropped from MetadataToCopy.
215 void CollectMetadataToCopy(Instruction *Src,
216 ArrayRef<unsigned> MetadataKinds) {
217 for (unsigned K : MetadataKinds)
218 AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
219 }
220
221 /// Get location information used by debugging information.
222 DebugLoc getCurrentDebugLocation() const {
223 for (auto &KV : MetadataToCopy)
224 if (KV.first == LLVMContext::MD_dbg)
225 return {cast<DILocation>(KV.second)};
226
227 return {};
228 }
229
230 /// If this builder has a current debug location, set it on the
231 /// specified instruction.
232 void SetInstDebugLocation(Instruction *I) const {
233 for (const auto &KV : MetadataToCopy)
234 if (KV.first == LLVMContext::MD_dbg) {
235 I->setDebugLoc(DebugLoc(KV.second));
236 return;
237 }
238 }
239
240 /// Add all entries in MetadataToCopy to \p I.
241 void AddMetadataToInst(Instruction *I) const {
242 for (auto &KV : MetadataToCopy)
243 I->setMetadata(KV.first, KV.second);
244 }
245
246 /// Get the return type of the current function that we're emitting
247 /// into.
248 Type *getCurrentFunctionReturnType() const;
249
250 /// InsertPoint - A saved insertion point.
251 class InsertPoint {
252 BasicBlock *Block = nullptr;
253 BasicBlock::iterator Point;
254
255 public:
256 /// Creates a new insertion point which doesn't point to anything.
257 InsertPoint() = default;
258
259 /// Creates a new insertion point at the given location.
260 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
261 : Block(InsertBlock), Point(InsertPoint) {}
262
263 /// Returns true if this insert point is set.
264 bool isSet() const { return (Block != nullptr); }
265
266 BasicBlock *getBlock() const { return Block; }
267 BasicBlock::iterator getPoint() const { return Point; }
268 };
269
270 /// Returns the current insert point.
271 InsertPoint saveIP() const {
272 return InsertPoint(GetInsertBlock(), GetInsertPoint());
273 }
274
275 /// Returns the current insert point, clearing it in the process.
276 InsertPoint saveAndClearIP() {
277 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
278 ClearInsertionPoint();
279 return IP;
280 }
281
282 /// Sets the current insert point to a previously-saved location.
283 void restoreIP(InsertPoint IP) {
284 if (IP.isSet())
285 SetInsertPoint(IP.getBlock(), IP.getPoint());
286 else
287 ClearInsertionPoint();
288 }
289
290 /// Get the floating point math metadata being used.
291 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
292
293 /// Get the flags to be applied to created floating point ops
294 FastMathFlags getFastMathFlags() const { return FMF; }
295
296 FastMathFlags &getFastMathFlags() { return FMF; }
297
298 /// Clear the fast-math flags.
299 void clearFastMathFlags() { FMF.clear(); }
300
301 /// Set the floating point math metadata to be used.
302 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
303
304 /// Set the fast-math flags to be used with generated fp-math operators
305 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
306
307 /// Enable/Disable use of constrained floating point math. When
308 /// enabled the CreateF<op>() calls instead create constrained
309 /// floating point intrinsic calls. Fast math flags are unaffected
310 /// by this setting.
311 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
312
313 /// Query for the use of constrained floating point math
314 bool getIsFPConstrained() { return IsFPConstrained; }
315
316 /// Set the exception handling to be used with constrained floating point
317 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
318#ifndef NDEBUG
319 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(NewExcept);
320 assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 320, __extension__ __PRETTY_FUNCTION__
))
;
321#endif
322 DefaultConstrainedExcept = NewExcept;
323 }
324
325 /// Set the rounding mode handling to be used with constrained floating point
326 void setDefaultConstrainedRounding(RoundingMode NewRounding) {
327#ifndef NDEBUG
328 Optional<StringRef> RoundingStr = convertRoundingModeToStr(NewRounding);
329 assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 329, __extension__ __PRETTY_FUNCTION__
))
;
330#endif
331 DefaultConstrainedRounding = NewRounding;
332 }
333
334 /// Get the exception handling used with constrained floating point
335 fp::ExceptionBehavior getDefaultConstrainedExcept() {
336 return DefaultConstrainedExcept;
337 }
338
339 /// Get the rounding mode handling used with constrained floating point
340 RoundingMode getDefaultConstrainedRounding() {
341 return DefaultConstrainedRounding;
342 }
343
344 void setConstrainedFPFunctionAttr() {
345 assert(BB && "Must have a basic block to set any function attributes!")(static_cast <bool> (BB && "Must have a basic block to set any function attributes!"
) ? void (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "llvm/include/llvm/IR/IRBuilder.h", 345, __extension__ __PRETTY_FUNCTION__
))
;
346
347 Function *F = BB->getParent();
348 if (!F->hasFnAttribute(Attribute::StrictFP)) {
349 F->addFnAttr(Attribute::StrictFP);
350 }
351 }
352
353 void setConstrainedFPCallAttr(CallBase *I) {
354 I->addFnAttr(Attribute::StrictFP);
355 }
356
357 void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
358 DefaultOperandBundles = OpBundles;
359 }
360
361 //===--------------------------------------------------------------------===//
362 // RAII helpers.
363 //===--------------------------------------------------------------------===//
364
365 // RAII object that stores the current insertion point and restores it
366 // when the object is destroyed. This includes the debug location.
367 class InsertPointGuard {
368 IRBuilderBase &Builder;
369 AssertingVH<BasicBlock> Block;
370 BasicBlock::iterator Point;
371 DebugLoc DbgLoc;
372
373 public:
374 InsertPointGuard(IRBuilderBase &B)
375 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
376 DbgLoc(B.getCurrentDebugLocation()) {}
377
378 InsertPointGuard(const InsertPointGuard &) = delete;
379 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
380
381 ~InsertPointGuard() {
382 Builder.restoreIP(InsertPoint(Block, Point));
383 Builder.SetCurrentDebugLocation(DbgLoc);
384 }
385 };
386
387 // RAII object that stores the current fast math settings and restores
388 // them when the object is destroyed.
389 class FastMathFlagGuard {
390 IRBuilderBase &Builder;
391 FastMathFlags FMF;
392 MDNode *FPMathTag;
393 bool IsFPConstrained;
394 fp::ExceptionBehavior DefaultConstrainedExcept;
395 RoundingMode DefaultConstrainedRounding;
396
397 public:
398 FastMathFlagGuard(IRBuilderBase &B)
399 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
400 IsFPConstrained(B.IsFPConstrained),
401 DefaultConstrainedExcept(B.DefaultConstrainedExcept),
402 DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}
403
404 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
405 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
406
407 ~FastMathFlagGuard() {
408 Builder.FMF = FMF;
409 Builder.DefaultFPMathTag = FPMathTag;
410 Builder.IsFPConstrained = IsFPConstrained;
411 Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
412 Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
413 }
414 };
415
416 // RAII object that stores the current default operand bundles and restores
417 // them when the object is destroyed.
418 class OperandBundlesGuard {
419 IRBuilderBase &Builder;
420 ArrayRef<OperandBundleDef> DefaultOperandBundles;
421
422 public:
423 OperandBundlesGuard(IRBuilderBase &B)
424 : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}
425
426 OperandBundlesGuard(const OperandBundlesGuard &) = delete;
427 OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;
428
429 ~OperandBundlesGuard() {
430 Builder.DefaultOperandBundles = DefaultOperandBundles;
431 }
432 };
433
434
435 //===--------------------------------------------------------------------===//
436 // Miscellaneous creation methods.
437 //===--------------------------------------------------------------------===//
438
439 /// Make a new global variable with initializer type i8*
440 ///
441 /// Make a new global variable with an initializer that has array of i8 type
442 /// filled in with the null terminated string value specified. The new global
443 /// variable will be marked mergable with any others of the same contents. If
444 /// Name is specified, it is the name of the global variable created.
445 ///
446 /// If no module is given via \p M, it is take from the insertion point basic
447 /// block.
448 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
449 unsigned AddressSpace = 0,
450 Module *M = nullptr);
451
452 /// Get a constant value representing either true or false.
453 ConstantInt *getInt1(bool V) {
454 return ConstantInt::get(getInt1Ty(), V);
455 }
456
457 /// Get the constant value for i1 true.
458 ConstantInt *getTrue() {
459 return ConstantInt::getTrue(Context);
460 }
461
462 /// Get the constant value for i1 false.
463 ConstantInt *getFalse() {
464 return ConstantInt::getFalse(Context);
465 }
466
467 /// Get a constant 8-bit value.
468 ConstantInt *getInt8(uint8_t C) {
469 return ConstantInt::get(getInt8Ty(), C);
470 }
471
472 /// Get a constant 16-bit value.
473 ConstantInt *getInt16(uint16_t C) {
474 return ConstantInt::get(getInt16Ty(), C);
475 }
476
477 /// Get a constant 32-bit value.
478 ConstantInt *getInt32(uint32_t C) {
479 return ConstantInt::get(getInt32Ty(), C);
480 }
481
482 /// Get a constant 64-bit value.
483 ConstantInt *getInt64(uint64_t C) {
484 return ConstantInt::get(getInt64Ty(), C);
485 }
486
487 /// Get a constant N-bit value, zero extended or truncated from
488 /// a 64-bit value.
489 ConstantInt *getIntN(unsigned N, uint64_t C) {
490 return ConstantInt::get(getIntNTy(N), C);
491 }
492
493 /// Get a constant integer value.
494 ConstantInt *getInt(const APInt &AI) {
495 return ConstantInt::get(Context, AI);
496 }
497
498 //===--------------------------------------------------------------------===//
499 // Type creation methods
500 //===--------------------------------------------------------------------===//
501
502 /// Fetch the type representing a single bit
503 IntegerType *getInt1Ty() {
504 return Type::getInt1Ty(Context);
505 }
506
507 /// Fetch the type representing an 8-bit integer.
508 IntegerType *getInt8Ty() {
509 return Type::getInt8Ty(Context);
510 }
511
512 /// Fetch the type representing a 16-bit integer.
513 IntegerType *getInt16Ty() {
514 return Type::getInt16Ty(Context);
515 }
516
517 /// Fetch the type representing a 32-bit integer.
518 IntegerType *getInt32Ty() {
519 return Type::getInt32Ty(Context);
520 }
521
522 /// Fetch the type representing a 64-bit integer.
523 IntegerType *getInt64Ty() {
524 return Type::getInt64Ty(Context);
525 }
526
527 /// Fetch the type representing a 128-bit integer.
528 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
529
530 /// Fetch the type representing an N-bit integer.
531 IntegerType *getIntNTy(unsigned N) {
532 return Type::getIntNTy(Context, N);
533 }
534
535 /// Fetch the type representing a 16-bit floating point value.
536 Type *getHalfTy() {
537 return Type::getHalfTy(Context);
538 }
539
540 /// Fetch the type representing a 16-bit brain floating point value.
541 Type *getBFloatTy() {
542 return Type::getBFloatTy(Context);
543 }
544
545 /// Fetch the type representing a 32-bit floating point value.
546 Type *getFloatTy() {
547 return Type::getFloatTy(Context);
548 }
549
550 /// Fetch the type representing a 64-bit floating point value.
551 Type *getDoubleTy() {
552 return Type::getDoubleTy(Context);
553 }
554
555 /// Fetch the type representing void.
556 Type *getVoidTy() {
557 return Type::getVoidTy(Context);
558 }
559
560 /// Fetch the type representing a pointer to an 8-bit integer value.
561 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
562 return Type::getInt8PtrTy(Context, AddrSpace);
563 }
564
565 /// Fetch the type representing a pointer to an integer value.
566 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
567 return DL.getIntPtrType(Context, AddrSpace);
568 }
569
570 //===--------------------------------------------------------------------===//
571 // Intrinsic creation methods
572 //===--------------------------------------------------------------------===//
573
574 /// Create and insert a memset to the specified pointer and the
575 /// specified value.
576 ///
577 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
578 /// specified, it will be added to the instruction. Likewise with alias.scope
579 /// and noalias tags.
580 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
581 MaybeAlign Align, bool isVolatile = false,
582 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
583 MDNode *NoAliasTag = nullptr) {
584 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
585 TBAATag, ScopeTag, NoAliasTag);
586 }
587
588 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
589 bool isVolatile = false, MDNode *TBAATag = nullptr,
590 MDNode *ScopeTag = nullptr,
591 MDNode *NoAliasTag = nullptr);
592
593 /// Create and insert an element unordered-atomic memset of the region of
594 /// memory starting at the given pointer to the given value.
595 ///
596 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
597 /// specified, it will be added to the instruction. Likewise with alias.scope
598 /// and noalias tags.
599 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
600 uint64_t Size, Align Alignment,
601 uint32_t ElementSize,
602 MDNode *TBAATag = nullptr,
603 MDNode *ScopeTag = nullptr,
604 MDNode *NoAliasTag = nullptr) {
605 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
606 Align(Alignment), ElementSize,
607 TBAATag, ScopeTag, NoAliasTag);
608 }
609
610 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
611 Value *Size, Align Alignment,
612 uint32_t ElementSize,
613 MDNode *TBAATag = nullptr,
614 MDNode *ScopeTag = nullptr,
615 MDNode *NoAliasTag = nullptr);
616
617 /// Create and insert a memcpy between the specified pointers.
618 ///
619 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
620 /// specified, it will be added to the instruction. Likewise with alias.scope
621 /// and noalias tags.
622 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
623 MaybeAlign SrcAlign, uint64_t Size,
624 bool isVolatile = false, MDNode *TBAATag = nullptr,
625 MDNode *TBAAStructTag = nullptr,
626 MDNode *ScopeTag = nullptr,
627 MDNode *NoAliasTag = nullptr) {
628 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
629 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
630 NoAliasTag);
631 }
632
633 CallInst *CreateMemTransferInst(
634 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
635 MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
636 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
637 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
638
639 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
640 MaybeAlign SrcAlign, Value *Size,
641 bool isVolatile = false, MDNode *TBAATag = nullptr,
642 MDNode *TBAAStructTag = nullptr,
643 MDNode *ScopeTag = nullptr,
644 MDNode *NoAliasTag = nullptr) {
645 return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
646 SrcAlign, Size, isVolatile, TBAATag,
647 TBAAStructTag, ScopeTag, NoAliasTag);
648 }
649
650 CallInst *
651 CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
652 MaybeAlign SrcAlign, Value *Size, bool IsVolatile = false,
653 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
654 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
655
656 /// Create and insert an element unordered-atomic memcpy between the
657 /// specified pointers.
658 ///
659 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
660 ///
661 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
662 /// specified, it will be added to the instruction. Likewise with alias.scope
663 /// and noalias tags.
664 CallInst *CreateElementUnorderedAtomicMemCpy(
665 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
666 uint32_t ElementSize, MDNode *TBAATag = nullptr,
667 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
668 MDNode *NoAliasTag = nullptr);
669
670 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
671 MaybeAlign SrcAlign, uint64_t Size,
672 bool isVolatile = false, MDNode *TBAATag = nullptr,
673 MDNode *ScopeTag = nullptr,
674 MDNode *NoAliasTag = nullptr) {
675 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
676 isVolatile, TBAATag, ScopeTag, NoAliasTag);
677 }
678
679 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
680 MaybeAlign SrcAlign, Value *Size,
681 bool isVolatile = false, MDNode *TBAATag = nullptr,
682 MDNode *ScopeTag = nullptr,
683 MDNode *NoAliasTag = nullptr);
684
685 /// \brief Create and insert an element unordered-atomic memmove between the
686 /// specified pointers.
687 ///
688 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
689 /// respectively.
690 ///
691 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
692 /// specified, it will be added to the instruction. Likewise with alias.scope
693 /// and noalias tags.
694 CallInst *CreateElementUnorderedAtomicMemMove(
695 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
696 uint32_t ElementSize, MDNode *TBAATag = nullptr,
697 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
698 MDNode *NoAliasTag = nullptr);
699
700 /// Create a sequential vector fadd reduction intrinsic of the source vector.
701 /// The first parameter is a scalar accumulator value. An unordered reduction
702 /// can be created by adding the reassoc fast-math flag to the resulting
703 /// sequential reduction.
704 CallInst *CreateFAddReduce(Value *Acc, Value *Src);
705
706 /// Create a sequential vector fmul reduction intrinsic of the source vector.
707 /// The first parameter is a scalar accumulator value. An unordered reduction
708 /// can be created by adding the reassoc fast-math flag to the resulting
709 /// sequential reduction.
710 CallInst *CreateFMulReduce(Value *Acc, Value *Src);
711
712 /// Create a vector int add reduction intrinsic of the source vector.
713 CallInst *CreateAddReduce(Value *Src);
714
715 /// Create a vector int mul reduction intrinsic of the source vector.
716 CallInst *CreateMulReduce(Value *Src);
717
718 /// Create a vector int AND reduction intrinsic of the source vector.
719 CallInst *CreateAndReduce(Value *Src);
720
721 /// Create a vector int OR reduction intrinsic of the source vector.
722 CallInst *CreateOrReduce(Value *Src);
723
724 /// Create a vector int XOR reduction intrinsic of the source vector.
725 CallInst *CreateXorReduce(Value *Src);
726
727 /// Create a vector integer max reduction intrinsic of the source
728 /// vector.
729 CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
730
731 /// Create a vector integer min reduction intrinsic of the source
732 /// vector.
733 CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
734
735 /// Create a vector float max reduction intrinsic of the source
736 /// vector.
737 CallInst *CreateFPMaxReduce(Value *Src);
738
739 /// Create a vector float min reduction intrinsic of the source
740 /// vector.
741 CallInst *CreateFPMinReduce(Value *Src);
742
743 /// Create a lifetime.start intrinsic.
744 ///
745 /// If the pointer isn't i8* it will be converted.
746 CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
747
748 /// Create a lifetime.end intrinsic.
749 ///
750 /// If the pointer isn't i8* it will be converted.
751 CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
752
753 /// Create a call to invariant.start intrinsic.
754 ///
755 /// If the pointer isn't i8* it will be converted.
756 CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
757
758 /// Create a call to Masked Load intrinsic
759 CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask,
760 Value *PassThru = nullptr, const Twine &Name = "");
761
762 /// Create a call to Masked Store intrinsic
763 CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
764 Value *Mask);
765
766 /// Create a call to Masked Gather intrinsic
767 CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
768 Value *Mask = nullptr, Value *PassThru = nullptr,
769 const Twine &Name = "");
770
771 /// Create a call to Masked Scatter intrinsic
772 CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
773 Value *Mask = nullptr);
774
775 /// Create an assume intrinsic call that allows the optimizer to
776 /// assume that the provided condition will be true.
777 ///
778 /// The optional argument \p OpBundles specifies operand bundles that are
779 /// added to the call instruction.
780 CallInst *CreateAssumption(Value *Cond,
781 ArrayRef<OperandBundleDef> OpBundles = llvm::None);
782
783 /// Create a llvm.experimental.noalias.scope.decl intrinsic call.
784 Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
785 Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
786 return CreateNoAliasScopeDeclaration(
787 MetadataAsValue::get(Context, ScopeTag));
788 }
789
790 /// Create a call to the experimental.gc.statepoint intrinsic to
791 /// start a new statepoint sequence.
792 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
793 Value *ActualCallee,
794 ArrayRef<Value *> CallArgs,
795 Optional<ArrayRef<Value *>> DeoptArgs,
796 ArrayRef<Value *> GCArgs,
797 const Twine &Name = "");
798
799 /// Create a call to the experimental.gc.statepoint intrinsic to
800 /// start a new statepoint sequence.
801 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
802 Value *ActualCallee, uint32_t Flags,
803 ArrayRef<Value *> CallArgs,
804 Optional<ArrayRef<Use>> TransitionArgs,
805 Optional<ArrayRef<Use>> DeoptArgs,
806 ArrayRef<Value *> GCArgs,
807 const Twine &Name = "");
808
809 /// Conveninence function for the common case when CallArgs are filled
810 /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
811 /// .get()'ed to get the Value pointer.
812 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
813 Value *ActualCallee, ArrayRef<Use> CallArgs,
814 Optional<ArrayRef<Value *>> DeoptArgs,
815 ArrayRef<Value *> GCArgs,
816 const Twine &Name = "");
817
818 /// Create an invoke to the experimental.gc.statepoint intrinsic to
819 /// start a new statepoint sequence.
820 InvokeInst *
821 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
822 Value *ActualInvokee, BasicBlock *NormalDest,
823 BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
824 Optional<ArrayRef<Value *>> DeoptArgs,
825 ArrayRef<Value *> GCArgs, const Twine &Name = "");
826
827 /// Create an invoke to the experimental.gc.statepoint intrinsic to
828 /// start a new statepoint sequence.
829 InvokeInst *CreateGCStatepointInvoke(
830 uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
831 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
832 ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
833 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
834 const Twine &Name = "");
835
836 // Convenience function for the common case when CallArgs are filled in using
837 // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
838 // get the Value *.
839 InvokeInst *
840 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
841 Value *ActualInvokee, BasicBlock *NormalDest,
842 BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
843 Optional<ArrayRef<Value *>> DeoptArgs,
844 ArrayRef<Value *> GCArgs, const Twine &Name = "");
845
846 /// Create a call to the experimental.gc.result intrinsic to extract
847 /// the result from a call wrapped in a statepoint.
848 CallInst *CreateGCResult(Instruction *Statepoint,
849 Type *ResultType,
850 const Twine &Name = "");
851
852 /// Create a call to the experimental.gc.relocate intrinsics to
853 /// project the relocated value of one pointer from the statepoint.
854 CallInst *CreateGCRelocate(Instruction *Statepoint,
855 int BaseOffset,
856 int DerivedOffset,
857 Type *ResultType,
858 const Twine &Name = "");
859
860 /// Create a call to the experimental.gc.pointer.base intrinsic to get the
861 /// base pointer for the specified derived pointer.
862 CallInst *CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name = "");
863
864 /// Create a call to the experimental.gc.get.pointer.offset intrinsic to get
865 /// the offset of the specified derived pointer from its base.
866 CallInst *CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name = "");
867
868 /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
869 /// will be the same type as that of \p Scaling.
870 Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
871
872 /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
873 Value *CreateStepVector(Type *DstType, const Twine &Name = "");
874
875 /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
876 /// type.
877 CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
878 Instruction *FMFSource = nullptr,
879 const Twine &Name = "");
880
881 /// Create a call to intrinsic \p ID with 2 operands which is mangled on the
882 /// first type.
883 CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
884 Instruction *FMFSource = nullptr,
885 const Twine &Name = "");
886
887 /// Create a call to intrinsic \p ID with \p args, mangled using \p Types. If
888 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
889 /// the intrinsic.
890 CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
891 ArrayRef<Value *> Args,
892 Instruction *FMFSource = nullptr,
893 const Twine &Name = "");
894
895 /// Create call to the minnum intrinsic.
896 CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
897 return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name);
898 }
899
900 /// Create call to the maxnum intrinsic.
901 CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
902 return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name);
903 }
904
905 /// Create call to the minimum intrinsic.
906 CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
907 return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
908 }
909
910 /// Create call to the maximum intrinsic.
911 CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
912 return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
913 }
914
915 /// Create a call to the arithmetic_fence intrinsic.
916 CallInst *CreateArithmeticFence(Value *Val, Type *DstType,
917 const Twine &Name = "") {
918 return CreateIntrinsic(Intrinsic::arithmetic_fence, DstType, Val, nullptr,
919 Name);
920 }
921
922 /// Create a call to the experimental.vector.extract intrinsic.
923 CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
924 const Twine &Name = "") {
925 return CreateIntrinsic(Intrinsic::experimental_vector_extract,
926 {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
927 Name);
928 }
929
930 /// Create a call to the experimental.vector.insert intrinsic.
931 CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
932 Value *Idx, const Twine &Name = "") {
933 return CreateIntrinsic(Intrinsic::experimental_vector_insert,
934 {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
935 nullptr, Name);
936 }
937
938private:
939 /// Create a call to a masked intrinsic with given Id.
940 CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
941 ArrayRef<Type *> OverloadedTypes,
942 const Twine &Name = "");
943
944 Value *getCastedInt8PtrValue(Value *Ptr);
945
946 //===--------------------------------------------------------------------===//
947 // Instruction creation methods: Terminators
948 //===--------------------------------------------------------------------===//
949
950private:
951 /// Helper to add branch weight and unpredictable metadata onto an
952 /// instruction.
953 /// \returns The annotated instruction.
954 template <typename InstTy>
955 InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
956 if (Weights)
957 I->setMetadata(LLVMContext::MD_prof, Weights);
958 if (Unpredictable)
959 I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
960 return I;
961 }
962
963public:
964 /// Create a 'ret void' instruction.
965 ReturnInst *CreateRetVoid() {
966 return Insert(ReturnInst::Create(Context));
967 }
968
969 /// Create a 'ret <val>' instruction.
970 ReturnInst *CreateRet(Value *V) {
971 return Insert(ReturnInst::Create(Context, V));
972 }
973
974 /// Create a sequence of N insertvalue instructions,
975 /// with one Value from the retVals array each, that build a aggregate
976 /// return value one value at a time, and a ret instruction to return
977 /// the resulting aggregate value.
978 ///
979 /// This is a convenience function for code that uses aggregate return values
980 /// as a vehicle for having multiple return values.
981 ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) {
982 Value *V = UndefValue::get(getCurrentFunctionReturnType());
983 for (unsigned i = 0; i != N; ++i)
984 V = CreateInsertValue(V, retVals[i], i, "mrv");
985 return Insert(ReturnInst::Create(Context, V));
986 }
987
988 /// Create an unconditional 'br label X' instruction.
989 BranchInst *CreateBr(BasicBlock *Dest) {
990 return Insert(BranchInst::Create(Dest));
991 }
992
993 /// Create a conditional 'br Cond, TrueDest, FalseDest'
994 /// instruction.
995 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
996 MDNode *BranchWeights = nullptr,
997 MDNode *Unpredictable = nullptr) {
998 return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
999 BranchWeights, Unpredictable));
1000 }
1001
1002 /// Create a conditional 'br Cond, TrueDest, FalseDest'
1003 /// instruction. Copy branch meta data if available.
1004 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
1005 Instruction *MDSrc) {
1006 BranchInst *Br = BranchInst::Create(True, False, Cond);
1007 if (MDSrc) {
1008 unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
1009 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
1010 Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4));
1011 }
1012 return Insert(Br);
1013 }
1014
1015 /// Create a switch instruction with the specified value, default dest,
1016 /// and with a hint for the number of cases that will be added (for efficient
1017 /// allocation).
1018 SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
1019 MDNode *BranchWeights = nullptr,
1020 MDNode *Unpredictable = nullptr) {
1021 return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
1022 BranchWeights, Unpredictable));
1023 }
1024
1025 /// Create an indirect branch instruction with the specified address
1026 /// operand, with an optional hint for the number of destinations that will be
1027 /// added (for efficient allocation).
1028 IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) {
1029 return Insert(IndirectBrInst::Create(Addr, NumDests));
1030 }
1031
1032 /// Create an invoke instruction.
1033 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1034 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1035 ArrayRef<Value *> Args,
1036 ArrayRef<OperandBundleDef> OpBundles,
1037 const Twine &Name = "") {
1038 InvokeInst *II =
1039 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles);
1040 if (IsFPConstrained)
1041 setConstrainedFPCallAttr(II);
1042 return Insert(II, Name);
1043 }
1044 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1045 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1046 ArrayRef<Value *> Args = None,
1047 const Twine &Name = "") {
1048 InvokeInst *II =
1049 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args);
1050 if (IsFPConstrained)
1051 setConstrainedFPCallAttr(II);
1052 return Insert(II, Name);
1053 }
1054
1055 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1056 BasicBlock *UnwindDest, ArrayRef<Value *> Args,
1057 ArrayRef<OperandBundleDef> OpBundles,
1058 const Twine &Name = "") {
1059 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1060 NormalDest, UnwindDest, Args, OpBundles, Name);
1061 }
1062
1063 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1064 BasicBlock *UnwindDest,
1065 ArrayRef<Value *> Args = None,
1066 const Twine &Name = "") {
1067 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1068 NormalDest, UnwindDest, Args, Name);
1069 }
1070
1071 /// \brief Create a callbr instruction.
1072 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1073 BasicBlock *DefaultDest,
1074 ArrayRef<BasicBlock *> IndirectDests,
1075 ArrayRef<Value *> Args = None,
1076 const Twine &Name = "") {
1077 return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
1078 Args), Name);
1079 }
1080 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1081 BasicBlock *DefaultDest,
1082 ArrayRef<BasicBlock *> IndirectDests,
1083 ArrayRef<Value *> Args,
1084 ArrayRef<OperandBundleDef> OpBundles,
1085 const Twine &Name = "") {
1086 return Insert(
1087 CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
1088 OpBundles), Name);
1089 }
1090
1091 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1092 ArrayRef<BasicBlock *> IndirectDests,
1093 ArrayRef<Value *> Args = None,
1094 const Twine &Name = "") {
1095 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1096 DefaultDest, IndirectDests, Args, Name);
1097 }
1098 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1099 ArrayRef<BasicBlock *> IndirectDests,
1100 ArrayRef<Value *> Args,
1101 ArrayRef<OperandBundleDef> OpBundles,
1102 const Twine &Name = "") {
1103 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1104 DefaultDest, IndirectDests, Args, Name);
1105 }
1106
1107 ResumeInst *CreateResume(Value *Exn) {
1108 return Insert(ResumeInst::Create(Exn));
1109 }
1110
1111 CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad,
1112 BasicBlock *UnwindBB = nullptr) {
1113 return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB));
1114 }
1115
1116 CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB,
1117 unsigned NumHandlers,
1118 const Twine &Name = "") {
1119 return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers),
1120 Name);
1121 }
1122
1123 CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args,
1124 const Twine &Name = "") {
1125 return Insert(CatchPadInst::Create(ParentPad, Args), Name);
1126 }
1127
1128 CleanupPadInst *CreateCleanupPad(Value *ParentPad,
1129 ArrayRef<Value *> Args = None,
1130 const Twine &Name = "") {
1131 return Insert(CleanupPadInst::Create(ParentPad, Args), Name);
1132 }
1133
1134 CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) {
1135 return Insert(CatchReturnInst::Create(CatchPad, BB));
1136 }
1137
1138 UnreachableInst *CreateUnreachable() {
1139 return Insert(new UnreachableInst(Context));
1140 }
1141
1142 //===--------------------------------------------------------------------===//
1143 // Instruction creation methods: Binary Operators
1144 //===--------------------------------------------------------------------===//
1145private:
1146 BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc,
1147 Value *LHS, Value *RHS,
1148 const Twine &Name,
1149 bool HasNUW, bool HasNSW) {
1150 BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name);
1151 if (HasNUW) BO->setHasNoUnsignedWrap();
1152 if (HasNSW) BO->setHasNoSignedWrap();
1153 return BO;
1154 }
1155
1156 Instruction *setFPAttrs(Instruction *I, MDNode *FPMD,
1157 FastMathFlags FMF) const {
1158 if (!FPMD)
1159 FPMD = DefaultFPMathTag;
1160 if (FPMD)
1161 I->setMetadata(LLVMContext::MD_fpmath, FPMD);
1162 I->setFastMathFlags(FMF);
1163 return I;
1164 }
1165
1166 Value *foldConstant(Instruction::BinaryOps Opc, Value *L,
1167 Value *R, const Twine &Name) const {
1168 auto *LC = dyn_cast<Constant>(L);
1169 auto *RC = dyn_cast<Constant>(R);
1170 return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr;
1171 }
1172
1173 Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) {
1174 RoundingMode UseRounding = DefaultConstrainedRounding;
1175
1176 if (Rounding.hasValue())
1177 UseRounding = Rounding.getValue();
1178
1179 Optional<StringRef> RoundingStr = convertRoundingModeToStr(UseRounding);
1180 assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr.hasValue() && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1180, __extension__ __PRETTY_FUNCTION__
))
;
1181 auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue());
1182
1183 return MetadataAsValue::get(Context, RoundingMDS);
1184 }
1185
1186 Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) {
1187 fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept;
1188
1189 if (Except.hasValue())
1190 UseExcept = Except.getValue();
1191
1192 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(UseExcept);
1193 assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr.hasValue() && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1193, __extension__ __PRETTY_FUNCTION__
))
;
1194 auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue());
1195
1196 return MetadataAsValue::get(Context, ExceptMDS);
1197 }
1198
1199 Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) {
1200 assert(CmpInst::isFPPredicate(Predicate) &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
1201 Predicate != CmpInst::FCMP_FALSE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
1202 Predicate != CmpInst::FCMP_TRUE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
1203 "Invalid constrained FP comparison predicate!")(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1203, __extension__ __PRETTY_FUNCTION__
))
;
1204
1205 StringRef PredicateStr = CmpInst::getPredicateName(Predicate);
1206 auto *PredicateMDS = MDString::get(Context, PredicateStr);
1207
1208 return MetadataAsValue::get(Context, PredicateMDS);
1209 }
1210
1211public:
1212 Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "",
1213 bool HasNUW = false, bool HasNSW = false) {
1214 if (auto *V = Folder.FoldAdd(LHS, RHS, HasNUW, HasNSW))
1215 return V;
1216 return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name,
1217 HasNUW, HasNSW);
1218 }
1219
1220 Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1221 return CreateAdd(LHS, RHS, Name, false, true);
1222 }
1223
1224 Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1225 return CreateAdd(LHS, RHS, Name, true, false);
1226 }
1227
1228 Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "",
1229 bool HasNUW = false, bool HasNSW = false) {
1230 if (auto *LC = dyn_cast<Constant>(LHS))
1231 if (auto *RC = dyn_cast<Constant>(RHS))
1232 return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name);
1233 return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name,
1234 HasNUW, HasNSW);
1235 }
1236
1237 Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1238 return CreateSub(LHS, RHS, Name, false, true);
1239 }
1240
1241 Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1242 return CreateSub(LHS, RHS, Name, true, false);
1243 }
1244
1245 Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "",
1246 bool HasNUW = false, bool HasNSW = false) {
1247 if (auto *LC = dyn_cast<Constant>(LHS))
1248 if (auto *RC = dyn_cast<Constant>(RHS))
1249 return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name);
1250 return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name,
1251 HasNUW, HasNSW);
1252 }
1253
1254 Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1255 return CreateMul(LHS, RHS, Name, false, true);
1256 }
1257
1258 Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1259 return CreateMul(LHS, RHS, Name, true, false);
1260 }
1261
1262 Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1263 bool isExact = false) {
1264 if (auto *LC = dyn_cast<Constant>(LHS))
1265 if (auto *RC = dyn_cast<Constant>(RHS))
1266 return Insert(Folder.CreateUDiv(LC, RC, isExact), Name);
1267 if (!isExact)
1268 return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name);
1269 return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name);
1270 }
1271
1272 Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1273 return CreateUDiv(LHS, RHS, Name, true);
1274 }
1275
1276 Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1277 bool isExact = false) {
1278 if (auto *LC = dyn_cast<Constant>(LHS))
1279 if (auto *RC = dyn_cast<Constant>(RHS))
1280 return Insert(Folder.CreateSDiv(LC, RC, isExact), Name);
1281 if (!isExact)
1282 return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name);
1283 return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name);
1284 }
1285
1286 Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1287 return CreateSDiv(LHS, RHS, Name, true);
1288 }
1289
1290 Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") {
1291 if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V;
1292 return Insert(BinaryOperator::CreateURem(LHS, RHS), Name);
1293 }
1294
1295 Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") {
1296 if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V;
1297 return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name);
1298 }
1299
1300 Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "",
1301 bool HasNUW = false, bool HasNSW = false) {
1302 if (auto *LC = dyn_cast<Constant>(LHS))
1303 if (auto *RC = dyn_cast<Constant>(RHS))
1304 return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name);
1305 return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name,
1306 HasNUW, HasNSW);
1307 }
1308
1309 Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "",
1310 bool HasNUW = false, bool HasNSW = false) {
1311 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1312 HasNUW, HasNSW);
1313 }
1314
1315 Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "",
1316 bool HasNUW = false, bool HasNSW = false) {
1317 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1318 HasNUW, HasNSW);
1319 }
1320
1321 Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "",
1322 bool isExact = false) {
1323 if (auto *LC = dyn_cast<Constant>(LHS))
1324 if (auto *RC = dyn_cast<Constant>(RHS))
1325 return Insert(Folder.CreateLShr(LC, RC, isExact), Name);
1326 if (!isExact)
1327 return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name);
1328 return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name);
1329 }
1330
1331 Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1332 bool isExact = false) {
1333 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1334 }
1335
1336 Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1337 bool isExact = false) {
1338 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1339 }
1340
1341 Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "",
1342 bool isExact = false) {
1343 if (auto *LC = dyn_cast<Constant>(LHS))
1344 if (auto *RC = dyn_cast<Constant>(RHS))
1345 return Insert(Folder.CreateAShr(LC, RC, isExact), Name);
1346 if (!isExact)
1347 return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name);
1348 return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name);
1349 }
1350
1351 Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1352 bool isExact = false) {
1353 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1354 }
1355
1356 Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1357 bool isExact = false) {
1358 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1359 }
1360
1361 Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") {
1362 if (auto *RC = dyn_cast<Constant>(RHS)) {
1363 if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne())
1364 return LHS; // LHS & -1 -> LHS
1365 if (auto *LC = dyn_cast<Constant>(LHS))
1366 return Insert(Folder.CreateAnd(LC, RC), Name);
1367 }
1368 return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name);
1369 }
1370
1371 Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1372 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1373 }
1374
1375 Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1376 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1377 }
1378
1379 Value *CreateAnd(ArrayRef<Value*> Ops) {
1380 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1380, __extension__
__PRETTY_FUNCTION__))
;
1381 Value *Accum = Ops[0];
1382 for (unsigned i = 1; i < Ops.size(); i++)
1383 Accum = CreateAnd(Accum, Ops[i]);
1384 return Accum;
1385 }
1386
1387 Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
1388 if (auto *V = Folder.FoldOr(LHS, RHS))
1389 return V;
1390 return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
1391 }
1392
1393 Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1394 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1395 }
1396
1397 Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1398 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1399 }
1400
1401 Value *CreateOr(ArrayRef<Value*> Ops) {
1402 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1402, __extension__
__PRETTY_FUNCTION__))
;
1403 Value *Accum = Ops[0];
1404 for (unsigned i = 1; i < Ops.size(); i++)
1405 Accum = CreateOr(Accum, Ops[i]);
1406 return Accum;
1407 }
1408
1409 Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") {
1410 if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V;
1411 return Insert(BinaryOperator::CreateXor(LHS, RHS), Name);
1412 }
1413
1414 Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1415 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1416 }
1417
1418 Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1419 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1420 }
1421
1422 Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "",
1423 MDNode *FPMD = nullptr) {
1424 if (IsFPConstrained)
1425 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1426 L, R, nullptr, Name, FPMD);
1427
1428 if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V;
1429 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF);
1430 return Insert(I, Name);
1431 }
1432
1433 /// Copy fast-math-flags from an instruction rather than using the builder's
1434 /// default FMF.
1435 Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource,
1436 const Twine &Name = "") {
1437 if (IsFPConstrained)
1438 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1439 L, R, FMFSource, Name);
1440
1441 if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V;
1442 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr,
1443 FMFSource->getFastMathFlags());
1444 return Insert(I, Name);
1445 }
1446
1447 Value *CreateFSub(Value *L, Value *R, const Twine &Name = "",
1448 MDNode *FPMD = nullptr) {
1449 if (IsFPConstrained)
1450 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1451 L, R, nullptr, Name, FPMD);
1452
1453 if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V;
1454 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF);
1455 return Insert(I, Name);
1456 }
1457
1458 /// Copy fast-math-flags from an instruction rather than using the builder's
1459 /// default FMF.
1460 Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource,
1461 const Twine &Name = "") {
1462 if (IsFPConstrained)
1463 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1464 L, R, FMFSource, Name);
1465
1466 if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V;
1467 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr,
1468 FMFSource->getFastMathFlags());
1469 return Insert(I, Name);
1470 }
1471
1472 Value *CreateFMul(Value *L, Value *R, const Twine &Name = "",
1473 MDNode *FPMD = nullptr) {
1474 if (IsFPConstrained)
1475 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1476 L, R, nullptr, Name, FPMD);
1477
1478 if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V;
1479 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF);
1480 return Insert(I, Name);
1481 }
1482
1483 /// Copy fast-math-flags from an instruction rather than using the builder's
1484 /// default FMF.
1485 Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource,
1486 const Twine &Name = "") {
1487 if (IsFPConstrained)
1488 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1489 L, R, FMFSource, Name);
1490
1491 if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V;
1492 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr,
1493 FMFSource->getFastMathFlags());
1494 return Insert(I, Name);
1495 }
1496
1497 Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "",
1498 MDNode *FPMD = nullptr) {
1499 if (IsFPConstrained)
1500 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1501 L, R, nullptr, Name, FPMD);
1502
1503 if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V;
1504 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF);
1505 return Insert(I, Name);
1506 }
1507
1508 /// Copy fast-math-flags from an instruction rather than using the builder's
1509 /// default FMF.
1510 Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource,
1511 const Twine &Name = "") {
1512 if (IsFPConstrained)
1513 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1514 L, R, FMFSource, Name);
1515
1516 if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V;
1517 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr,
1518 FMFSource->getFastMathFlags());
1519 return Insert(I, Name);
1520 }
1521
1522 Value *CreateFRem(Value *L, Value *R, const Twine &Name = "",
1523 MDNode *FPMD = nullptr) {
1524 if (IsFPConstrained)
1525 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1526 L, R, nullptr, Name, FPMD);
1527
1528 if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V;
1529 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF);
1530 return Insert(I, Name);
1531 }
1532
1533 /// Copy fast-math-flags from an instruction rather than using the builder's
1534 /// default FMF.
1535 Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource,
1536 const Twine &Name = "") {
1537 if (IsFPConstrained)
1538 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1539 L, R, FMFSource, Name);
1540
1541 if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V;
1542 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr,
1543 FMFSource->getFastMathFlags());
1544 return Insert(I, Name);
1545 }
1546
1547 Value *CreateBinOp(Instruction::BinaryOps Opc,
1548 Value *LHS, Value *RHS, const Twine &Name = "",
1549 MDNode *FPMathTag = nullptr) {
1550 if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V;
1551 Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS);
1552 if (isa<FPMathOperator>(BinOp))
1553 setFPAttrs(BinOp, FPMathTag, FMF);
1554 return Insert(BinOp, Name);
1555 }
1556
1557 Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1558 assert(Cond2->getType()->isIntOrIntVectorTy(1))(static_cast <bool> (Cond2->getType()->isIntOrIntVectorTy
(1)) ? void (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "llvm/include/llvm/IR/IRBuilder.h", 1558, __extension__ __PRETTY_FUNCTION__
))
;
1559 return CreateSelect(Cond1, Cond2,
1560 ConstantInt::getNullValue(Cond2->getType()), Name);
1561 }
1562
1563 Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1564 assert(Cond2->getType()->isIntOrIntVectorTy(1))(static_cast <bool> (Cond2->getType()->isIntOrIntVectorTy
(1)) ? void (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "llvm/include/llvm/IR/IRBuilder.h", 1564, __extension__ __PRETTY_FUNCTION__
))
;
1565 return CreateSelect(Cond1, ConstantInt::getAllOnesValue(Cond2->getType()),
1566 Cond2, Name);
1567 }
1568
1569 // NOTE: this is sequential, non-commutative, ordered reduction!
1570 Value *CreateLogicalOr(ArrayRef<Value *> Ops) {
1571 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1571, __extension__
__PRETTY_FUNCTION__))
;
1572 Value *Accum = Ops[0];
1573 for (unsigned i = 1; i < Ops.size(); i++)
1574 Accum = CreateLogicalOr(Accum, Ops[i]);
1575 return Accum;
1576 }
1577
1578 CallInst *CreateConstrainedFPBinOp(
1579 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr,
1580 const Twine &Name = "", MDNode *FPMathTag = nullptr,
1581 Optional<RoundingMode> Rounding = None,
1582 Optional<fp::ExceptionBehavior> Except = None);
1583
1584 Value *CreateNeg(Value *V, const Twine &Name = "",
1585 bool HasNUW = false, bool HasNSW = false) {
1586 if (auto *VC = dyn_cast<Constant>(V))
1587 return Insert(Folder.CreateNeg(VC, HasNUW, HasNSW), Name);
1588 BinaryOperator *BO = Insert(BinaryOperator::CreateNeg(V), Name);
1589 if (HasNUW) BO->setHasNoUnsignedWrap();
1590 if (HasNSW) BO->setHasNoSignedWrap();
1591 return BO;
1592 }
1593
1594 Value *CreateNSWNeg(Value *V, const Twine &Name = "") {
1595 return CreateNeg(V, Name, false, true);
1596 }
1597
1598 Value *CreateNUWNeg(Value *V, const Twine &Name = "") {
1599 return CreateNeg(V, Name, true, false);
1600 }
1601
1602 Value *CreateFNeg(Value *V, const Twine &Name = "",
1603 MDNode *FPMathTag = nullptr) {
1604 if (auto *VC = dyn_cast<Constant>(V))
1605 return Insert(Folder.CreateFNeg(VC), Name);
1606 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF),
1607 Name);
1608 }
1609
1610 /// Copy fast-math-flags from an instruction rather than using the builder's
1611 /// default FMF.
1612 Value *CreateFNegFMF(Value *V, Instruction *FMFSource,
1613 const Twine &Name = "") {
1614 if (auto *VC = dyn_cast<Constant>(V))
1615 return Insert(Folder.CreateFNeg(VC), Name);
1616 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr,
1617 FMFSource->getFastMathFlags()),
1618 Name);
1619 }
1620
1621 Value *CreateNot(Value *V, const Twine &Name = "") {
1622 if (auto *VC = dyn_cast<Constant>(V))
1623 return Insert(Folder.CreateNot(VC), Name);
1624 return Insert(BinaryOperator::CreateNot(V), Name);
1625 }
1626
1627 Value *CreateUnOp(Instruction::UnaryOps Opc,
1628 Value *V, const Twine &Name = "",
1629 MDNode *FPMathTag = nullptr) {
1630 if (auto *VC = dyn_cast<Constant>(V))
1631 return Insert(Folder.CreateUnOp(Opc, VC), Name);
1632 Instruction *UnOp = UnaryOperator::Create(Opc, V);
1633 if (isa<FPMathOperator>(UnOp))
1634 setFPAttrs(UnOp, FPMathTag, FMF);
1635 return Insert(UnOp, Name);
1636 }
1637
1638 /// Create either a UnaryOperator or BinaryOperator depending on \p Opc.
1639 /// Correct number of operands must be passed accordingly.
1640 Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1641 const Twine &Name = "", MDNode *FPMathTag = nullptr);
1642
1643 //===--------------------------------------------------------------------===//
1644 // Instruction creation methods: Memory Instructions
1645 //===--------------------------------------------------------------------===//
1646
1647 AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace,
1648 Value *ArraySize = nullptr, const Twine &Name = "") {
1649 const DataLayout &DL = BB->getModule()->getDataLayout();
1650 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1651 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1652 }
1653
1654 AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr,
1655 const Twine &Name = "") {
1656 const DataLayout &DL = BB->getModule()->getDataLayout();
1657 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1658 unsigned AddrSpace = DL.getAllocaAddrSpace();
1659 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1660 }
1661
1662 /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of
1663 /// converting the string to 'bool' for the isVolatile parameter.
1664 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) {
1665 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1666 }
1667
1668 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
1669 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1670 }
1671
1672 LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile,
1673 const Twine &Name = "") {
1674 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), isVolatile, Name);
1675 }
1676
1677 StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
1678 return CreateAlignedStore(Val, Ptr, MaybeAlign(), isVolatile);
1679 }
1680
1681 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1682 const char *Name) {
1683 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1684 }
1685
1686 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1687 const Twine &Name = "") {
1688 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1689 }
1690
1691 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1692 bool isVolatile, const Twine &Name = "") {
1693 if (!Align) {
1694 const DataLayout &DL = BB->getModule()->getDataLayout();
1695 Align = DL.getABITypeAlign(Ty);
1696 }
1697 return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile, *Align), Name);
1698 }
1699
1700 StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align,
1701 bool isVolatile = false) {
1702 if (!Align) {
1703 const DataLayout &DL = BB->getModule()->getDataLayout();
1704 Align = DL.getABITypeAlign(Val->getType());
1705 }
1706 return Insert(new StoreInst(Val, Ptr, isVolatile, *Align));
1707 }
1708 FenceInst *CreateFence(AtomicOrdering Ordering,
1709 SyncScope::ID SSID = SyncScope::System,
1710 const Twine &Name = "") {
1711 return Insert(new FenceInst(Context, Ordering, SSID), Name);
1712 }
1713
1714 AtomicCmpXchgInst *
1715 CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
1716 AtomicOrdering SuccessOrdering,
1717 AtomicOrdering FailureOrdering,
1718 SyncScope::ID SSID = SyncScope::System) {
1719 if (!Align) {
1720 const DataLayout &DL = BB->getModule()->getDataLayout();
1721 Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
1722 }
1723
1724 return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
1725 FailureOrdering, SSID));
1726 }
1727
1728 AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
1729 Value *Val, MaybeAlign Align,
1730 AtomicOrdering Ordering,
1731 SyncScope::ID SSID = SyncScope::System) {
1732 if (!Align) {
1733 const DataLayout &DL = BB->getModule()->getDataLayout();
1734 Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
1735 }
1736
1737 return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
1738 }
1739
1740 Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1741 const Twine &Name = "") {
1742 if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, /*IsInBounds=*/false))
1743 return V;
1744 return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name);
1745 }
1746
1747 Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1748 const Twine &Name = "") {
1749 if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, /*IsInBounds=*/true))
1750 return V;
1751 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name);
1752 }
1753
1754 Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") {
1755 if (auto *V = Folder.FoldGEP(Ty, Ptr, {Idx}, /*IsInBounds=*/false))
1756 return V;
1757 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1758 }
1759
1760 Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx,
1761 const Twine &Name = "") {
1762 if (auto *V = Folder.FoldGEP(Ty, Ptr, {Idx}, /*IsInBounds=*/true))
1763 return V;
1764 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1765 }
1766
1767 Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1768 const Twine &Name = "") {
1769 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1770
1771 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false))
1772 return V;
1773
1774 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1775 }
1776
1777 Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1778 const Twine &Name = "") {
1779 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1780
1781 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true))
1782 return V;
1783
1784 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1785 }
1786
1787 Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1,
1788 const Twine &Name = "") {
1789 Value *Idxs[] = {
1790 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1791 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1792 };
1793
1794 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false))
1795 return V;
1796
1797 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1798 }
1799
1800 Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0,
1801 unsigned Idx1, const Twine &Name = "") {
1802 Value *Idxs[] = {
1803 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1804 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1805 };
1806
1807 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true))
1808 return V;
1809
1810 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1811 }
1812
1813 Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1814 const Twine &Name = "") {
1815 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1816
1817 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false))
1818 return V;
1819
1820 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1821 }
1822
1823 Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1824 const Twine &Name = "") {
1825 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1826
1827 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true))
1828 return V;
1829
1830 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1831 }
1832
1833 Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1,
1834 const Twine &Name = "") {
1835 Value *Idxs[] = {
1836 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1837 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1838 };
1839
1840 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false))
1841 return V;
1842
1843 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1844 }
1845
1846 Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1847 uint64_t Idx1, const Twine &Name = "") {
1848 Value *Idxs[] = {
1849 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1850 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1851 };
1852
1853 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true))
1854 return V;
1855
1856 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1857 }
1858
1859 Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx,
1860 const Twine &Name = "") {
1861 return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name);
1862 }
1863
1864 /// Same as CreateGlobalString, but return a pointer with "i8*" type
1865 /// instead of a pointer to array of i8.
1866 ///
1867 /// If no module is given via \p M, it is take from the insertion point basic
1868 /// block.
1869 Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "",
1870 unsigned AddressSpace = 0,
1871 Module *M = nullptr) {
1872 GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace, M);
1873 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1874 Constant *Indices[] = {Zero, Zero};
1875 return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV,
1876 Indices);
1877 }
1878
1879 //===--------------------------------------------------------------------===//
1880 // Instruction creation methods: Cast/Conversion Operators
1881 //===--------------------------------------------------------------------===//
1882
1883 Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
1884 return CreateCast(Instruction::Trunc, V, DestTy, Name);
1885 }
1886
1887 Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") {
1888 return CreateCast(Instruction::ZExt, V, DestTy, Name);
1889 }
1890
1891 Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") {
1892 return CreateCast(Instruction::SExt, V, DestTy, Name);
1893 }
1894
1895 /// Create a ZExt or Trunc from the integer value V to DestTy. Return
1896 /// the value untouched if the type of V is already DestTy.
1897 Value *CreateZExtOrTrunc(Value *V, Type *DestTy,
1898 const Twine &Name = "") {
1899 assert(V->getType()->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1901, __extension__ __PRETTY_FUNCTION__
))
1900 DestTy->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1901, __extension__ __PRETTY_FUNCTION__
))
1901 "Can only zero extend/truncate integers!")(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1901, __extension__ __PRETTY_FUNCTION__
))
;
1902 Type *VTy = V->getType();
1903 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1904 return CreateZExt(V, DestTy, Name);
1905 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1906 return CreateTrunc(V, DestTy, Name);
1907 return V;
1908 }
1909
1910 /// Create a SExt or Trunc from the integer value V to DestTy. Return
1911 /// the value untouched if the type of V is already DestTy.
1912 Value *CreateSExtOrTrunc(Value *V, Type *DestTy,
1913 const Twine &Name = "") {
1914 assert(V->getType()->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1916, __extension__ __PRETTY_FUNCTION__
))
1915 DestTy->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1916, __extension__ __PRETTY_FUNCTION__
))
1916 "Can only sign extend/truncate integers!")(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1916, __extension__ __PRETTY_FUNCTION__
))
;
1917 Type *VTy = V->getType();
1918 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1919 return CreateSExt(V, DestTy, Name);
1920 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1921 return CreateTrunc(V, DestTy, Name);
1922 return V;
1923 }
1924
1925 Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") {
1926 if (IsFPConstrained)
1927 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui,
1928 V, DestTy, nullptr, Name);
1929 return CreateCast(Instruction::FPToUI, V, DestTy, Name);
1930 }
1931
1932 Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") {
1933 if (IsFPConstrained)
1934 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi,
1935 V, DestTy, nullptr, Name);
1936 return CreateCast(Instruction::FPToSI, V, DestTy, Name);
1937 }
1938
1939 Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
1940 if (IsFPConstrained)
1941 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp,
1942 V, DestTy, nullptr, Name);
1943 return CreateCast(Instruction::UIToFP, V, DestTy, Name);
1944 }
1945
1946 Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
1947 if (IsFPConstrained)
1948 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp,
1949 V, DestTy, nullptr, Name);
1950 return CreateCast(Instruction::SIToFP, V, DestTy, Name);
1951 }
1952
1953 Value *CreateFPTrunc(Value *V, Type *DestTy,
1954 const Twine &Name = "") {
1955 if (IsFPConstrained)
1956 return CreateConstrainedFPCast(
1957 Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr,
1958 Name);
1959 return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
1960 }
1961
1962 Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
1963 if (IsFPConstrained)
1964 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext,
1965 V, DestTy, nullptr, Name);
1966 return CreateCast(Instruction::FPExt, V, DestTy, Name);
1967 }
1968
1969 Value *CreatePtrToInt(Value *V, Type *DestTy,
1970 const Twine &Name = "") {
1971 return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
1972 }
1973
1974 Value *CreateIntToPtr(Value *V, Type *DestTy,
1975 const Twine &Name = "") {
1976 return CreateCast(Instruction::IntToPtr, V, DestTy, Name);
1977 }
1978
1979 Value *CreateBitCast(Value *V, Type *DestTy,
1980 const Twine &Name = "") {
1981 return CreateCast(Instruction::BitCast, V, DestTy, Name);
1982 }
1983
1984 Value *CreateAddrSpaceCast(Value *V, Type *DestTy,
1985 const Twine &Name = "") {
1986 return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name);
1987 }
1988
1989 Value *CreateZExtOrBitCast(Value *V, Type *DestTy,
1990 const Twine &Name = "") {
1991 if (V->getType() == DestTy)
1992 return V;
1993 if (auto *VC = dyn_cast<Constant>(V))
1994 return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name);
1995 return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name);
1996 }
1997
1998 Value *CreateSExtOrBitCast(Value *V, Type *DestTy,
1999 const Twine &Name = "") {
2000 if (V->getType() == DestTy)
2001 return V;
2002 if (auto *VC = dyn_cast<Constant>(V))
2003 return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name);
2004 return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name);
2005 }
2006
2007 Value *CreateTruncOrBitCast(Value *V, Type *DestTy,
2008 const Twine &Name = "") {
2009 if (V->getType() == DestTy)
2010 return V;
2011 if (auto *VC = dyn_cast<Constant>(V))
2012 return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name);
2013 return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name);
2014 }
2015
2016 Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
2017 const Twine &Name = "") {
2018 if (V->getType() == DestTy)
2019 return V;
2020 if (auto *VC = dyn_cast<Constant>(V))
2021 return Insert(Folder.CreateCast(Op, VC, DestTy), Name);
2022 return Insert(CastInst::Create(Op, V, DestTy), Name);
2023 }
2024
2025 Value *CreatePointerCast(Value *V, Type *DestTy,
2026 const Twine &Name = "") {
2027 if (V->getType() == DestTy)
2028 return V;
2029 if (auto *VC = dyn_cast<Constant>(V))
2030 return Insert(Folder.CreatePointerCast(VC, DestTy), Name);
2031 return Insert(CastInst::CreatePointerCast(V, DestTy), Name);
2032 }
2033
2034 Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy,
2035 const Twine &Name = "") {
2036 if (V->getType() == DestTy)
2037 return V;
2038
2039 if (auto *VC = dyn_cast<Constant>(V)) {
2040 return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy),
2041 Name);
2042 }
2043
2044 return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy),
2045 Name);
2046 }
2047
2048 Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned,
2049 const Twine &Name = "") {
2050 if (V->getType() == DestTy)
2051 return V;
2052 if (auto *VC = dyn_cast<Constant>(V))
2053 return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name);
2054 return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name);
2055 }
2056
2057 Value *CreateBitOrPointerCast(Value *V, Type *DestTy,
2058 const Twine &Name = "") {
2059 if (V->getType() == DestTy)
2060 return V;
2061 if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy())
2062 return CreatePtrToInt(V, DestTy, Name);
2063 if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy())
2064 return CreateIntToPtr(V, DestTy, Name);
2065
2066 return CreateBitCast(V, DestTy, Name);
2067 }
2068
2069 Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
2070 if (V->getType() == DestTy)
2071 return V;
2072 if (auto *VC = dyn_cast<Constant>(V))
2073 return Insert(Folder.CreateFPCast(VC, DestTy), Name);
2074 return Insert(CastInst::CreateFPCast(V, DestTy), Name);
2075 }
2076
2077 CallInst *CreateConstrainedFPCast(
2078 Intrinsic::ID ID, Value *V, Type *DestTy,
2079 Instruction *FMFSource = nullptr, const Twine &Name = "",
2080 MDNode *FPMathTag = nullptr,
2081 Optional<RoundingMode> Rounding = None,
2082 Optional<fp::ExceptionBehavior> Except = None);
2083
2084 // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a
2085 // compile time error, instead of converting the string to bool for the
2086 // isSigned parameter.
2087 Value *CreateIntCast(Value *, Type *, const char *) = delete;
2088
2089 //===--------------------------------------------------------------------===//
2090 // Instruction creation methods: Compare Instructions
2091 //===--------------------------------------------------------------------===//
2092
2093 Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
2094 return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name);
2095 }
2096
2097 Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") {
2098 return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name);
2099 }
2100
2101 Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2102 return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name);
2103 }
2104
2105 Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2106 return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name);
2107 }
2108
2109 Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") {
2110 return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name);
2111 }
2112
2113 Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") {
2114 return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name);
2115 }
2116
2117 Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2118 return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name);
2119 }
2120
2121 Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2122 return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name);
2123 }
2124
2125 Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") {
2126 return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name);
2127 }
2128
2129 Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") {
2130 return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name);
2131 }
2132
2133 Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2134 MDNode *FPMathTag = nullptr) {
2135 return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag);
2136 }
2137
2138 Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "",
2139 MDNode *FPMathTag = nullptr) {
2140 return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag);
2141 }
2142
2143 Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "",
2144 MDNode *FPMathTag = nullptr) {
2145 return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag);
2146 }
2147
2148 Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "",
2149 MDNode *FPMathTag = nullptr) {
2150 return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag);
2151 }
2152
2153 Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "",
2154 MDNode *FPMathTag = nullptr) {
2155 return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag);
2156 }
2157
2158 Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "",
2159 MDNode *FPMathTag = nullptr) {
2160 return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag);
2161 }
2162
2163 Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "",
2164 MDNode *FPMathTag = nullptr) {
2165 return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag);
2166 }
2167
2168 Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "",
2169 MDNode *FPMathTag = nullptr) {
2170 return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag);
2171 }
2172
2173 Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2174 MDNode *FPMathTag = nullptr) {
2175 return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag);
2176 }
2177
2178 Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "",
2179 MDNode *FPMathTag = nullptr) {
2180 return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag);
2181 }
2182
2183 Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "",
2184 MDNode *FPMathTag = nullptr) {
2185 return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag);
2186 }
2187
2188 Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "",
2189 MDNode *FPMathTag = nullptr) {
2190 return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag);
2191 }
2192
2193 Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "",
2194 MDNode *FPMathTag = nullptr) {
2195 return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag);
2196 }
2197
2198 Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "",
2199 MDNode *FPMathTag = nullptr) {
2200 return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag);
2201 }
2202
2203 Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2204 const Twine &Name = "") {
2205 if (auto *V = Folder.FoldICmp(P, LHS, RHS))
2206 return V;
2207 return Insert(new ICmpInst(P, LHS, RHS), Name);
2208 }
2209
2210 // Create a quiet floating-point comparison (i.e. one that raises an FP
2211 // exception only in the case where an input is a signaling NaN).
2212 // Note that this differs from CreateFCmpS only if IsFPConstrained is true.
2213 Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2214 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2215 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, false);
2216 }
2217
2218 Value *CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
2219 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2220 return CmpInst::isFPPredicate(Pred)
2221 ? CreateFCmp(Pred, LHS, RHS, Name, FPMathTag)
2222 : CreateICmp(Pred, LHS, RHS, Name);
2223 }
2224
2225 // Create a signaling floating-point comparison (i.e. one that raises an FP
2226 // exception whenever an input is any NaN, signaling or quiet).
2227 // Note that this differs from CreateFCmp only if IsFPConstrained is true.
2228 Value *CreateFCmpS(CmpInst::Predicate P, Value *LHS, Value *RHS,
2229 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2230 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, true);
2231 }
2232
2233private:
2234 // Helper routine to create either a signaling or a quiet FP comparison.
2235 Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
2236 const Twine &Name, MDNode *FPMathTag,
2237 bool IsSignaling);
2238
2239public:
2240 CallInst *CreateConstrainedFPCmp(
2241 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
2242 const Twine &Name = "", Optional<fp::ExceptionBehavior> Except = None);
2243
2244 //===--------------------------------------------------------------------===//
2245 // Instruction creation methods: Other Instructions
2246 //===--------------------------------------------------------------------===//
2247
2248 PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
2249 const Twine &Name = "") {
2250 PHINode *Phi = PHINode::Create(Ty, NumReservedValues);
2251 if (isa<FPMathOperator>(Phi))
2252 setFPAttrs(Phi, nullptr /* MDNode* */, FMF);
2253 return Insert(Phi, Name);
2254 }
2255
2256 CallInst *CreateCall(FunctionType *FTy, Value *Callee,
2257 ArrayRef<Value *> Args = None, const Twine &Name = "",
2258 MDNode *FPMathTag = nullptr) {
2259 CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
2260 if (IsFPConstrained)
2261 setConstrainedFPCallAttr(CI);
2262 if (isa<FPMathOperator>(CI))
2263 setFPAttrs(CI, FPMathTag, FMF);
2264 return Insert(CI, Name);
2265 }
2266
2267 CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
2268 ArrayRef<OperandBundleDef> OpBundles,
2269 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2270 CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
2271 if (IsFPConstrained)
2272 setConstrainedFPCallAttr(CI);
2273 if (isa<FPMathOperator>(CI))
2274 setFPAttrs(CI, FPMathTag, FMF);
2275 return Insert(CI, Name);
2276 }
2277
2278 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None,
2279 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2280 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name,
2281 FPMathTag);
2282 }
2283
2284 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args,
2285 ArrayRef<OperandBundleDef> OpBundles,
2286 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2287 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args,
2288 OpBundles, Name, FPMathTag);
2289 }
2290
2291 CallInst *CreateConstrainedFPCall(
2292 Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
2293 Optional<RoundingMode> Rounding = None,
2294 Optional<fp::ExceptionBehavior> Except = None);
2295
2296 Value *CreateSelect(Value *C, Value *True, Value *False,
2297 const Twine &Name = "", Instruction *MDFrom = nullptr);
2298
2299 VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
2300 return Insert(new VAArgInst(List, Ty), Name);
2301 }
2302
2303 Value *CreateExtractElement(Value *Vec, Value *Idx,
2304 const Twine &Name = "") {
2305 if (auto *VC = dyn_cast<Constant>(Vec))
2306 if (auto *IC = dyn_cast<Constant>(Idx))
2307 return Insert(Folder.CreateExtractElement(VC, IC), Name);
2308 return Insert(ExtractElementInst::Create(Vec, Idx), Name);
2309 }
2310
2311 Value *CreateExtractElement(Value *Vec, uint64_t Idx,
2312 const Twine &Name = "") {
2313 return CreateExtractElement(Vec, getInt64(Idx), Name);
2314 }
2315
2316 Value *CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx,
2317 const Twine &Name = "") {
2318 return CreateInsertElement(PoisonValue::get(VecTy), NewElt, Idx, Name);
2319 }
2320
2321 Value *CreateInsertElement(Type *VecTy, Value *NewElt, uint64_t Idx,
2322 const Twine &Name = "") {
2323 return CreateInsertElement(PoisonValue::get(VecTy), NewElt, Idx, Name);
2324 }
2325
2326 Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx,
2327 const Twine &Name = "") {
2328 if (auto *VC = dyn_cast<Constant>(Vec))
2329 if (auto *NC = dyn_cast<Constant>(NewElt))
2330 if (auto *IC = dyn_cast<Constant>(Idx))
2331 return Insert(Folder.CreateInsertElement(VC, NC, IC), Name);
2332 return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name);
2333 }
2334
2335 Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx,
2336 const Twine &Name = "") {
2337 return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
2338 }
2339
2340 Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
2341 const Twine &Name = "") {
2342 SmallVector<int, 16> IntMask;
2343 ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
2344 return CreateShuffleVector(V1, V2, IntMask, Name);
2345 }
2346
2347 LLVM_ATTRIBUTE_DEPRECATED(Value *CreateShuffleVector(Value *V1, Value *V2,[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
2348 ArrayRef<uint32_t> Mask,[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
2349 const Twine &Name = ""),[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
2350 "Pass indices as 'int' instead")[[deprecated("Pass indices as 'int' instead")]] Value *CreateShuffleVector
(Value *V1, Value *V2, ArrayRef<uint32_t> Mask, const Twine
&Name = "")
{
2351 SmallVector<int, 16> IntMask;
2352 IntMask.assign(Mask.begin(), Mask.end());
2353 return CreateShuffleVector(V1, V2, IntMask, Name);
2354 }
2355
2356 /// See class ShuffleVectorInst for a description of the mask representation.
2357 Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
2358 const Twine &Name = "") {
2359 if (auto *V1C = dyn_cast<Constant>(V1))
2360 if (auto *V2C = dyn_cast<Constant>(V2))
2361 return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
2362 return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
2363 }
2364
2365 /// Create a unary shuffle. The second vector operand of the IR instruction
2366 /// is poison.
2367 Value *CreateShuffleVector(Value *V, ArrayRef<int> Mask,
2368 const Twine &Name = "") {
2369 return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name);
2370 }
2371
2372 Value *CreateExtractValue(Value *Agg,
2373 ArrayRef<unsigned> Idxs,
2374 const Twine &Name = "") {
2375 if (auto *AggC = dyn_cast<Constant>(Agg))
2376 return Insert(Folder.CreateExtractValue(AggC, Idxs), Name);
2377 return Insert(ExtractValueInst::Create(Agg, Idxs), Name);
2378 }
2379
2380 Value *CreateInsertValue(Value *Agg, Value *Val,
2381 ArrayRef<unsigned> Idxs,
2382 const Twine &Name = "") {
2383 if (auto *AggC = dyn_cast<Constant>(Agg))
2384 if (auto *ValC = dyn_cast<Constant>(Val))
2385 return Insert(Folder.CreateInsertValue(AggC, ValC, Idxs), Name);
2386 return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
2387 }
2388
2389 LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
2390 const Twine &Name = "") {
2391 return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
2392 }
2393
2394 Value *CreateFreeze(Value *V, const Twine &Name = "") {
2395 return Insert(new FreezeInst(V), Name);
2396 }
2397
2398 //===--------------------------------------------------------------------===//
2399 // Utility creation methods
2400 //===--------------------------------------------------------------------===//
2401
2402 /// Return an i1 value testing if \p Arg is null.
2403 Value *CreateIsNull(Value *Arg, const Twine &Name = "") {
2404 return CreateICmpEQ(Arg, Constant::getNullValue(Arg->getType()),
2405 Name);
2406 }
2407
2408 /// Return an i1 value testing if \p Arg is not null.
2409 Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") {
2410 return CreateICmpNE(Arg, Constant::getNullValue(Arg->getType()),
2411 Name);
2412 }
2413
2414 /// Return the i64 difference between two pointer values, dividing out
2415 /// the size of the pointed-to objects.
2416 ///
2417 /// This is intended to implement C-style pointer subtraction. As such, the
2418 /// pointers must be appropriately aligned for their element types and
2419 /// pointing into the same object.
2420 Value *CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name = "");
2421
2422 /// Create a launder.invariant.group intrinsic call. If Ptr type is
2423 /// different from pointer to i8, it's casted to pointer to i8 in the same
2424 /// address space before call and casted back to Ptr type after call.
2425 Value *CreateLaunderInvariantGroup(Value *Ptr);
2426
2427 /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is
2428 /// different from pointer to i8, it's casted to pointer to i8 in the same
2429 /// address space before call and casted back to Ptr type after call.
2430 Value *CreateStripInvariantGroup(Value *Ptr);
2431
2432 /// Return a vector value that contains the vector V reversed
2433 Value *CreateVectorReverse(Value *V, const Twine &Name = "");
2434
2435 /// Return a vector splice intrinsic if using scalable vectors, otherwise
2436 /// return a shufflevector. If the immediate is positive, a vector is
2437 /// extracted from concat(V1, V2), starting at Imm. If the immediate
2438 /// is negative, we extract -Imm elements from V1 and the remaining
2439 /// elements from V2. Imm is a signed integer in the range
2440 /// -VL <= Imm < VL (where VL is the runtime vector length of the
2441 /// source/result vector)
2442 Value *CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
2443 const Twine &Name = "");
2444
2445 /// Return a vector value that contains \arg V broadcasted to \p
2446 /// NumElts elements.
2447 Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
2448
2449 /// Return a vector value that contains \arg V broadcasted to \p
2450 /// EC elements.
2451 Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");
2452
2453 /// Return a value that has been extracted from a larger integer type.
2454 Value *CreateExtractInteger(const DataLayout &DL, Value *From,
2455 IntegerType *ExtractedTy, uint64_t Offset,
2456 const Twine &Name);
2457
2458 Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
2459 unsigned Dimension, unsigned LastIndex,
2460 MDNode *DbgInfo);
2461
2462 Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex,
2463 MDNode *DbgInfo);
2464
2465 Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
2466 unsigned Index, unsigned FieldIndex,
2467 MDNode *DbgInfo);
2468
2469private:
2470 /// Helper function that creates an assume intrinsic call that
2471 /// represents an alignment assumption on the provided pointer \p PtrValue
2472 /// with offset \p OffsetValue and alignment value \p AlignValue.
2473 CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
2474 Value *PtrValue, Value *AlignValue,
2475 Value *OffsetValue);
2476
2477public:
2478 /// Create an assume intrinsic call that represents an alignment
2479 /// assumption on the provided pointer.
2480 ///
2481 /// An optional offset can be provided, and if it is provided, the offset
2482 /// must be subtracted from the provided pointer to get the pointer with the
2483 /// specified alignment.
2484 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2485 unsigned Alignment,
2486 Value *OffsetValue = nullptr);
2487
2488 /// Create an assume intrinsic call that represents an alignment
2489 /// assumption on the provided pointer.
2490 ///
2491 /// An optional offset can be provided, and if it is provided, the offset
2492 /// must be subtracted from the provided pointer to get the pointer with the
2493 /// specified alignment.
2494 ///
2495 /// This overload handles the condition where the Alignment is dependent
2496 /// on an existing value rather than a static value.
2497 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2498 Value *Alignment,
2499 Value *OffsetValue = nullptr);
2500};
2501
2502/// This provides a uniform API for creating instructions and inserting
2503/// them into a basic block: either at the end of a BasicBlock, or at a specific
2504/// iterator location in a block.
2505///
2506/// Note that the builder does not expose the full generality of LLVM
2507/// instructions. For access to extra instruction properties, use the mutators
2508/// (e.g. setVolatile) on the instructions after they have been
2509/// created. Convenience state exists to specify fast-math flags and fp-math
2510/// tags.
2511///
2512/// The first template argument specifies a class to use for creating constants.
2513/// This defaults to creating minimally folded constants. The second template
2514/// argument allows clients to specify custom insertion hooks that are called on
2515/// every newly created insertion.
2516template <typename FolderTy = ConstantFolder,
2517 typename InserterTy = IRBuilderDefaultInserter>
2518class IRBuilder : public IRBuilderBase {
2519private:
2520 FolderTy Folder;
2521 InserterTy Inserter;
2522
2523public:
2524 IRBuilder(LLVMContext &C, FolderTy Folder, InserterTy Inserter = InserterTy(),
2525 MDNode *FPMathTag = nullptr,
2526 ArrayRef<OperandBundleDef> OpBundles = None)
2527 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles),
2528 Folder(Folder), Inserter(Inserter) {}
2529
2530 explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr,
2531 ArrayRef<OperandBundleDef> OpBundles = None)
2532 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles) {}
2533
2534 explicit IRBuilder(BasicBlock *TheBB, FolderTy Folder,
2535 MDNode *FPMathTag = nullptr,
2536 ArrayRef<OperandBundleDef> OpBundles = None)
2537 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2538 FPMathTag, OpBundles), Folder(Folder) {
2539 SetInsertPoint(TheBB);
2540 }
2541
2542 explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr,
2543 ArrayRef<OperandBundleDef> OpBundles = None)
2544 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2545 FPMathTag, OpBundles) {
2546 SetInsertPoint(TheBB);
2547 }
2548
2549 explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr,
2550 ArrayRef<OperandBundleDef> OpBundles = None)
2551 : IRBuilderBase(IP->getContext(), this->Folder, this->Inserter,
22
Called C++ object pointer is null
2552 FPMathTag, OpBundles) {
2553 SetInsertPoint(IP);
2554 }
2555
2556 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, FolderTy Folder,
2557 MDNode *FPMathTag = nullptr,
2558 ArrayRef<OperandBundleDef> OpBundles = None)
2559 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2560 FPMathTag, OpBundles), Folder(Folder) {
2561 SetInsertPoint(TheBB, IP);
2562 }
2563
2564 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP,
2565 MDNode *FPMathTag = nullptr,
2566 ArrayRef<OperandBundleDef> OpBundles = None)
2567 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2568 FPMathTag, OpBundles) {
2569 SetInsertPoint(TheBB, IP);
2570 }
2571
2572 /// Avoid copying the full IRBuilder. Prefer using InsertPointGuard
2573 /// or FastMathFlagGuard instead.
2574 IRBuilder(const IRBuilder &) = delete;
2575
2576 InserterTy &getInserter() { return Inserter; }
2577};
2578
2579// Create wrappers for C Binding types (see CBindingWrapping.h).
2580DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast
<IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const
IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef
>(const_cast<IRBuilder<>*>(P)); }
2581
2582} // end namespace llvm
2583
2584#endif // LLVM_IR_IRBUILDER_H