Bug Summary

File:build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include/llvm/IR/IRBuilder.h
Warning:line 2557, column 23
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name X86LowerAMXType.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/X86 -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/X86 -I include -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-10-03-140002-15933-1 -x c++ /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/X86/X86LowerAMXType.cpp

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/X86/X86LowerAMXType.cpp

1//===- Target/X86/X86LowerAMXType.cpp - -------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to transform <256 x i32> load/store
10/// <256 x i32> is bitcasted to x86_amx on X86, and AMX instruction set only
11/// provides simple operation on x86_amx. The basic elementwise operation
12/// is not supported by AMX. Since x86_amx is bitcasted from vector <256 x i32>
13/// and only AMX intrinsics can operate on the type, we need transform
14/// load/store <256 x i32> instruction to AMX load/store. If the bitcast can
15/// not be combined with load/store, we transform the bitcast to amx load/store
16/// and <256 x i32> store/load.
17///
18/// If Front End not use O0 but the Mid/Back end use O0, (e.g. "Clang -O2 -S
19/// -emit-llvm t.c" + "llc t.ll") we should make sure the amx data is volatile,
20/// because that is necessary for AMX fast register allocation. (In Fast
21/// registera allocation, register will be allocated before spill/reload, so
22/// there is no additional register for amx to identify the step in spill.)
23/// The volatileTileData() will handle this case.
24/// e.g.
25/// ----------------------------------------------------------
26/// | def %td = ... |
27/// | ... |
28/// | "use %td" |
29/// ----------------------------------------------------------
30/// will transfer to -->
31/// ----------------------------------------------------------
32/// | def %td = ... |
33/// | call void @llvm.x86.tilestored64.internal(mem, %td) |
34/// | ... |
35/// | %td2 = call x86_amx @llvm.x86.tileloadd64.internal(mem)|
36/// | "use %td2" |
37/// ----------------------------------------------------------
38//
39//===----------------------------------------------------------------------===//
40//
41#include "X86.h"
42#include "llvm/ADT/PostOrderIterator.h"
43#include "llvm/ADT/SetVector.h"
44#include "llvm/ADT/SmallSet.h"
45#include "llvm/Analysis/OptimizationRemarkEmitter.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/CodeGen/Passes.h"
49#include "llvm/CodeGen/TargetPassConfig.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/IR/DataLayout.h"
52#include "llvm/IR/Function.h"
53#include "llvm/IR/IRBuilder.h"
54#include "llvm/IR/Instructions.h"
55#include "llvm/IR/IntrinsicInst.h"
56#include "llvm/IR/IntrinsicsX86.h"
57#include "llvm/IR/PatternMatch.h"
58#include "llvm/InitializePasses.h"
59#include "llvm/Pass.h"
60#include "llvm/Target/TargetMachine.h"
61#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
62#include "llvm/Transforms/Utils/Local.h"
63
64#include <map>
65
66using namespace llvm;
67using namespace PatternMatch;
68
69#define DEBUG_TYPE"lower-amx-type" "lower-amx-type"
70
71static bool isAMXCast(Instruction *II) {
72 return match(II,
73 m_Intrinsic<Intrinsic::x86_cast_vector_to_tile>(m_Value())) ||
74 match(II, m_Intrinsic<Intrinsic::x86_cast_tile_to_vector>(m_Value()));
75}
76
77static bool isAMXIntrinsic(Value *I) {
78 auto *II = dyn_cast<IntrinsicInst>(I);
79 if (!II)
80 return false;
81 if (isAMXCast(II))
82 return false;
83 // Check if return type or parameter is x86_amx. If it is x86_amx
84 // the intrinsic must be x86 amx intrinsics.
85 if (II->getType()->isX86_AMXTy())
86 return true;
87 for (Value *V : II->args()) {
88 if (V->getType()->isX86_AMXTy())
89 return true;
90 }
91
92 return false;
93}
94
95static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB,
96 Type *Ty) {
97 Function &F = *BB->getParent();
98 Module *M = BB->getModule();
99 const DataLayout &DL = M->getDataLayout();
100
101 LLVMContext &Ctx = Builder.getContext();
102 auto AllocaAlignment = DL.getPrefTypeAlign(Type::getX86_AMXTy(Ctx));
103 unsigned AllocaAS = DL.getAllocaAddrSpace();
104 AllocaInst *AllocaRes =
105 new AllocaInst(Ty, AllocaAS, "", &F.getEntryBlock().front());
106 AllocaRes->setAlignment(AllocaAlignment);
107 return AllocaRes;
108}
109
110static Instruction *getFirstNonAllocaInTheEntryBlock(Function &F) {
111 for (Instruction &I : F.getEntryBlock())
112 if (!isa<AllocaInst>(&I))
113 return &I;
114 llvm_unreachable("No terminator in the entry block!")::llvm::llvm_unreachable_internal("No terminator in the entry block!"
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 114)
;
115}
116
117static std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
118 IRBuilder<> Builder(II);
119 Value *Row = nullptr, *Col = nullptr;
120 switch (II->getIntrinsicID()) {
121 default:
122 llvm_unreachable("Expect amx intrinsics")::llvm::llvm_unreachable_internal("Expect amx intrinsics", "llvm/lib/Target/X86/X86LowerAMXType.cpp"
, 122)
;
123 case Intrinsic::x86_tileloadd64_internal:
124 case Intrinsic::x86_tileloaddt164_internal:
125 case Intrinsic::x86_tilestored64_internal: {
126 Row = II->getArgOperand(0);
127 Col = II->getArgOperand(1);
128 break;
129 }
130 // a * b + c
131 // The shape depends on which operand.
132 case Intrinsic::x86_tdpbssd_internal:
133 case Intrinsic::x86_tdpbsud_internal:
134 case Intrinsic::x86_tdpbusd_internal:
135 case Intrinsic::x86_tdpbuud_internal:
136 case Intrinsic::x86_tdpbf16ps_internal: {
137 switch (OpNo) {
138 case 3:
139 Row = II->getArgOperand(0);
140 Col = II->getArgOperand(1);
141 break;
142 case 4:
143 Row = II->getArgOperand(0);
144 Col = II->getArgOperand(2);
145 break;
146 case 5:
147 if (isa<ConstantInt>(II->getArgOperand(2)))
148 Row = Builder.getInt16(
149 (cast<ConstantInt>(II->getOperand(2))->getSExtValue()) / 4);
150 else if (isa<Instruction>(II->getArgOperand(2))) {
151 // When it is not a const value and it is not a function argument, we
152 // create Row after the definition of II->getOperand(2) instead of
153 // before II. For example, II is %118, we try to getshape for %117:
154 // %117 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x
155 // i32> %115).
156 // %118 = call x86_amx @llvm.x86.tdpbf16ps.internal(i16
157 // %104, i16 %105, i16 %106, x86_amx %110, x86_amx %114, x86_amx
158 // %117).
159 // If we create %row = udiv i16 %106, 4 before %118(aka. II), then its
160 // definition is after its user(new tileload for %117).
161 // So, the best choice is to create %row right after the definition of
162 // %106.
163 Builder.SetInsertPoint(cast<Instruction>(II->getOperand(2)));
164 Row = Builder.CreateUDiv(II->getOperand(2), Builder.getInt16(4));
165 cast<Instruction>(Row)->moveAfter(cast<Instruction>(II->getOperand(2)));
166 } else {
167 // When it is not a const value and it is a function argument, we create
168 // Row at the entry bb.
169 IRBuilder<> NewBuilder(
170 getFirstNonAllocaInTheEntryBlock(*II->getFunction()));
171 Row = NewBuilder.CreateUDiv(II->getOperand(2), NewBuilder.getInt16(4));
172 }
173 Col = II->getArgOperand(1);
174 break;
175 }
176 break;
177 }
178 }
179
180 return std::make_pair(Row, Col);
181}
182
183static std::pair<Value *, Value *> getShape(PHINode *Phi) {
184 Use &U = *(Phi->use_begin());
185 unsigned OpNo = U.getOperandNo();
186 User *V = U.getUser();
187 // TODO We don't traverse all users. To make the algorithm simple, here we
188 // just traverse the first user. If we can find shape, then return the shape,
189 // otherwise just return nullptr and the optimization for undef/zero will be
190 // abandoned.
191 while (V) {
192 if (isAMXCast(dyn_cast<Instruction>(V))) {
193 if (V->use_empty())
194 break;
195 Use &U = *(V->use_begin());
196 OpNo = U.getOperandNo();
197 V = U.getUser();
198 } else if (isAMXIntrinsic(V)) {
199 return getShape(cast<IntrinsicInst>(V), OpNo);
200 } else if (isa<PHINode>(V)) {
201 if (V->use_empty())
202 break;
203 Use &U = *(V->use_begin());
204 V = U.getUser();
205 } else {
206 break;
207 }
208 }
209
210 return std::make_pair(nullptr, nullptr);
211}
212
213namespace {
214class X86LowerAMXType {
215 Function &Func;
216
217 // In AMX intrinsics we let Shape = {Row, Col}, but the
218 // RealCol = Col / ElementSize. We may use the RealCol
219 // as a new Row for other new created AMX intrinsics.
220 std::map<Value *, Value *> Col2Row;
221
222public:
223 X86LowerAMXType(Function &F) : Func(F) {}
224 bool visit();
225 void combineLoadBitcast(LoadInst *LD, BitCastInst *Bitcast);
226 void combineBitcastStore(BitCastInst *Bitcast, StoreInst *ST);
227 bool transformBitcast(BitCastInst *Bitcast);
228};
229
230// %src = load <256 x i32>, <256 x i32>* %addr, align 64
231// %2 = bitcast <256 x i32> %src to x86_amx
232// -->
233// %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
234// i8* %addr, i64 %stride64)
235void X86LowerAMXType::combineLoadBitcast(LoadInst *LD, BitCastInst *Bitcast) {
236 Value *Row = nullptr, *Col = nullptr;
237 Use &U = *(Bitcast->use_begin());
238 unsigned OpNo = U.getOperandNo();
239 auto *II = cast<IntrinsicInst>(U.getUser());
240 std::tie(Row, Col) = getShape(II, OpNo);
241 IRBuilder<> Builder(Bitcast);
242 // Use the maximun column as stride.
243 Value *Stride = Builder.getInt64(64);
244 Value *I8Ptr =
245 Builder.CreateBitCast(LD->getOperand(0), Builder.getInt8PtrTy());
246 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
247
248 Value *NewInst =
249 Builder.CreateIntrinsic(Intrinsic::x86_tileloadd64_internal, None, Args);
250 Bitcast->replaceAllUsesWith(NewInst);
251}
252
253// %src = call x86_amx @llvm.x86.tileloadd64.internal(%row, %col, %addr,
254// %stride);
255// %13 = bitcast x86_amx %src to <256 x i32>
256// store <256 x i32> %13, <256 x i32>* %addr, align 64
257// -->
258// call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
259// %stride64, %13)
260void X86LowerAMXType::combineBitcastStore(BitCastInst *Bitcast, StoreInst *ST) {
261
262 Value *Tile = Bitcast->getOperand(0);
263 auto *II = cast<IntrinsicInst>(Tile);
264 // Tile is output from AMX intrinsic. The first operand of the
265 // intrinsic is row, the second operand of the intrinsic is column.
266 Value *Row = II->getOperand(0);
267 Value *Col = II->getOperand(1);
268 IRBuilder<> Builder(ST);
269 // Use the maximum column as stride. It must be the same with load
270 // stride.
271 Value *Stride = Builder.getInt64(64);
272 Value *I8Ptr =
273 Builder.CreateBitCast(ST->getOperand(1), Builder.getInt8PtrTy());
274 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Tile};
275 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
276 if (Bitcast->hasOneUse())
277 return;
278 // %13 = bitcast x86_amx %src to <256 x i32>
279 // store <256 x i32> %13, <256 x i32>* %addr, align 64
280 // %add = <256 x i32> %13, <256 x i32> %src2
281 // -->
282 // %13 = bitcast x86_amx %src to <256 x i32>
283 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
284 // %stride64, %13)
285 // %14 = load <256 x i32>, %addr
286 // %add = <256 x i32> %14, <256 x i32> %src2
287 Value *Vec = Builder.CreateLoad(Bitcast->getType(), ST->getOperand(1));
288 Bitcast->replaceAllUsesWith(Vec);
289}
290
291// transform bitcast to <store, load> instructions.
292bool X86LowerAMXType::transformBitcast(BitCastInst *Bitcast) {
293 IRBuilder<> Builder(Bitcast);
294 AllocaInst *AllocaAddr;
295 Value *I8Ptr, *Stride;
296 auto *Src = Bitcast->getOperand(0);
297
298 auto Prepare = [&](Type *MemTy) {
299 AllocaAddr = createAllocaInstAtEntry(Builder, Bitcast->getParent(), MemTy);
300 I8Ptr = Builder.CreateBitCast(AllocaAddr, Builder.getInt8PtrTy());
301 Stride = Builder.getInt64(64);
302 };
303
304 if (Bitcast->getType()->isX86_AMXTy()) {
305 // %2 = bitcast <256 x i32> %src to x86_amx
306 // -->
307 // %addr = alloca <256 x i32>, align 64
308 // store <256 x i32> %src, <256 x i32>* %addr, align 64
309 // %addr2 = bitcast <256 x i32>* to i8*
310 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
311 // i8* %addr2,
312 // i64 64)
313 Use &U = *(Bitcast->use_begin());
314 unsigned OpNo = U.getOperandNo();
315 auto *II = dyn_cast<IntrinsicInst>(U.getUser());
316 if (!II)
317 return false; // May be bitcast from x86amx to <256 x i32>.
318 Prepare(Bitcast->getOperand(0)->getType());
319 Builder.CreateStore(Src, AllocaAddr);
320 // TODO we can pick an constant operand for the shape.
321 Value *Row = nullptr, *Col = nullptr;
322 std::tie(Row, Col) = getShape(II, OpNo);
323 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
324 Value *NewInst = Builder.CreateIntrinsic(
325 Intrinsic::x86_tileloadd64_internal, None, Args);
326 Bitcast->replaceAllUsesWith(NewInst);
327 } else {
328 // %2 = bitcast x86_amx %src to <256 x i32>
329 // -->
330 // %addr = alloca <256 x i32>, align 64
331 // %addr2 = bitcast <256 x i32>* to i8*
332 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col,
333 // i8* %addr2, i64 %stride)
334 // %2 = load <256 x i32>, <256 x i32>* %addr, align 64
335 auto *II = dyn_cast<IntrinsicInst>(Src);
336 if (!II)
337 return false; // May be bitcast from <256 x i32> to x86amx.
338 Prepare(Bitcast->getType());
339 Value *Row = II->getOperand(0);
340 Value *Col = II->getOperand(1);
341 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Src};
342 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
343 Value *NewInst = Builder.CreateLoad(Bitcast->getType(), AllocaAddr);
344 Bitcast->replaceAllUsesWith(NewInst);
345 }
346
347 return true;
348}
349
350bool X86LowerAMXType::visit() {
351 SmallVector<Instruction *, 8> DeadInsts;
352 Col2Row.clear();
353
354 for (BasicBlock *BB : post_order(&Func)) {
355 for (Instruction &Inst : llvm::make_early_inc_range(llvm::reverse(*BB))) {
356 auto *Bitcast = dyn_cast<BitCastInst>(&Inst);
357 if (!Bitcast)
358 continue;
359
360 Value *Src = Bitcast->getOperand(0);
361 if (Bitcast->getType()->isX86_AMXTy()) {
362 if (Bitcast->user_empty()) {
363 DeadInsts.push_back(Bitcast);
364 continue;
365 }
366 LoadInst *LD = dyn_cast<LoadInst>(Src);
367 if (!LD) {
368 if (transformBitcast(Bitcast))
369 DeadInsts.push_back(Bitcast);
370 continue;
371 }
372 // If load has mutli-user, duplicate a vector load.
373 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
374 // %2 = bitcast <256 x i32> %src to x86_amx
375 // %add = add <256 x i32> %src, <256 x i32> %src2
376 // -->
377 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
378 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
379 // i8* %addr, i64 %stride64)
380 // %add = add <256 x i32> %src, <256 x i32> %src2
381
382 // If load has one user, the load will be eliminated in DAG ISel.
383 // %src = load <256 x i32>, <256 x i32>* %addr, align 64
384 // %2 = bitcast <256 x i32> %src to x86_amx
385 // -->
386 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
387 // i8* %addr, i64 %stride64)
388 combineLoadBitcast(LD, Bitcast);
389 DeadInsts.push_back(Bitcast);
390 if (LD->hasOneUse())
391 DeadInsts.push_back(LD);
392 } else if (Src->getType()->isX86_AMXTy()) {
393 if (Bitcast->user_empty()) {
394 DeadInsts.push_back(Bitcast);
395 continue;
396 }
397 StoreInst *ST = nullptr;
398 for (Use &U : Bitcast->uses()) {
399 ST = dyn_cast<StoreInst>(U.getUser());
400 if (ST)
401 break;
402 }
403 if (!ST) {
404 if (transformBitcast(Bitcast))
405 DeadInsts.push_back(Bitcast);
406 continue;
407 }
408 // If bitcast (%13) has one use, combine bitcast and store to amx store.
409 // %src = call x86_amx @llvm.x86.tileloadd64.internal(%row, %col, %addr,
410 // %stride);
411 // %13 = bitcast x86_amx %src to <256 x i32>
412 // store <256 x i32> %13, <256 x i32>* %addr, align 64
413 // -->
414 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
415 // %stride64, %13)
416 //
417 // If bitcast (%13) has multi-use, transform as below.
418 // %13 = bitcast x86_amx %src to <256 x i32>
419 // store <256 x i32> %13, <256 x i32>* %addr, align 64
420 // %add = <256 x i32> %13, <256 x i32> %src2
421 // -->
422 // %13 = bitcast x86_amx %src to <256 x i32>
423 // call void @llvm.x86.tilestored64.internal(%row, %col, %addr,
424 // %stride64, %13)
425 // %14 = load <256 x i32>, %addr
426 // %add = <256 x i32> %14, <256 x i32> %src2
427 //
428 combineBitcastStore(Bitcast, ST);
429 // Delete user first.
430 DeadInsts.push_back(ST);
431 DeadInsts.push_back(Bitcast);
432 }
433 }
434 }
435
436 bool C = !DeadInsts.empty();
437
438 for (auto *Inst : DeadInsts)
439 Inst->eraseFromParent();
440
441 return C;
442}
443} // anonymous namespace
444
445static Value *getAllocaPos(BasicBlock *BB) {
446 Module *M = BB->getModule();
447 Function *F = BB->getParent();
448 IRBuilder<> Builder(&F->getEntryBlock().front());
449 const DataLayout &DL = M->getDataLayout();
450 unsigned AllocaAS = DL.getAllocaAddrSpace();
451 Type *V256I32Ty = VectorType::get(Builder.getInt32Ty(), 256, false);
452 AllocaInst *AllocaRes =
453 new AllocaInst(V256I32Ty, AllocaAS, "", &F->getEntryBlock().front());
454 BasicBlock::iterator Iter = AllocaRes->getIterator();
455 ++Iter;
456 Builder.SetInsertPoint(&*Iter);
457 Value *I8Ptr = Builder.CreateBitCast(AllocaRes, Builder.getInt8PtrTy());
458 return I8Ptr;
459}
460
461static Instruction *createTileStore(Instruction *TileDef, Value *Ptr) {
462 assert(TileDef->getType()->isX86_AMXTy() && "Not define tile!")(static_cast <bool> (TileDef->getType()->isX86_AMXTy
() && "Not define tile!") ? void (0) : __assert_fail (
"TileDef->getType()->isX86_AMXTy() && \"Not define tile!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 462, __extension__
__PRETTY_FUNCTION__))
;
463 auto *II = cast<IntrinsicInst>(TileDef);
464 assert(II && "Not tile intrinsic!")(static_cast <bool> (II && "Not tile intrinsic!"
) ? void (0) : __assert_fail ("II && \"Not tile intrinsic!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 464, __extension__
__PRETTY_FUNCTION__))
;
465 Value *Row = II->getOperand(0);
466 Value *Col = II->getOperand(1);
467
468 BasicBlock *BB = TileDef->getParent();
469 BasicBlock::iterator Iter = TileDef->getIterator();
470 IRBuilder<> Builder(BB, ++Iter);
471 Value *Stride = Builder.getInt64(64);
472 std::array<Value *, 5> Args = {Row, Col, Ptr, Stride, TileDef};
473
474 Instruction *TileStore =
475 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
476 return TileStore;
477}
478
479static void replaceWithTileLoad(Use &U, Value *Ptr, bool IsPHI = false) {
480 Value *V = U.get();
481 assert(V->getType()->isX86_AMXTy() && "Not define tile!")(static_cast <bool> (V->getType()->isX86_AMXTy() &&
"Not define tile!") ? void (0) : __assert_fail ("V->getType()->isX86_AMXTy() && \"Not define tile!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 481, __extension__
__PRETTY_FUNCTION__))
;
14
'?' condition is true
482
483 // Get tile shape.
484 IntrinsicInst *II = nullptr;
485 if (IsPHI
14.1
'IsPHI' is true
14.1
'IsPHI' is true
) {
15
Taking true branch
486 Value *PhiOp = dyn_cast<PHINode>(V)->getIncomingValue(0);
16
Assuming 'V' is a 'CastReturnType'
487 II = cast<IntrinsicInst>(PhiOp);
17
'PhiOp' is a 'CastReturnType'
488 } else {
489 II = cast<IntrinsicInst>(V);
490 }
491 Value *Row = II->getOperand(0);
492 Value *Col = II->getOperand(1);
493
494 Instruction *UserI = dyn_cast<Instruction>(U.getUser());
18
Assuming the object is not a 'CastReturnType'
19
'UserI' initialized to a null pointer value
495 IRBuilder<> Builder(UserI);
20
Passing null pointer value via 1st parameter 'IP'
21
Calling constructor for 'IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>'
496 Value *Stride = Builder.getInt64(64);
497 std::array<Value *, 4> Args = {Row, Col, Ptr, Stride};
498
499 Value *TileLoad =
500 Builder.CreateIntrinsic(Intrinsic::x86_tileloadd64_internal, None, Args);
501 UserI->replaceUsesOfWith(V, TileLoad);
502}
503
504static bool isIncomingOfPHI(Instruction *I) {
505 for (Use &U : I->uses()) {
506 User *V = U.getUser();
507 if (isa<PHINode>(V))
508 return true;
509 }
510 return false;
511}
512
513// Let all AMX tile data become volatile data, shorten the life range
514// of each tile register before fast register allocation.
515namespace {
516class X86VolatileTileData {
517 Function &F;
518
519public:
520 X86VolatileTileData(Function &Func) : F(Func) {}
521 Value *updatePhiIncomings(BasicBlock *BB,
522 SmallVector<Instruction *, 2> &Incomings);
523 void replacePhiDefWithLoad(Instruction *PHI, Value *StorePtr);
524 bool volatileTileData();
525 void volatileTilePHI(PHINode *Inst);
526 void volatileTileNonPHI(Instruction *I);
527};
528
529Value *X86VolatileTileData::updatePhiIncomings(
530 BasicBlock *BB, SmallVector<Instruction *, 2> &Incomings) {
531 Value *I8Ptr = getAllocaPos(BB);
532
533 for (auto *I : Incomings) {
534 User *Store = createTileStore(I, I8Ptr);
535
536 // All its uses (except phi) should load from stored mem.
537 for (Use &U : I->uses()) {
538 User *V = U.getUser();
539 if (isa<PHINode>(V) || V == Store)
540 continue;
541 replaceWithTileLoad(U, I8Ptr);
542 }
543 }
544 return I8Ptr;
545}
546
547void X86VolatileTileData::replacePhiDefWithLoad(Instruction *PHI,
548 Value *StorePtr) {
549 for (Use &U : PHI->uses())
550 replaceWithTileLoad(U, StorePtr, true);
13
Calling 'replaceWithTileLoad'
551 PHI->eraseFromParent();
552}
553
554// Smilar with volatileTileNonPHI, this function only handle PHI Nodes
555// and their related AMX intrinsics.
556// 1) PHI Def should change to tileload.
557// 2) PHI Incoming Values should tilestored in just after their def.
558// 3) The mem of these tileload and tilestores should be same.
559// e.g.
560// ------------------------------------------------------
561// bb_dom:
562// ...
563// br i1 %bool.cond, label %if.else, label %if.then
564//
565// if.then:
566// def %t0 = ...
567// ...
568// use %t0
569// ...
570// br label %if.end
571//
572// if.else:
573// def %t1 = ...
574// br label %if.end
575//
576// if.end:
577// %td = phi x86_amx [ %t1, %if.else ], [ %t0, %if.then ]
578// ...
579// use %td
580// ------------------------------------------------------
581// -->
582// ------------------------------------------------------
583// bb_entry:
584// %mem = alloca <256 x i32>, align 1024 *
585// ...
586// bb_dom:
587// ...
588// br i1 %bool.cond, label %if.else, label %if.then
589//
590// if.then:
591// def %t0 = ...
592// call void @llvm.x86.tilestored64.internal(mem, %t0) *
593// ...
594// %t0` = call x86_amx @llvm.x86.tileloadd64.internal(mem)*
595// use %t0` *
596// ...
597// br label %if.end
598//
599// if.else:
600// def %t1 = ...
601// call void @llvm.x86.tilestored64.internal(mem, %t1) *
602// br label %if.end
603//
604// if.end:
605// ...
606// %td = call x86_amx @llvm.x86.tileloadd64.internal(mem) *
607// use %td
608// ------------------------------------------------------
609void X86VolatileTileData::volatileTilePHI(PHINode *PHI) {
610 BasicBlock *BB = PHI->getParent();
611 SmallVector<Instruction *, 2> Incomings;
612
613 for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) {
10
Assuming 'I' is equal to 'E'
11
Loop condition is false. Execution continues on line 620
614 Value *Op = PHI->getIncomingValue(I);
615 Instruction *Inst = dyn_cast<Instruction>(Op);
616 assert(Inst && "We shouldn't fold AMX instrution!")(static_cast <bool> (Inst && "We shouldn't fold AMX instrution!"
) ? void (0) : __assert_fail ("Inst && \"We shouldn't fold AMX instrution!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 616, __extension__
__PRETTY_FUNCTION__))
;
617 Incomings.push_back(Inst);
618 }
619
620 Value *StorePtr = updatePhiIncomings(BB, Incomings);
621 replacePhiDefWithLoad(PHI, StorePtr);
12
Calling 'X86VolatileTileData::replacePhiDefWithLoad'
622}
623
624// Store the defined tile and load it before use.
625// All its users are not PHI.
626// e.g.
627// ------------------------------------------------------
628// def %td = ...
629// ...
630// "use %td"
631// ------------------------------------------------------
632// -->
633// ------------------------------------------------------
634// def %td = ...
635// call void @llvm.x86.tilestored64.internal(mem, %td)
636// ...
637// %td2 = call x86_amx @llvm.x86.tileloadd64.internal(mem)
638// "use %td2"
639// ------------------------------------------------------
640void X86VolatileTileData::volatileTileNonPHI(Instruction *I) {
641 BasicBlock *BB = I->getParent();
642 Value *I8Ptr = getAllocaPos(BB);
643 User *Store = createTileStore(I, I8Ptr);
644
645 // All its uses should load from stored mem.
646 for (Use &U : I->uses()) {
647 User *V = U.getUser();
648 assert(!isa<PHINode>(V) && "PHI Nodes should be excluded!")(static_cast <bool> (!isa<PHINode>(V) && "PHI Nodes should be excluded!"
) ? void (0) : __assert_fail ("!isa<PHINode>(V) && \"PHI Nodes should be excluded!\""
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 648, __extension__
__PRETTY_FUNCTION__))
;
649 if (V != Store)
650 replaceWithTileLoad(U, I8Ptr);
651 }
652}
653
654// Volatile Tile Model:
655// 1) All the uses of tile data comes from tileload in time.
656// 2) All the defs of tile data tilestore into mem immediately.
657// For example:
658// --------------------------------------------------------------------------
659// %t1 = call x86_amx @llvm.x86.tileloadd64.internal(m, k, ...) key
660// %t2 = call x86_amx @llvm.x86.tileloadd64.internal(k, n, ...)
661// %t3 = call x86_amx @llvm.x86.tileloadd64.internal(m, n, ...) amx
662// %td = tail call x86_amx @llvm.x86.tdpbssd.internal(m, n, k, t1, t2, t3)
663// call void @llvm.x86.tilestored64.internal(... td) area
664// --------------------------------------------------------------------------
665// 3) No terminator, call or other amx instructions in the key amx area.
666bool X86VolatileTileData::volatileTileData() {
667 bool Changed = false;
668 for (BasicBlock &BB : F) {
669 SmallVector<Instruction *, 2> PHIInsts;
670 SmallVector<Instruction *, 8> AMXDefInsts;
671
672 for (Instruction &I : BB) {
673 if (!I.getType()->isX86_AMXTy())
674 continue;
675 if (isa<PHINode>(&I))
676 PHIInsts.push_back(&I);
677 else
678 AMXDefInsts.push_back(&I);
679 }
680
681 // First we "volatile" the non-phi related amx intrinsics.
682 for (Instruction *I : AMXDefInsts) {
6
Assuming '__begin2' is equal to '__end2'
683 if (isIncomingOfPHI(I))
684 continue;
685 volatileTileNonPHI(I);
686 Changed = true;
687 }
688
689 for (Instruction *I : PHIInsts) {
7
Assuming '__begin2' is not equal to '__end2'
690 volatileTilePHI(dyn_cast<PHINode>(I));
8
Assuming 'I' is a 'CastReturnType'
9
Calling 'X86VolatileTileData::volatileTilePHI'
691 Changed = true;
692 }
693 }
694 return Changed;
695}
696
697} // anonymous namespace
698
699namespace {
700
701class X86LowerAMXCast {
702 Function &Func;
703
704public:
705 X86LowerAMXCast(Function &F) : Func(F) {}
706 void combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
707 void combineLoadCast(IntrinsicInst *Cast, LoadInst *LD);
708 bool combineLdSt(SmallVectorImpl<Instruction *> &Casts);
709 bool combineAMXcast(TargetLibraryInfo *TLI);
710 bool transformAMXCast(IntrinsicInst *AMXCast);
711 bool transformAllAMXCast();
712 bool optimizeAMXCastFromPhi(IntrinsicInst *CI, PHINode *PN,
713 SmallSetVector<Instruction *, 16> &DeadInst);
714};
715
716static bool DCEInstruction(Instruction *I,
717 SmallSetVector<Instruction *, 16> &WorkList,
718 const TargetLibraryInfo *TLI) {
719 if (isInstructionTriviallyDead(I, TLI)) {
720 salvageDebugInfo(*I);
721 salvageKnowledge(I);
722
723 // Null out all of the instruction's operands to see if any operand becomes
724 // dead as we go.
725 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
726 Value *OpV = I->getOperand(i);
727 I->setOperand(i, nullptr);
728
729 if (!OpV->use_empty() || I == OpV)
730 continue;
731
732 // If the operand is an instruction that became dead as we nulled out the
733 // operand, and if it is 'trivially' dead, delete it in a future loop
734 // iteration.
735 if (Instruction *OpI = dyn_cast<Instruction>(OpV)) {
736 if (isInstructionTriviallyDead(OpI, TLI)) {
737 WorkList.insert(OpI);
738 }
739 }
740 }
741 I->eraseFromParent();
742 return true;
743 }
744 return false;
745}
746
747/// This function handles following case
748///
749/// A -> B amxcast
750/// PHI
751/// B -> A amxcast
752///
753/// All the related PHI nodes can be replaced by new PHI nodes with type A.
754/// The uses of \p CI can be changed to the new PHI node corresponding to \p PN.
755bool X86LowerAMXCast::optimizeAMXCastFromPhi(
756 IntrinsicInst *CI, PHINode *PN,
757 SmallSetVector<Instruction *, 16> &DeadInst) {
758 IRBuilder<> Builder(CI);
759 Value *Src = CI->getOperand(0);
760 Type *SrcTy = Src->getType(); // Type B
761 Type *DestTy = CI->getType(); // Type A
762
763 SmallVector<PHINode *, 4> PhiWorklist;
764 SmallSetVector<PHINode *, 4> OldPhiNodes;
765
766 // Find all of the A->B casts and PHI nodes.
767 // We need to inspect all related PHI nodes, but PHIs can be cyclic, so
768 // OldPhiNodes is used to track all known PHI nodes, before adding a new
769 // PHI to PhiWorklist, it is checked against and added to OldPhiNodes first.
770 PhiWorklist.push_back(PN);
771 OldPhiNodes.insert(PN);
772 while (!PhiWorklist.empty()) {
773 auto *OldPN = PhiWorklist.pop_back_val();
774 for (unsigned I = 0; I < OldPN->getNumOperands(); ++I) {
775 Value *IncValue = OldPN->getIncomingValue(I);
776 // TODO: currently, We ignore cases where it is a const. In the future, we
777 // might support const.
778 if (isa<Constant>(IncValue)) {
779 auto *IncConst = dyn_cast<Constant>(IncValue);
780 if (!isa<UndefValue>(IncValue) && !IncConst->isZeroValue())
781 return false;
782 Value *Row = nullptr, *Col = nullptr;
783 std::tie(Row, Col) = getShape(OldPN);
784 // TODO: If it is not constant the Row and Col must domoniate tilezero
785 // that we are going to create.
786 if (!Row || !Col || !isa<Constant>(Row) || !isa<Constant>(Col))
787 return false;
788 // Create tilezero at the end of incoming block.
789 auto *Block = OldPN->getIncomingBlock(I);
790 BasicBlock::iterator Iter = Block->getTerminator()->getIterator();
791 Instruction *NewInst = Builder.CreateIntrinsic(
792 Intrinsic::x86_tilezero_internal, None, {Row, Col});
793 NewInst->moveBefore(&*Iter);
794 NewInst = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector,
795 {IncValue->getType()}, {NewInst});
796 NewInst->moveBefore(&*Iter);
797 // Replace InValue with new Value.
798 OldPN->setIncomingValue(I, NewInst);
799 IncValue = NewInst;
800 }
801
802 if (auto *PNode = dyn_cast<PHINode>(IncValue)) {
803 if (OldPhiNodes.insert(PNode))
804 PhiWorklist.push_back(PNode);
805 continue;
806 }
807 Instruction *ACI = dyn_cast<Instruction>(IncValue);
808 if (ACI && isAMXCast(ACI)) {
809 // Verify it's a A->B cast.
810 Type *TyA = ACI->getOperand(0)->getType();
811 Type *TyB = ACI->getType();
812 if (TyA != DestTy || TyB != SrcTy)
813 return false;
814 continue;
815 }
816 return false;
817 }
818 }
819
820 // Check that each user of each old PHI node is something that we can
821 // rewrite, so that all of the old PHI nodes can be cleaned up afterwards.
822 for (auto *OldPN : OldPhiNodes) {
823 for (User *V : OldPN->users()) {
824 Instruction *ACI = dyn_cast<Instruction>(V);
825 if (ACI && isAMXCast(ACI)) {
826 // Verify it's a B->A cast.
827 Type *TyB = ACI->getOperand(0)->getType();
828 Type *TyA = ACI->getType();
829 if (TyA != DestTy || TyB != SrcTy)
830 return false;
831 } else if (auto *PHI = dyn_cast<PHINode>(V)) {
832 // As long as the user is another old PHI node, then even if we don't
833 // rewrite it, the PHI web we're considering won't have any users
834 // outside itself, so it'll be dead.
835 // example:
836 // bb.0:
837 // %0 = amxcast ...
838 // bb.1:
839 // %1 = amxcast ...
840 // bb.2:
841 // %goodphi = phi %0, %1
842 // %3 = amxcast %goodphi
843 // bb.3:
844 // %goodphi2 = phi %0, %goodphi
845 // %4 = amxcast %goodphi2
846 // When optimizeAMXCastFromPhi process %3 and %goodphi, %goodphi2 is
847 // outside the phi-web, so the combination stop When
848 // optimizeAMXCastFromPhi process %4 and %goodphi2, the optimization
849 // will be done.
850 if (OldPhiNodes.count(PHI) == 0)
851 return false;
852 } else
853 return false;
854 }
855 }
856
857 // For each old PHI node, create a corresponding new PHI node with a type A.
858 SmallDenseMap<PHINode *, PHINode *> NewPNodes;
859 for (auto *OldPN : OldPhiNodes) {
860 Builder.SetInsertPoint(OldPN);
861 PHINode *NewPN = Builder.CreatePHI(DestTy, OldPN->getNumOperands());
862 NewPNodes[OldPN] = NewPN;
863 }
864
865 // Fill in the operands of new PHI nodes.
866 for (auto *OldPN : OldPhiNodes) {
867 PHINode *NewPN = NewPNodes[OldPN];
868 for (unsigned j = 0, e = OldPN->getNumOperands(); j != e; ++j) {
869 Value *V = OldPN->getOperand(j);
870 Value *NewV = nullptr;
871 Instruction *ACI = dyn_cast<Instruction>(V);
872 // There should not be a AMXcast from a const.
873 if (ACI && isAMXCast(ACI))
874 NewV = ACI->getOperand(0);
875 else if (auto *PrevPN = dyn_cast<PHINode>(V))
876 NewV = NewPNodes[PrevPN];
877 assert(NewV)(static_cast <bool> (NewV) ? void (0) : __assert_fail (
"NewV", "llvm/lib/Target/X86/X86LowerAMXType.cpp", 877, __extension__
__PRETTY_FUNCTION__))
;
878 NewPN->addIncoming(NewV, OldPN->getIncomingBlock(j));
879 }
880 }
881
882 // Traverse all accumulated PHI nodes and process its users,
883 // which are Stores and BitcCasts. Without this processing
884 // NewPHI nodes could be replicated and could lead to extra
885 // moves generated after DeSSA.
886 // If there is a store with type B, change it to type A.
887
888 // Replace users of BitCast B->A with NewPHI. These will help
889 // later to get rid of a closure formed by OldPHI nodes.
890 for (auto *OldPN : OldPhiNodes) {
891 PHINode *NewPN = NewPNodes[OldPN];
892 for (User *V : make_early_inc_range(OldPN->users())) {
893 Instruction *ACI = dyn_cast<Instruction>(V);
894 if (ACI && isAMXCast(ACI)) {
895 Type *TyB = ACI->getOperand(0)->getType();
896 Type *TyA = ACI->getType();
897 assert(TyA == DestTy && TyB == SrcTy)(static_cast <bool> (TyA == DestTy && TyB == SrcTy
) ? void (0) : __assert_fail ("TyA == DestTy && TyB == SrcTy"
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 897, __extension__
__PRETTY_FUNCTION__))
;
898 (void)TyA;
899 (void)TyB;
900 ACI->replaceAllUsesWith(NewPN);
901 DeadInst.insert(ACI);
902 } else if (auto *PHI = dyn_cast<PHINode>(V)) {
903 // We don't need to push PHINode into DeadInst since they are operands
904 // of rootPN DCE can safely delete rootPN's operands if rootPN is dead.
905 assert(OldPhiNodes.contains(PHI))(static_cast <bool> (OldPhiNodes.contains(PHI)) ? void (
0) : __assert_fail ("OldPhiNodes.contains(PHI)", "llvm/lib/Target/X86/X86LowerAMXType.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))
;
906 (void)PHI;
907 } else
908 llvm_unreachable("all uses should be handled")::llvm::llvm_unreachable_internal("all uses should be handled"
, "llvm/lib/Target/X86/X86LowerAMXType.cpp", 908)
;
909 }
910 }
911 return true;
912}
913
914// %43 = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %42)
915// store <256 x i32> %43, <256 x i32>* %p, align 64
916// -->
917// call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, i8* %p,
918// i64 64, x86_amx %42)
919void X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
920 Value *Tile = Cast->getOperand(0);
921 // TODO: If it is cast intrinsic or phi node, we can propagate the
922 // shape information through def-use chain.
923 if (!isAMXIntrinsic(Tile))
924 return;
925 auto *II = cast<IntrinsicInst>(Tile);
926 // Tile is output from AMX intrinsic. The first operand of the
927 // intrinsic is row, the second operand of the intrinsic is column.
928 Value *Row = II->getOperand(0);
929 Value *Col = II->getOperand(1);
930 IRBuilder<> Builder(ST);
931 // Use the maximum column as stride. It must be the same with load
932 // stride.
933 Value *Stride = Builder.getInt64(64);
934 Value *I8Ptr =
935 Builder.CreateBitCast(ST->getOperand(1), Builder.getInt8PtrTy());
936 std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Tile};
937 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
938}
939
940// %65 = load <256 x i32>, <256 x i32>* %p, align 64
941// %66 = call x86_amx @llvm.x86.cast.vector.to.tile(<256 x i32> %65)
942// -->
943// %66 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
944// i8* %p, i64 64)
945void X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
946 Value *Row = nullptr, *Col = nullptr;
947 Use &U = *(Cast->use_begin());
948 unsigned OpNo = U.getOperandNo();
949 auto *II = cast<IntrinsicInst>(U.getUser());
950 // TODO: If it is cast intrinsic or phi node, we can propagate the
951 // shape information through def-use chain.
952 if (!isAMXIntrinsic(II))
953 return;
954 std::tie(Row, Col) = getShape(II, OpNo);
955 IRBuilder<> Builder(LD);
956 // Use the maximun column as stride.
957 Value *Stride = Builder.getInt64(64);
958 Value *I8Ptr =
959 Builder.CreateBitCast(LD->getOperand(0), Builder.getInt8PtrTy());
960 std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
961
962 Value *NewInst =
963 Builder.CreateIntrinsic(Intrinsic::x86_tileloadd64_internal, None, Args);
964 Cast->replaceAllUsesWith(NewInst);
965}
966
967bool X86LowerAMXCast::combineLdSt(SmallVectorImpl<Instruction *> &Casts) {
968 bool Change = false;
969 for (auto *Cast : Casts) {
970 auto *II = cast<IntrinsicInst>(Cast);
971 // %43 = call <256 x i32> @llvm.x86.cast.tile.to.vector(x86_amx %42)
972 // store <256 x i32> %43, <256 x i32>* %p, align 64
973 // -->
974 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col, i8* %p,
975 // i64 64, x86_amx %42)
976 if (II->getIntrinsicID() == Intrinsic::x86_cast_tile_to_vector) {
977 SmallVector<Instruction *, 2> DeadStores;
978 for (User *U : Cast->users()) {
979 StoreInst *Store = dyn_cast<StoreInst>(U);
980 if (!Store)
981 continue;
982 combineCastStore(cast<IntrinsicInst>(Cast), Store);
983 DeadStores.push_back(Store);
984 Change = true;
985 }
986 for (auto *Store : DeadStores)
987 Store->eraseFromParent();
988 } else { // x86_cast_vector_to_tile
989 SmallVector<Instruction *, 2> DeadLoads;
990 auto *Load = dyn_cast<LoadInst>(Cast->getOperand(0));
991 if (!Load || !Load->hasOneUse())
992 continue;
993 // %65 = load <256 x i32>, <256 x i32>* %p, align 64
994 // %66 = call x86_amx @llvm.x86.cast.vector.to.tile(<256 x i32> %65)
995 // -->
996 // %66 = call x86_amx @llvm.x86.tileloadd64.internal(i16 %row, i16 %col,
997 // i8* %p, i64 64)
998 combineLoadCast(cast<IntrinsicInst>(Cast), Load);
999 // Set the operand is null so that load instruction can be erased.
1000 Cast->setOperand(0, nullptr);
1001 Load->eraseFromParent();
1002 }
1003 }
1004 return Change;
1005}
1006
1007bool X86LowerAMXCast::combineAMXcast(TargetLibraryInfo *TLI) {
1008 bool Change = false;
1009 // Collect tile cast instruction.
1010 SmallVector<Instruction *, 8> Vec2TileInsts;
1011 SmallVector<Instruction *, 8> Tile2VecInsts;
1012 SmallVector<Instruction *, 8> PhiCastWorkList;
1013 SmallSetVector<Instruction *, 16> DeadInst;
1014 for (BasicBlock &BB : Func) {
1015 for (Instruction &I : BB) {
1016 Value *Vec;
1017 if (match(&I,
1018 m_Intrinsic<Intrinsic::x86_cast_vector_to_tile>(m_Value(Vec))))
1019 Vec2TileInsts.push_back(&I);
1020 else if (match(&I, m_Intrinsic<Intrinsic::x86_cast_tile_to_vector>(
1021 m_Value(Vec))))
1022 Tile2VecInsts.push_back(&I);
1023 }
1024 }
1025
1026 auto Convert = [&](SmallVectorImpl<Instruction *> &Insts, Intrinsic::ID IID) {
1027 for (auto *Inst : Insts) {
1028 for (User *U : Inst->users()) {
1029 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
1030 if (!II || II->getIntrinsicID() != IID)
1031 continue;
1032 // T1 = vec2tile V0
1033 // V2 = tile2vec T1
1034 // V3 = OP V2
1035 // -->
1036 // T1 = vec2tile V0
1037 // V2 = tile2vec T1
1038 // V3 = OP V0
1039 II->replaceAllUsesWith(Inst->getOperand(0));
1040 Change = true;
1041 }
1042 }
1043 };
1044
1045 Convert(Vec2TileInsts, Intrinsic::x86_cast_tile_to_vector);
1046 Convert(Tile2VecInsts, Intrinsic::x86_cast_vector_to_tile);
1047
1048 SmallVector<Instruction *, 8> LiveCasts;
1049 auto EraseInst = [&](SmallVectorImpl<Instruction *> &Insts) {
1050 for (auto *Inst : Insts) {
1051 if (Inst->use_empty()) {
1052 Inst->eraseFromParent();
1053 Change = true;
1054 } else {
1055 LiveCasts.push_back(Inst);
1056 }
1057 }
1058 };
1059
1060 EraseInst(Vec2TileInsts);
1061 EraseInst(Tile2VecInsts);
1062 Change |= combineLdSt(LiveCasts);
1063 EraseInst(LiveCasts);
1064
1065 // Handle the A->B->A cast, and there is an intervening PHI node.
1066 for (BasicBlock &BB : Func) {
1067 for (Instruction &I : BB) {
1068 if (isAMXCast(&I)) {
1069 if (isa<PHINode>(I.getOperand(0)))
1070 PhiCastWorkList.push_back(&I);
1071 }
1072 }
1073 }
1074 for (auto *I : PhiCastWorkList) {
1075 // We skip the dead Amxcast.
1076 if (DeadInst.contains(I))
1077 continue;
1078 PHINode *PN = cast<PHINode>(I->getOperand(0));
1079 if (optimizeAMXCastFromPhi(cast<IntrinsicInst>(I), PN, DeadInst)) {
1080 DeadInst.insert(PN);
1081 Change = true;
1082 }
1083 }
1084
1085 // Since we create new phi and merge AMXCast, some old phis and AMXCast might
1086 // have no uses. We do some DeadCodeElimination for them.
1087 while (!DeadInst.empty()) {
1088 Instruction *I = DeadInst.pop_back_val();
1089 Change |= DCEInstruction(I, DeadInst, TLI);
1090 }
1091 return Change;
1092}
1093
1094// There might be remaining AMXcast after combineAMXcast and they should be
1095// handled elegantly.
1096bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
1097 IRBuilder<> Builder(AMXCast);
1098 AllocaInst *AllocaAddr;
1099 Value *I8Ptr, *Stride;
1100 auto *Src = AMXCast->getOperand(0);
1101
1102 auto Prepare = [&](Type *MemTy) {
1103 AllocaAddr = createAllocaInstAtEntry(Builder, AMXCast->getParent(), MemTy);
1104 I8Ptr = Builder.CreateBitCast(AllocaAddr, Builder.getInt8PtrTy());
1105 Stride = Builder.getInt64(64);
1106 };
1107
1108 if (AMXCast->getType()->isX86_AMXTy()) {
1109 // %2 = amxcast <225 x i32> %src to x86_amx
1110 // call void @llvm.x86.tilestored64.internal(i16 15, i16 60,
1111 // i8* %addr3, i64 60, x86_amx %2)
1112 // -->
1113 // %addr = alloca <225 x i32>, align 64
1114 // store <225 x i32> %src, <225 x i32>* %addr, align 64
1115 // %addr2 = bitcast <225 x i32>* %addr to i8*
1116 // %2 = call x86_amx @llvm.x86.tileloadd64.internal(i16 15, i16 60,
1117 // i8* %addr2,
1118 // i64 60)
1119 // call void @llvm.x86.tilestored64.internal(i16 15, i16 60,
1120 // i8* %addr3, i64 60, x86_amx %2)
1121 if (AMXCast->use_empty()) {
1122 AMXCast->eraseFromParent();
1123 return true;
1124 }
1125 Use &U = *(AMXCast->use_begin());
1126 unsigned OpNo = U.getOperandNo();
1127 auto *II = dyn_cast<IntrinsicInst>(U.getUser());
1128 if (!II)
1129 return false; // May be bitcast from x86amx to <256 x i32>.
1130 Prepare(AMXCast->getOperand(0)->getType());
1131 Builder.CreateStore(Src, AllocaAddr);
1132 // TODO we can pick an constant operand for the shape.
1133 Value *Row = nullptr, *Col = nullptr;
1134 std::tie(Row, Col) = getShape(II, OpNo);
1135 std::array<Value *, 4> Args = {
1136 Row, Col, I8Ptr, Builder.CreateSExt(Col, Builder.getInt64Ty())};
1137 Value *NewInst = Builder.CreateIntrinsic(
1138 Intrinsic::x86_tileloadd64_internal, None, Args);
1139 AMXCast->replaceAllUsesWith(NewInst);
1140 AMXCast->eraseFromParent();
1141 } else {
1142 // %2 = amxcast x86_amx %src to <225 x i32>
1143 // -->
1144 // %addr = alloca <225 x i32>, align 64
1145 // %addr2 = bitcast <225 x i32>* to i8*
1146 // call void @llvm.x86.tilestored64.internal(i16 %row, i16 %col,
1147 // i8* %addr2, i64 %stride)
1148 // %2 = load <225 x i32>, <225 x i32>* %addr, align 64
1149 auto *II = dyn_cast<IntrinsicInst>(Src);
1150 if (!II)
1151 return false; // May be bitcast from <256 x i32> to x86amx.
1152 Prepare(AMXCast->getType());
1153 Value *Row = II->getOperand(0);
1154 Value *Col = II->getOperand(1);
1155 std::array<Value *, 5> Args = {
1156 Row, Col, I8Ptr, Builder.CreateSExt(Col, Builder.getInt64Ty()), Src};
1157 Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
1158 Value *NewInst = Builder.CreateLoad(AMXCast->getType(), AllocaAddr);
1159 AMXCast->replaceAllUsesWith(NewInst);
1160 AMXCast->eraseFromParent();
1161 }
1162
1163 return true;
1164}
1165
1166bool X86LowerAMXCast::transformAllAMXCast() {
1167 bool Change = false;
1168 // Collect tile cast instruction.
1169 SmallVector<Instruction *, 8> WorkLists;
1170 for (BasicBlock &BB : Func) {
1171 for (Instruction &I : BB) {
1172 if (isAMXCast(&I))
1173 WorkLists.push_back(&I);
1174 }
1175 }
1176
1177 for (auto *Inst : WorkLists) {
1178 Change |= transformAMXCast(cast<IntrinsicInst>(Inst));
1179 }
1180
1181 return Change;
1182}
1183
1184} // anonymous namespace
1185
1186namespace {
1187
1188class X86LowerAMXTypeLegacyPass : public FunctionPass {
1189public:
1190 static char ID;
1191
1192 X86LowerAMXTypeLegacyPass() : FunctionPass(ID) {
1193 initializeX86LowerAMXTypeLegacyPassPass(*PassRegistry::getPassRegistry());
1194 }
1195
1196 bool runOnFunction(Function &F) override {
1197 bool C = false;
1198 TargetMachine *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
1199 TargetLibraryInfo *TLI =
1200 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
1201 X86LowerAMXCast LAC(F);
1202 C |= LAC.combineAMXcast(TLI);
1203 // There might be remaining AMXcast after combineAMXcast and they should be
1204 // handled elegantly.
1205 C |= LAC.transformAllAMXCast();
1206
1207 X86LowerAMXType LAT(F);
1208 C |= LAT.visit();
1209
1210 // Prepare for fast register allocation at O0.
1211 // Todo: May better check the volatile model of AMX code, not just
1212 // by checking Attribute::OptimizeNone and CodeGenOpt::None.
1213 if (TM->getOptLevel() == CodeGenOpt::None) {
1
Assuming the condition is true
2
Taking true branch
1214 // If Front End not use O0 but the Mid/Back end use O0, (e.g.
1215 // "Clang -O2 -S -emit-llvm t.c" + "llc t.ll") we should make
1216 // sure the amx data is volatile, that is nessary for AMX fast
1217 // register allocation.
1218 if (!F.hasFnAttribute(Attribute::OptimizeNone)) {
3
Assuming the condition is true
4
Taking true branch
1219 X86VolatileTileData VTD(F);
1220 C = VTD.volatileTileData() || C;
5
Calling 'X86VolatileTileData::volatileTileData'
1221 }
1222 }
1223
1224 return C;
1225 }
1226
1227 void getAnalysisUsage(AnalysisUsage &AU) const override {
1228 AU.setPreservesCFG();
1229 AU.addRequired<TargetPassConfig>();
1230 AU.addRequired<TargetLibraryInfoWrapperPass>();
1231 }
1232};
1233
1234} // anonymous namespace
1235
1236static const char PassName[] = "Lower AMX type for load/store";
1237char X86LowerAMXTypeLegacyPass::ID = 0;
1238INITIALIZE_PASS_BEGIN(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,static void *initializeX86LowerAMXTypeLegacyPassPassOnce(PassRegistry
&Registry) {
1239 false)static void *initializeX86LowerAMXTypeLegacyPassPassOnce(PassRegistry
&Registry) {
1240INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)initializeTargetPassConfigPass(Registry);
1241INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
1242INITIALIZE_PASS_END(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,PassInfo *PI = new PassInfo( PassName, "lower-amx-type", &
X86LowerAMXTypeLegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<X86LowerAMXTypeLegacyPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeX86LowerAMXTypeLegacyPassPassFlag
; void llvm::initializeX86LowerAMXTypeLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeX86LowerAMXTypeLegacyPassPassFlag
, initializeX86LowerAMXTypeLegacyPassPassOnce, std::ref(Registry
)); }
1243 false)PassInfo *PI = new PassInfo( PassName, "lower-amx-type", &
X86LowerAMXTypeLegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<X86LowerAMXTypeLegacyPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeX86LowerAMXTypeLegacyPassPassFlag
; void llvm::initializeX86LowerAMXTypeLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeX86LowerAMXTypeLegacyPassPassFlag
, initializeX86LowerAMXTypeLegacyPassPassOnce, std::ref(Registry
)); }
1244
1245FunctionPass *llvm::createX86LowerAMXTypePass() {
1246 return new X86LowerAMXTypeLegacyPass();
1247}

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugLoc.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/FPEnv.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstdint>
48#include <functional>
49#include <utility>
50
51namespace llvm {
52
53class APInt;
54class Use;
55
56/// This provides the default implementation of the IRBuilder
57/// 'InsertHelper' method that is called whenever an instruction is created by
58/// IRBuilder and needs to be inserted.
59///
60/// By default, this inserts the instruction at the insertion point.
61class IRBuilderDefaultInserter {
62public:
63 virtual ~IRBuilderDefaultInserter();
64
65 virtual void InsertHelper(Instruction *I, const Twine &Name,
66 BasicBlock *BB,
67 BasicBlock::iterator InsertPt) const {
68 if (BB) BB->getInstList().insert(InsertPt, I);
69 I->setName(Name);
70 }
71};
72
73/// Provides an 'InsertHelper' that calls a user-provided callback after
74/// performing the default insertion.
75class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
76 std::function<void(Instruction *)> Callback;
77
78public:
79 ~IRBuilderCallbackInserter() override;
80
81 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
82 : Callback(std::move(Callback)) {}
83
84 void InsertHelper(Instruction *I, const Twine &Name,
85 BasicBlock *BB,
86 BasicBlock::iterator InsertPt) const override {
87 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
88 Callback(I);
89 }
90};
91
92/// Common base class shared among various IRBuilders.
93class IRBuilderBase {
94 /// Pairs of (metadata kind, MDNode *) that should be added to all newly
95 /// created instructions, like !dbg metadata.
96 SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
97
98 /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
99 /// null. If \p MD is null, remove the entry with \p Kind.
100 void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
101 if (!MD) {
102 erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
103 return KV.first == Kind;
104 });
105 return;
106 }
107
108 for (auto &KV : MetadataToCopy)
109 if (KV.first == Kind) {
110 KV.second = MD;
111 return;
112 }
113
114 MetadataToCopy.emplace_back(Kind, MD);
115 }
116
117protected:
118 BasicBlock *BB;
119 BasicBlock::iterator InsertPt;
120 LLVMContext &Context;
121 const IRBuilderFolder &Folder;
122 const IRBuilderDefaultInserter &Inserter;
123
124 MDNode *DefaultFPMathTag;
125 FastMathFlags FMF;
126
127 bool IsFPConstrained = false;
128 fp::ExceptionBehavior DefaultConstrainedExcept = fp::ebStrict;
129 RoundingMode DefaultConstrainedRounding = RoundingMode::Dynamic;
130
131 ArrayRef<OperandBundleDef> DefaultOperandBundles;
132
133public:
134 IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
135 const IRBuilderDefaultInserter &Inserter, MDNode *FPMathTag,
136 ArrayRef<OperandBundleDef> OpBundles)
137 : Context(context), Folder(Folder), Inserter(Inserter),
138 DefaultFPMathTag(FPMathTag), DefaultOperandBundles(OpBundles) {
139 ClearInsertionPoint();
140 }
141
142 /// Insert and return the specified instruction.
143 template<typename InstTy>
144 InstTy *Insert(InstTy *I, const Twine &Name = "") const {
145 Inserter.InsertHelper(I, Name, BB, InsertPt);
146 AddMetadataToInst(I);
147 return I;
148 }
149
150 /// No-op overload to handle constants.
151 Constant *Insert(Constant *C, const Twine& = "") const {
152 return C;
153 }
154
155 Value *Insert(Value *V, const Twine &Name = "") const {
156 if (Instruction *I = dyn_cast<Instruction>(V))
157 return Insert(I, Name);
158 assert(isa<Constant>(V))(static_cast <bool> (isa<Constant>(V)) ? void (0)
: __assert_fail ("isa<Constant>(V)", "llvm/include/llvm/IR/IRBuilder.h"
, 158, __extension__ __PRETTY_FUNCTION__))
;
159 return V;
160 }
161
162 //===--------------------------------------------------------------------===//
163 // Builder configuration methods
164 //===--------------------------------------------------------------------===//
165
166 /// Clear the insertion point: created instructions will not be
167 /// inserted into a block.
168 void ClearInsertionPoint() {
169 BB = nullptr;
170 InsertPt = BasicBlock::iterator();
171 }
172
173 BasicBlock *GetInsertBlock() const { return BB; }
174 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
175 LLVMContext &getContext() const { return Context; }
176
177 /// This specifies that created instructions should be appended to the
178 /// end of the specified block.
179 void SetInsertPoint(BasicBlock *TheBB) {
180 BB = TheBB;
181 InsertPt = BB->end();
182 }
183
184 /// This specifies that created instructions should be inserted before
185 /// the specified instruction.
186 void SetInsertPoint(Instruction *I) {
187 BB = I->getParent();
188 InsertPt = I->getIterator();
189 assert(InsertPt != BB->end() && "Can't read debug loc from end()")(static_cast <bool> (InsertPt != BB->end() &&
"Can't read debug loc from end()") ? void (0) : __assert_fail
("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "llvm/include/llvm/IR/IRBuilder.h", 189, __extension__ __PRETTY_FUNCTION__
))
;
190 SetCurrentDebugLocation(I->getDebugLoc());
191 }
192
193 /// This specifies that created instructions should be inserted at the
194 /// specified point.
195 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
196 BB = TheBB;
197 InsertPt = IP;
198 if (IP != TheBB->end())
199 SetCurrentDebugLocation(IP->getDebugLoc());
200 }
201
202 /// This specifies that created instructions should inserted at the beginning
203 /// end of the specified function, but after already existing static alloca
204 /// instructions that are at the start.
205 void SetInsertPointPastAllocas(Function *F) {
206 BB = &F->getEntryBlock();
207 InsertPt = BB->getFirstNonPHIOrDbgOrAlloca();
208 }
209
210 /// Set location information used by debugging information.
211 void SetCurrentDebugLocation(DebugLoc L) {
212 AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
213 }
214
215 /// Collect metadata with IDs \p MetadataKinds from \p Src which should be
216 /// added to all created instructions. Entries present in MedataDataToCopy but
217 /// not on \p Src will be dropped from MetadataToCopy.
218 void CollectMetadataToCopy(Instruction *Src,
219 ArrayRef<unsigned> MetadataKinds) {
220 for (unsigned K : MetadataKinds)
221 AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
222 }
223
224 /// Get location information used by debugging information.
225 DebugLoc getCurrentDebugLocation() const;
226
227 /// If this builder has a current debug location, set it on the
228 /// specified instruction.
229 void SetInstDebugLocation(Instruction *I) const;
230
231 /// Add all entries in MetadataToCopy to \p I.
232 void AddMetadataToInst(Instruction *I) const {
233 for (const auto &KV : MetadataToCopy)
234 I->setMetadata(KV.first, KV.second);
235 }
236
237 /// Get the return type of the current function that we're emitting
238 /// into.
239 Type *getCurrentFunctionReturnType() const;
240
241 /// InsertPoint - A saved insertion point.
242 class InsertPoint {
243 BasicBlock *Block = nullptr;
244 BasicBlock::iterator Point;
245
246 public:
247 /// Creates a new insertion point which doesn't point to anything.
248 InsertPoint() = default;
249
250 /// Creates a new insertion point at the given location.
251 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
252 : Block(InsertBlock), Point(InsertPoint) {}
253
254 /// Returns true if this insert point is set.
255 bool isSet() const { return (Block != nullptr); }
256
257 BasicBlock *getBlock() const { return Block; }
258 BasicBlock::iterator getPoint() const { return Point; }
259 };
260
261 /// Returns the current insert point.
262 InsertPoint saveIP() const {
263 return InsertPoint(GetInsertBlock(), GetInsertPoint());
264 }
265
266 /// Returns the current insert point, clearing it in the process.
267 InsertPoint saveAndClearIP() {
268 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
269 ClearInsertionPoint();
270 return IP;
271 }
272
273 /// Sets the current insert point to a previously-saved location.
274 void restoreIP(InsertPoint IP) {
275 if (IP.isSet())
276 SetInsertPoint(IP.getBlock(), IP.getPoint());
277 else
278 ClearInsertionPoint();
279 }
280
281 /// Get the floating point math metadata being used.
282 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
283
284 /// Get the flags to be applied to created floating point ops
285 FastMathFlags getFastMathFlags() const { return FMF; }
286
287 FastMathFlags &getFastMathFlags() { return FMF; }
288
289 /// Clear the fast-math flags.
290 void clearFastMathFlags() { FMF.clear(); }
291
292 /// Set the floating point math metadata to be used.
293 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
294
295 /// Set the fast-math flags to be used with generated fp-math operators
296 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
297
298 /// Enable/Disable use of constrained floating point math. When
299 /// enabled the CreateF<op>() calls instead create constrained
300 /// floating point intrinsic calls. Fast math flags are unaffected
301 /// by this setting.
302 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
303
304 /// Query for the use of constrained floating point math
305 bool getIsFPConstrained() { return IsFPConstrained; }
306
307 /// Set the exception handling to be used with constrained floating point
308 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
309#ifndef NDEBUG
310 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(NewExcept);
311 assert(ExceptStr && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 311, __extension__ __PRETTY_FUNCTION__
))
;
312#endif
313 DefaultConstrainedExcept = NewExcept;
314 }
315
316 /// Set the rounding mode handling to be used with constrained floating point
317 void setDefaultConstrainedRounding(RoundingMode NewRounding) {
318#ifndef NDEBUG
319 Optional<StringRef> RoundingStr = convertRoundingModeToStr(NewRounding);
320 assert(RoundingStr && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 320, __extension__ __PRETTY_FUNCTION__
))
;
321#endif
322 DefaultConstrainedRounding = NewRounding;
323 }
324
325 /// Get the exception handling used with constrained floating point
326 fp::ExceptionBehavior getDefaultConstrainedExcept() {
327 return DefaultConstrainedExcept;
328 }
329
330 /// Get the rounding mode handling used with constrained floating point
331 RoundingMode getDefaultConstrainedRounding() {
332 return DefaultConstrainedRounding;
333 }
334
335 void setConstrainedFPFunctionAttr() {
336 assert(BB && "Must have a basic block to set any function attributes!")(static_cast <bool> (BB && "Must have a basic block to set any function attributes!"
) ? void (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "llvm/include/llvm/IR/IRBuilder.h", 336, __extension__ __PRETTY_FUNCTION__
))
;
337
338 Function *F = BB->getParent();
339 if (!F->hasFnAttribute(Attribute::StrictFP)) {
340 F->addFnAttr(Attribute::StrictFP);
341 }
342 }
343
344 void setConstrainedFPCallAttr(CallBase *I) {
345 I->addFnAttr(Attribute::StrictFP);
346 }
347
348 void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
349 DefaultOperandBundles = OpBundles;
350 }
351
352 //===--------------------------------------------------------------------===//
353 // RAII helpers.
354 //===--------------------------------------------------------------------===//
355
356 // RAII object that stores the current insertion point and restores it
357 // when the object is destroyed. This includes the debug location.
358 class InsertPointGuard {
359 IRBuilderBase &Builder;
360 AssertingVH<BasicBlock> Block;
361 BasicBlock::iterator Point;
362 DebugLoc DbgLoc;
363
364 public:
365 InsertPointGuard(IRBuilderBase &B)
366 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
367 DbgLoc(B.getCurrentDebugLocation()) {}
368
369 InsertPointGuard(const InsertPointGuard &) = delete;
370 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
371
372 ~InsertPointGuard() {
373 Builder.restoreIP(InsertPoint(Block, Point));
374 Builder.SetCurrentDebugLocation(DbgLoc);
375 }
376 };
377
378 // RAII object that stores the current fast math settings and restores
379 // them when the object is destroyed.
380 class FastMathFlagGuard {
381 IRBuilderBase &Builder;
382 FastMathFlags FMF;
383 MDNode *FPMathTag;
384 bool IsFPConstrained;
385 fp::ExceptionBehavior DefaultConstrainedExcept;
386 RoundingMode DefaultConstrainedRounding;
387
388 public:
389 FastMathFlagGuard(IRBuilderBase &B)
390 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
391 IsFPConstrained(B.IsFPConstrained),
392 DefaultConstrainedExcept(B.DefaultConstrainedExcept),
393 DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}
394
395 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
396 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
397
398 ~FastMathFlagGuard() {
399 Builder.FMF = FMF;
400 Builder.DefaultFPMathTag = FPMathTag;
401 Builder.IsFPConstrained = IsFPConstrained;
402 Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
403 Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
404 }
405 };
406
407 // RAII object that stores the current default operand bundles and restores
408 // them when the object is destroyed.
409 class OperandBundlesGuard {
410 IRBuilderBase &Builder;
411 ArrayRef<OperandBundleDef> DefaultOperandBundles;
412
413 public:
414 OperandBundlesGuard(IRBuilderBase &B)
415 : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}
416
417 OperandBundlesGuard(const OperandBundlesGuard &) = delete;
418 OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;
419
420 ~OperandBundlesGuard() {
421 Builder.DefaultOperandBundles = DefaultOperandBundles;
422 }
423 };
424
425
426 //===--------------------------------------------------------------------===//
427 // Miscellaneous creation methods.
428 //===--------------------------------------------------------------------===//
429
430 /// Make a new global variable with initializer type i8*
431 ///
432 /// Make a new global variable with an initializer that has array of i8 type
433 /// filled in with the null terminated string value specified. The new global
434 /// variable will be marked mergable with any others of the same contents. If
435 /// Name is specified, it is the name of the global variable created.
436 ///
437 /// If no module is given via \p M, it is take from the insertion point basic
438 /// block.
439 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
440 unsigned AddressSpace = 0,
441 Module *M = nullptr);
442
443 /// Get a constant value representing either true or false.
444 ConstantInt *getInt1(bool V) {
445 return ConstantInt::get(getInt1Ty(), V);
446 }
447
448 /// Get the constant value for i1 true.
449 ConstantInt *getTrue() {
450 return ConstantInt::getTrue(Context);
451 }
452
453 /// Get the constant value for i1 false.
454 ConstantInt *getFalse() {
455 return ConstantInt::getFalse(Context);
456 }
457
458 /// Get a constant 8-bit value.
459 ConstantInt *getInt8(uint8_t C) {
460 return ConstantInt::get(getInt8Ty(), C);
461 }
462
463 /// Get a constant 16-bit value.
464 ConstantInt *getInt16(uint16_t C) {
465 return ConstantInt::get(getInt16Ty(), C);
466 }
467
468 /// Get a constant 32-bit value.
469 ConstantInt *getInt32(uint32_t C) {
470 return ConstantInt::get(getInt32Ty(), C);
471 }
472
473 /// Get a constant 64-bit value.
474 ConstantInt *getInt64(uint64_t C) {
475 return ConstantInt::get(getInt64Ty(), C);
476 }
477
478 /// Get a constant N-bit value, zero extended or truncated from
479 /// a 64-bit value.
480 ConstantInt *getIntN(unsigned N, uint64_t C) {
481 return ConstantInt::get(getIntNTy(N), C);
482 }
483
484 /// Get a constant integer value.
485 ConstantInt *getInt(const APInt &AI) {
486 return ConstantInt::get(Context, AI);
487 }
488
489 //===--------------------------------------------------------------------===//
490 // Type creation methods
491 //===--------------------------------------------------------------------===//
492
493 /// Fetch the type representing a single bit
494 IntegerType *getInt1Ty() {
495 return Type::getInt1Ty(Context);
496 }
497
498 /// Fetch the type representing an 8-bit integer.
499 IntegerType *getInt8Ty() {
500 return Type::getInt8Ty(Context);
501 }
502
503 /// Fetch the type representing a 16-bit integer.
504 IntegerType *getInt16Ty() {
505 return Type::getInt16Ty(Context);
506 }
507
508 /// Fetch the type representing a 32-bit integer.
509 IntegerType *getInt32Ty() {
510 return Type::getInt32Ty(Context);
511 }
512
513 /// Fetch the type representing a 64-bit integer.
514 IntegerType *getInt64Ty() {
515 return Type::getInt64Ty(Context);
516 }
517
518 /// Fetch the type representing a 128-bit integer.
519 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
520
521 /// Fetch the type representing an N-bit integer.
522 IntegerType *getIntNTy(unsigned N) {
523 return Type::getIntNTy(Context, N);
524 }
525
526 /// Fetch the type representing a 16-bit floating point value.
527 Type *getHalfTy() {
528 return Type::getHalfTy(Context);
529 }
530
531 /// Fetch the type representing a 16-bit brain floating point value.
532 Type *getBFloatTy() {
533 return Type::getBFloatTy(Context);
534 }
535
536 /// Fetch the type representing a 32-bit floating point value.
537 Type *getFloatTy() {
538 return Type::getFloatTy(Context);
539 }
540
541 /// Fetch the type representing a 64-bit floating point value.
542 Type *getDoubleTy() {
543 return Type::getDoubleTy(Context);
544 }
545
546 /// Fetch the type representing void.
547 Type *getVoidTy() {
548 return Type::getVoidTy(Context);
549 }
550
551 /// Fetch the type representing a pointer.
552 PointerType *getPtrTy(unsigned AddrSpace = 0) {
553 return PointerType::get(Context, AddrSpace);
554 }
555
556 /// Fetch the type representing a pointer to an 8-bit integer value.
557 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
558 return Type::getInt8PtrTy(Context, AddrSpace);
559 }
560
561 /// Fetch the type representing a pointer to an integer value.
562 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
563 return DL.getIntPtrType(Context, AddrSpace);
564 }
565
566 //===--------------------------------------------------------------------===//
567 // Intrinsic creation methods
568 //===--------------------------------------------------------------------===//
569
570 /// Create and insert a memset to the specified pointer and the
571 /// specified value.
572 ///
573 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
574 /// specified, it will be added to the instruction. Likewise with alias.scope
575 /// and noalias tags.
576 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
577 MaybeAlign Align, bool isVolatile = false,
578 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
579 MDNode *NoAliasTag = nullptr) {
580 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
581 TBAATag, ScopeTag, NoAliasTag);
582 }
583
584 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
585 bool isVolatile = false, MDNode *TBAATag = nullptr,
586 MDNode *ScopeTag = nullptr,
587 MDNode *NoAliasTag = nullptr);
588
589 CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val,
590 Value *Size, bool IsVolatile = false,
591 MDNode *TBAATag = nullptr,
592 MDNode *ScopeTag = nullptr,
593 MDNode *NoAliasTag = nullptr);
594
595 /// Create and insert an element unordered-atomic memset of the region of
596 /// memory starting at the given pointer to the given value.
597 ///
598 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
599 /// specified, it will be added to the instruction. Likewise with alias.scope
600 /// and noalias tags.
601 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
602 uint64_t Size, Align Alignment,
603 uint32_t ElementSize,
604 MDNode *TBAATag = nullptr,
605 MDNode *ScopeTag = nullptr,
606 MDNode *NoAliasTag = nullptr) {
607 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
608 Align(Alignment), ElementSize,
609 TBAATag, ScopeTag, NoAliasTag);
610 }
611
612 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
613 Value *Size, Align Alignment,
614 uint32_t ElementSize,
615 MDNode *TBAATag = nullptr,
616 MDNode *ScopeTag = nullptr,
617 MDNode *NoAliasTag = nullptr);
618
619 /// Create and insert a memcpy between the specified pointers.
620 ///
621 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
622 /// specified, it will be added to the instruction. Likewise with alias.scope
623 /// and noalias tags.
624 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
625 MaybeAlign SrcAlign, uint64_t Size,
626 bool isVolatile = false, MDNode *TBAATag = nullptr,
627 MDNode *TBAAStructTag = nullptr,
628 MDNode *ScopeTag = nullptr,
629 MDNode *NoAliasTag = nullptr) {
630 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
631 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
632 NoAliasTag);
633 }
634
635 CallInst *CreateMemTransferInst(
636 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
637 MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
638 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
639 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
640
641 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
642 MaybeAlign SrcAlign, Value *Size,
643 bool isVolatile = false, MDNode *TBAATag = nullptr,
644 MDNode *TBAAStructTag = nullptr,
645 MDNode *ScopeTag = nullptr,
646 MDNode *NoAliasTag = nullptr) {
647 return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
648 SrcAlign, Size, isVolatile, TBAATag,
649 TBAAStructTag, ScopeTag, NoAliasTag);
650 }
651
652 CallInst *
653 CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
654 MaybeAlign SrcAlign, Value *Size, bool IsVolatile = false,
655 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
656 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
657
658 /// Create and insert an element unordered-atomic memcpy between the
659 /// specified pointers.
660 ///
661 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
662 ///
663 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
664 /// specified, it will be added to the instruction. Likewise with alias.scope
665 /// and noalias tags.
666 CallInst *CreateElementUnorderedAtomicMemCpy(
667 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
668 uint32_t ElementSize, MDNode *TBAATag = nullptr,
669 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
670 MDNode *NoAliasTag = nullptr);
671
672 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
673 MaybeAlign SrcAlign, uint64_t Size,
674 bool isVolatile = false, MDNode *TBAATag = nullptr,
675 MDNode *ScopeTag = nullptr,
676 MDNode *NoAliasTag = nullptr) {
677 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
678 isVolatile, TBAATag, ScopeTag, NoAliasTag);
679 }
680
681 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
682 MaybeAlign SrcAlign, Value *Size,
683 bool isVolatile = false, MDNode *TBAATag = nullptr,
684 MDNode *ScopeTag = nullptr,
685 MDNode *NoAliasTag = nullptr);
686
687 /// \brief Create and insert an element unordered-atomic memmove between the
688 /// specified pointers.
689 ///
690 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
691 /// respectively.
692 ///
693 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
694 /// specified, it will be added to the instruction. Likewise with alias.scope
695 /// and noalias tags.
696 CallInst *CreateElementUnorderedAtomicMemMove(
697 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
698 uint32_t ElementSize, MDNode *TBAATag = nullptr,
699 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
700 MDNode *NoAliasTag = nullptr);
701
702private:
703 CallInst *getReductionIntrinsic(Intrinsic::ID ID, Value *Src);
704
705public:
706 /// Create a sequential vector fadd reduction intrinsic of the source vector.
707 /// The first parameter is a scalar accumulator value. An unordered reduction
708 /// can be created by adding the reassoc fast-math flag to the resulting
709 /// sequential reduction.
710 CallInst *CreateFAddReduce(Value *Acc, Value *Src);
711
712 /// Create a sequential vector fmul reduction intrinsic of the source vector.
713 /// The first parameter is a scalar accumulator value. An unordered reduction
714 /// can be created by adding the reassoc fast-math flag to the resulting
715 /// sequential reduction.
716 CallInst *CreateFMulReduce(Value *Acc, Value *Src);
717
718 /// Create a vector int add reduction intrinsic of the source vector.
719 CallInst *CreateAddReduce(Value *Src);
720
721 /// Create a vector int mul reduction intrinsic of the source vector.
722 CallInst *CreateMulReduce(Value *Src);
723
724 /// Create a vector int AND reduction intrinsic of the source vector.
725 CallInst *CreateAndReduce(Value *Src);
726
727 /// Create a vector int OR reduction intrinsic of the source vector.
728 CallInst *CreateOrReduce(Value *Src);
729
730 /// Create a vector int XOR reduction intrinsic of the source vector.
731 CallInst *CreateXorReduce(Value *Src);
732
733 /// Create a vector integer max reduction intrinsic of the source
734 /// vector.
735 CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
736
737 /// Create a vector integer min reduction intrinsic of the source
738 /// vector.
739 CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
740
741 /// Create a vector float max reduction intrinsic of the source
742 /// vector.
743 CallInst *CreateFPMaxReduce(Value *Src);
744
745 /// Create a vector float min reduction intrinsic of the source
746 /// vector.
747 CallInst *CreateFPMinReduce(Value *Src);
748
749 /// Create a lifetime.start intrinsic.
750 ///
751 /// If the pointer isn't i8* it will be converted.
752 CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
753
754 /// Create a lifetime.end intrinsic.
755 ///
756 /// If the pointer isn't i8* it will be converted.
757 CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
758
759 /// Create a call to invariant.start intrinsic.
760 ///
761 /// If the pointer isn't i8* it will be converted.
762 CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
763
764 /// Create a call to llvm.threadlocal.address intrinsic.
765 CallInst *CreateThreadLocalAddress(Value *Ptr);
766
767 /// Create a call to Masked Load intrinsic
768 CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask,
769 Value *PassThru = nullptr, const Twine &Name = "");
770
771 /// Create a call to Masked Store intrinsic
772 CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
773 Value *Mask);
774
775 /// Create a call to Masked Gather intrinsic
776 CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
777 Value *Mask = nullptr, Value *PassThru = nullptr,
778 const Twine &Name = "");
779
780 /// Create a call to Masked Scatter intrinsic
781 CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
782 Value *Mask = nullptr);
783
784 /// Create a call to Masked Expand Load intrinsic
785 CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask = nullptr,
786 Value *PassThru = nullptr,
787 const Twine &Name = "");
788
789 /// Create a call to Masked Compress Store intrinsic
790 CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr,
791 Value *Mask = nullptr);
792
793 /// Create an assume intrinsic call that allows the optimizer to
794 /// assume that the provided condition will be true.
795 ///
796 /// The optional argument \p OpBundles specifies operand bundles that are
797 /// added to the call instruction.
798 CallInst *CreateAssumption(Value *Cond,
799 ArrayRef<OperandBundleDef> OpBundles = llvm::None);
800
801 /// Create a llvm.experimental.noalias.scope.decl intrinsic call.
802 Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
803 Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
804 return CreateNoAliasScopeDeclaration(
805 MetadataAsValue::get(Context, ScopeTag));
806 }
807
808 /// Create a call to the experimental.gc.statepoint intrinsic to
809 /// start a new statepoint sequence.
810 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
811 FunctionCallee ActualCallee,
812 ArrayRef<Value *> CallArgs,
813 Optional<ArrayRef<Value *>> DeoptArgs,
814 ArrayRef<Value *> GCArgs,
815 const Twine &Name = "");
816
817 /// Create a call to the experimental.gc.statepoint intrinsic to
818 /// start a new statepoint sequence.
819 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
820 FunctionCallee ActualCallee, uint32_t Flags,
821 ArrayRef<Value *> CallArgs,
822 Optional<ArrayRef<Use>> TransitionArgs,
823 Optional<ArrayRef<Use>> DeoptArgs,
824 ArrayRef<Value *> GCArgs,
825 const Twine &Name = "");
826
827 /// Conveninence function for the common case when CallArgs are filled
828 /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
829 /// .get()'ed to get the Value pointer.
830 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
831 FunctionCallee ActualCallee,
832 ArrayRef<Use> CallArgs,
833 Optional<ArrayRef<Value *>> DeoptArgs,
834 ArrayRef<Value *> GCArgs,
835 const Twine &Name = "");
836
837 /// Create an invoke to the experimental.gc.statepoint intrinsic to
838 /// start a new statepoint sequence.
839 InvokeInst *
840 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
841 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
842 BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
843 Optional<ArrayRef<Value *>> DeoptArgs,
844 ArrayRef<Value *> GCArgs, const Twine &Name = "");
845
846 /// Create an invoke to the experimental.gc.statepoint intrinsic to
847 /// start a new statepoint sequence.
848 InvokeInst *CreateGCStatepointInvoke(
849 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
850 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
851 ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
852 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
853 const Twine &Name = "");
854
855 // Convenience function for the common case when CallArgs are filled in using
856 // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
857 // get the Value *.
858 InvokeInst *
859 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
860 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
861 BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
862 Optional<ArrayRef<Value *>> DeoptArgs,
863 ArrayRef<Value *> GCArgs, const Twine &Name = "");
864
865 /// Create a call to the experimental.gc.result intrinsic to extract
866 /// the result from a call wrapped in a statepoint.
867 CallInst *CreateGCResult(Instruction *Statepoint,
868 Type *ResultType,
869 const Twine &Name = "");
870
871 /// Create a call to the experimental.gc.relocate intrinsics to
872 /// project the relocated value of one pointer from the statepoint.
873 CallInst *CreateGCRelocate(Instruction *Statepoint,
874 int BaseOffset,
875 int DerivedOffset,
876 Type *ResultType,
877 const Twine &Name = "");
878
879 /// Create a call to the experimental.gc.pointer.base intrinsic to get the
880 /// base pointer for the specified derived pointer.
881 CallInst *CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name = "");
882
883 /// Create a call to the experimental.gc.get.pointer.offset intrinsic to get
884 /// the offset of the specified derived pointer from its base.
885 CallInst *CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name = "");
886
887 /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
888 /// will be the same type as that of \p Scaling.
889 Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
890
891 /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
892 Value *CreateStepVector(Type *DstType, const Twine &Name = "");
893
894 /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
895 /// type.
896 CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
897 Instruction *FMFSource = nullptr,
898 const Twine &Name = "");
899
900 /// Create a call to intrinsic \p ID with 2 operands which is mangled on the
901 /// first type.
902 CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
903 Instruction *FMFSource = nullptr,
904 const Twine &Name = "");
905
906 /// Create a call to intrinsic \p ID with \p Args, mangled using \p Types. If
907 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
908 /// the intrinsic.
909 CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
910 ArrayRef<Value *> Args,
911 Instruction *FMFSource = nullptr,
912 const Twine &Name = "");
913
914 /// Create a call to intrinsic \p ID with \p RetTy and \p Args. If
915 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
916 /// the intrinsic.
917 CallInst *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
918 ArrayRef<Value *> Args,
919 Instruction *FMFSource = nullptr,
920 const Twine &Name = "");
921
922 /// Create call to the minnum intrinsic.
923 CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
924 return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name);
925 }
926
927 /// Create call to the maxnum intrinsic.
928 CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
929 return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name);
930 }
931
932 /// Create call to the minimum intrinsic.
933 CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
934 return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
935 }
936
937 /// Create call to the maximum intrinsic.
938 CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
939 return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
940 }
941
942 /// Create a call to the arithmetic_fence intrinsic.
943 CallInst *CreateArithmeticFence(Value *Val, Type *DstType,
944 const Twine &Name = "") {
945 return CreateIntrinsic(Intrinsic::arithmetic_fence, DstType, Val, nullptr,
946 Name);
947 }
948
949 /// Create a call to the vector.extract intrinsic.
950 CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
951 const Twine &Name = "") {
952 return CreateIntrinsic(Intrinsic::vector_extract,
953 {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
954 Name);
955 }
956
957 /// Create a call to the vector.insert intrinsic.
958 CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
959 Value *Idx, const Twine &Name = "") {
960 return CreateIntrinsic(Intrinsic::vector_insert,
961 {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
962 nullptr, Name);
963 }
964
965private:
966 /// Create a call to a masked intrinsic with given Id.
967 CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
968 ArrayRef<Type *> OverloadedTypes,
969 const Twine &Name = "");
970
971 Value *getCastedInt8PtrValue(Value *Ptr);
972
973 //===--------------------------------------------------------------------===//
974 // Instruction creation methods: Terminators
975 //===--------------------------------------------------------------------===//
976
977private:
978 /// Helper to add branch weight and unpredictable metadata onto an
979 /// instruction.
980 /// \returns The annotated instruction.
981 template <typename InstTy>
982 InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
983 if (Weights)
984 I->setMetadata(LLVMContext::MD_prof, Weights);
985 if (Unpredictable)
986 I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
987 return I;
988 }
989
990public:
991 /// Create a 'ret void' instruction.
992 ReturnInst *CreateRetVoid() {
993 return Insert(ReturnInst::Create(Context));
994 }
995
996 /// Create a 'ret <val>' instruction.
997 ReturnInst *CreateRet(Value *V) {
998 return Insert(ReturnInst::Create(Context, V));
999 }
1000
1001 /// Create a sequence of N insertvalue instructions,
1002 /// with one Value from the retVals array each, that build a aggregate
1003 /// return value one value at a time, and a ret instruction to return
1004 /// the resulting aggregate value.
1005 ///
1006 /// This is a convenience function for code that uses aggregate return values
1007 /// as a vehicle for having multiple return values.
1008 ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) {
1009 Value *V = PoisonValue::get(getCurrentFunctionReturnType());
1010 for (unsigned i = 0; i != N; ++i)
1011 V = CreateInsertValue(V, retVals[i], i, "mrv");
1012 return Insert(ReturnInst::Create(Context, V));
1013 }
1014
1015 /// Create an unconditional 'br label X' instruction.
1016 BranchInst *CreateBr(BasicBlock *Dest) {
1017 return Insert(BranchInst::Create(Dest));
1018 }
1019
1020 /// Create a conditional 'br Cond, TrueDest, FalseDest'
1021 /// instruction.
1022 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
1023 MDNode *BranchWeights = nullptr,
1024 MDNode *Unpredictable = nullptr) {
1025 return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
1026 BranchWeights, Unpredictable));
1027 }
1028
1029 /// Create a conditional 'br Cond, TrueDest, FalseDest'
1030 /// instruction. Copy branch meta data if available.
1031 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
1032 Instruction *MDSrc) {
1033 BranchInst *Br = BranchInst::Create(True, False, Cond);
1034 if (MDSrc) {
1035 unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
1036 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
1037 Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4));
1038 }
1039 return Insert(Br);
1040 }
1041
1042 /// Create a switch instruction with the specified value, default dest,
1043 /// and with a hint for the number of cases that will be added (for efficient
1044 /// allocation).
1045 SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
1046 MDNode *BranchWeights = nullptr,
1047 MDNode *Unpredictable = nullptr) {
1048 return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
1049 BranchWeights, Unpredictable));
1050 }
1051
1052 /// Create an indirect branch instruction with the specified address
1053 /// operand, with an optional hint for the number of destinations that will be
1054 /// added (for efficient allocation).
1055 IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) {
1056 return Insert(IndirectBrInst::Create(Addr, NumDests));
1057 }
1058
1059 /// Create an invoke instruction.
1060 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1061 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1062 ArrayRef<Value *> Args,
1063 ArrayRef<OperandBundleDef> OpBundles,
1064 const Twine &Name = "") {
1065 InvokeInst *II =
1066 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles);
1067 if (IsFPConstrained)
1068 setConstrainedFPCallAttr(II);
1069 return Insert(II, Name);
1070 }
1071 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1072 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1073 ArrayRef<Value *> Args = None,
1074 const Twine &Name = "") {
1075 InvokeInst *II =
1076 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args);
1077 if (IsFPConstrained)
1078 setConstrainedFPCallAttr(II);
1079 return Insert(II, Name);
1080 }
1081
1082 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1083 BasicBlock *UnwindDest, ArrayRef<Value *> Args,
1084 ArrayRef<OperandBundleDef> OpBundles,
1085 const Twine &Name = "") {
1086 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1087 NormalDest, UnwindDest, Args, OpBundles, Name);
1088 }
1089
1090 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1091 BasicBlock *UnwindDest,
1092 ArrayRef<Value *> Args = None,
1093 const Twine &Name = "") {
1094 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1095 NormalDest, UnwindDest, Args, Name);
1096 }
1097
1098 /// \brief Create a callbr instruction.
1099 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1100 BasicBlock *DefaultDest,
1101 ArrayRef<BasicBlock *> IndirectDests,
1102 ArrayRef<Value *> Args = None,
1103 const Twine &Name = "") {
1104 return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
1105 Args), Name);
1106 }
1107 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1108 BasicBlock *DefaultDest,
1109 ArrayRef<BasicBlock *> IndirectDests,
1110 ArrayRef<Value *> Args,
1111 ArrayRef<OperandBundleDef> OpBundles,
1112 const Twine &Name = "") {
1113 return Insert(
1114 CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
1115 OpBundles), Name);
1116 }
1117
1118 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1119 ArrayRef<BasicBlock *> IndirectDests,
1120 ArrayRef<Value *> Args = None,
1121 const Twine &Name = "") {
1122 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1123 DefaultDest, IndirectDests, Args, Name);
1124 }
1125 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1126 ArrayRef<BasicBlock *> IndirectDests,
1127 ArrayRef<Value *> Args,
1128 ArrayRef<OperandBundleDef> OpBundles,
1129 const Twine &Name = "") {
1130 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1131 DefaultDest, IndirectDests, Args, Name);
1132 }
1133
1134 ResumeInst *CreateResume(Value *Exn) {
1135 return Insert(ResumeInst::Create(Exn));
1136 }
1137
1138 CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad,
1139 BasicBlock *UnwindBB = nullptr) {
1140 return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB));
1141 }
1142
1143 CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB,
1144 unsigned NumHandlers,
1145 const Twine &Name = "") {
1146 return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers),
1147 Name);
1148 }
1149
1150 CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args,
1151 const Twine &Name = "") {
1152 return Insert(CatchPadInst::Create(ParentPad, Args), Name);
1153 }
1154
1155 CleanupPadInst *CreateCleanupPad(Value *ParentPad,
1156 ArrayRef<Value *> Args = None,
1157 const Twine &Name = "") {
1158 return Insert(CleanupPadInst::Create(ParentPad, Args), Name);
1159 }
1160
1161 CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) {
1162 return Insert(CatchReturnInst::Create(CatchPad, BB));
1163 }
1164
1165 UnreachableInst *CreateUnreachable() {
1166 return Insert(new UnreachableInst(Context));
1167 }
1168
1169 //===--------------------------------------------------------------------===//
1170 // Instruction creation methods: Binary Operators
1171 //===--------------------------------------------------------------------===//
1172private:
1173 BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc,
1174 Value *LHS, Value *RHS,
1175 const Twine &Name,
1176 bool HasNUW, bool HasNSW) {
1177 BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name);
1178 if (HasNUW) BO->setHasNoUnsignedWrap();
1179 if (HasNSW) BO->setHasNoSignedWrap();
1180 return BO;
1181 }
1182
1183 Instruction *setFPAttrs(Instruction *I, MDNode *FPMD,
1184 FastMathFlags FMF) const {
1185 if (!FPMD)
1186 FPMD = DefaultFPMathTag;
1187 if (FPMD)
1188 I->setMetadata(LLVMContext::MD_fpmath, FPMD);
1189 I->setFastMathFlags(FMF);
1190 return I;
1191 }
1192
1193 Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) {
1194 RoundingMode UseRounding = DefaultConstrainedRounding;
1195
1196 if (Rounding)
1197 UseRounding = Rounding.value();
1198
1199 Optional<StringRef> RoundingStr = convertRoundingModeToStr(UseRounding);
1200 assert(RoundingStr && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1200, __extension__ __PRETTY_FUNCTION__
))
;
1201 auto *RoundingMDS = MDString::get(Context, RoundingStr.value());
1202
1203 return MetadataAsValue::get(Context, RoundingMDS);
1204 }
1205
1206 Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) {
1207 fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept;
1208
1209 if (Except)
1210 UseExcept = Except.value();
1211
1212 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(UseExcept);
1213 assert(ExceptStr && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1213, __extension__ __PRETTY_FUNCTION__
))
;
1214 auto *ExceptMDS = MDString::get(Context, ExceptStr.value());
1215
1216 return MetadataAsValue::get(Context, ExceptMDS);
1217 }
1218
1219 Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) {
1220 assert(CmpInst::isFPPredicate(Predicate) &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
1221 Predicate != CmpInst::FCMP_FALSE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
1222 Predicate != CmpInst::FCMP_TRUE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
1223 "Invalid constrained FP comparison predicate!")(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
;
1224
1225 StringRef PredicateStr = CmpInst::getPredicateName(Predicate);
1226 auto *PredicateMDS = MDString::get(Context, PredicateStr);
1227
1228 return MetadataAsValue::get(Context, PredicateMDS);
1229 }
1230
1231public:
1232 Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "",
1233 bool HasNUW = false, bool HasNSW = false) {
1234 if (Value *V =
1235 Folder.FoldNoWrapBinOp(Instruction::Add, LHS, RHS, HasNUW, HasNSW))
1236 return V;
1237 return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, HasNUW,
1238 HasNSW);
1239 }
1240
1241 Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1242 return CreateAdd(LHS, RHS, Name, false, true);
1243 }
1244
1245 Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1246 return CreateAdd(LHS, RHS, Name, true, false);
1247 }
1248
1249 Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "",
1250 bool HasNUW = false, bool HasNSW = false) {
1251 if (Value *V =
1252 Folder.FoldNoWrapBinOp(Instruction::Sub, LHS, RHS, HasNUW, HasNSW))
1253 return V;
1254 return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name, HasNUW,
1255 HasNSW);
1256 }
1257
1258 Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1259 return CreateSub(LHS, RHS, Name, false, true);
1260 }
1261
1262 Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1263 return CreateSub(LHS, RHS, Name, true, false);
1264 }
1265
1266 Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "",
1267 bool HasNUW = false, bool HasNSW = false) {
1268 if (Value *V =
1269 Folder.FoldNoWrapBinOp(Instruction::Mul, LHS, RHS, HasNUW, HasNSW))
1270 return V;
1271 return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name, HasNUW,
1272 HasNSW);
1273 }
1274
1275 Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1276 return CreateMul(LHS, RHS, Name, false, true);
1277 }
1278
1279 Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1280 return CreateMul(LHS, RHS, Name, true, false);
1281 }
1282
1283 Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1284 bool isExact = false) {
1285 if (Value *V = Folder.FoldExactBinOp(Instruction::UDiv, LHS, RHS, isExact))
1286 return V;
1287 if (!isExact)
1288 return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name);
1289 return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name);
1290 }
1291
1292 Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1293 return CreateUDiv(LHS, RHS, Name, true);
1294 }
1295
1296 Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1297 bool isExact = false) {
1298 if (Value *V = Folder.FoldExactBinOp(Instruction::SDiv, LHS, RHS, isExact))
1299 return V;
1300 if (!isExact)
1301 return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name);
1302 return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name);
1303 }
1304
1305 Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1306 return CreateSDiv(LHS, RHS, Name, true);
1307 }
1308
1309 Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") {
1310 if (Value *V = Folder.FoldBinOp(Instruction::URem, LHS, RHS))
1311 return V;
1312 return Insert(BinaryOperator::CreateURem(LHS, RHS), Name);
1313 }
1314
1315 Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") {
1316 if (Value *V = Folder.FoldBinOp(Instruction::SRem, LHS, RHS))
1317 return V;
1318 return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name);
1319 }
1320
1321 Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "",
1322 bool HasNUW = false, bool HasNSW = false) {
1323 if (Value *V =
1324 Folder.FoldNoWrapBinOp(Instruction::Shl, LHS, RHS, HasNUW, HasNSW))
1325 return V;
1326 return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name,
1327 HasNUW, HasNSW);
1328 }
1329
1330 Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "",
1331 bool HasNUW = false, bool HasNSW = false) {
1332 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1333 HasNUW, HasNSW);
1334 }
1335
1336 Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "",
1337 bool HasNUW = false, bool HasNSW = false) {
1338 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1339 HasNUW, HasNSW);
1340 }
1341
1342 Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "",
1343 bool isExact = false) {
1344 if (Value *V = Folder.FoldExactBinOp(Instruction::LShr, LHS, RHS, isExact))
1345 return V;
1346 if (!isExact)
1347 return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name);
1348 return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name);
1349 }
1350
1351 Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1352 bool isExact = false) {
1353 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1354 }
1355
1356 Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1357 bool isExact = false) {
1358 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1359 }
1360
1361 Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "",
1362 bool isExact = false) {
1363 if (Value *V = Folder.FoldExactBinOp(Instruction::AShr, LHS, RHS, isExact))
1364 return V;
1365 if (!isExact)
1366 return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name);
1367 return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name);
1368 }
1369
1370 Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1371 bool isExact = false) {
1372 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1373 }
1374
1375 Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1376 bool isExact = false) {
1377 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1378 }
1379
1380 Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") {
1381 if (auto *V = Folder.FoldBinOp(Instruction::And, LHS, RHS))
1382 return V;
1383 return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name);
1384 }
1385
1386 Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1387 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1388 }
1389
1390 Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1391 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1392 }
1393
1394 Value *CreateAnd(ArrayRef<Value*> Ops) {
1395 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1395, __extension__
__PRETTY_FUNCTION__))
;
1396 Value *Accum = Ops[0];
1397 for (unsigned i = 1; i < Ops.size(); i++)
1398 Accum = CreateAnd(Accum, Ops[i]);
1399 return Accum;
1400 }
1401
1402 Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
1403 if (auto *V = Folder.FoldBinOp(Instruction::Or, LHS, RHS))
1404 return V;
1405 return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
1406 }
1407
1408 Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1409 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1410 }
1411
1412 Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1413 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1414 }
1415
1416 Value *CreateOr(ArrayRef<Value*> Ops) {
1417 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1417, __extension__
__PRETTY_FUNCTION__))
;
1418 Value *Accum = Ops[0];
1419 for (unsigned i = 1; i < Ops.size(); i++)
1420 Accum = CreateOr(Accum, Ops[i]);
1421 return Accum;
1422 }
1423
1424 Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") {
1425 if (Value *V = Folder.FoldBinOp(Instruction::Xor, LHS, RHS))
1426 return V;
1427 return Insert(BinaryOperator::CreateXor(LHS, RHS), Name);
1428 }
1429
1430 Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1431 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1432 }
1433
1434 Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1435 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1436 }
1437
1438 Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "",
1439 MDNode *FPMD = nullptr) {
1440 if (IsFPConstrained)
1441 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1442 L, R, nullptr, Name, FPMD);
1443
1444 if (Value *V = Folder.FoldBinOpFMF(Instruction::FAdd, L, R, FMF))
1445 return V;
1446 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF);
1447 return Insert(I, Name);
1448 }
1449
1450 /// Copy fast-math-flags from an instruction rather than using the builder's
1451 /// default FMF.
1452 Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource,
1453 const Twine &Name = "") {
1454 if (IsFPConstrained)
1455 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1456 L, R, FMFSource, Name);
1457
1458 FastMathFlags FMF = FMFSource->getFastMathFlags();
1459 if (Value *V = Folder.FoldBinOpFMF(Instruction::FAdd, L, R, FMF))
1460 return V;
1461 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, FMF);
1462 return Insert(I, Name);
1463 }
1464
1465 Value *CreateFSub(Value *L, Value *R, const Twine &Name = "",
1466 MDNode *FPMD = nullptr) {
1467 if (IsFPConstrained)
1468 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1469 L, R, nullptr, Name, FPMD);
1470
1471 if (Value *V = Folder.FoldBinOpFMF(Instruction::FSub, L, R, FMF))
1472 return V;
1473 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF);
1474 return Insert(I, Name);
1475 }
1476
1477 /// Copy fast-math-flags from an instruction rather than using the builder's
1478 /// default FMF.
1479 Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource,
1480 const Twine &Name = "") {
1481 if (IsFPConstrained)
1482 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1483 L, R, FMFSource, Name);
1484
1485 FastMathFlags FMF = FMFSource->getFastMathFlags();
1486 if (Value *V = Folder.FoldBinOpFMF(Instruction::FSub, L, R, FMF))
1487 return V;
1488 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, FMF);
1489 return Insert(I, Name);
1490 }
1491
1492 Value *CreateFMul(Value *L, Value *R, const Twine &Name = "",
1493 MDNode *FPMD = nullptr) {
1494 if (IsFPConstrained)
1495 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1496 L, R, nullptr, Name, FPMD);
1497
1498 if (Value *V = Folder.FoldBinOpFMF(Instruction::FMul, L, R, FMF))
1499 return V;
1500 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF);
1501 return Insert(I, Name);
1502 }
1503
1504 /// Copy fast-math-flags from an instruction rather than using the builder's
1505 /// default FMF.
1506 Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource,
1507 const Twine &Name = "") {
1508 if (IsFPConstrained)
1509 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1510 L, R, FMFSource, Name);
1511
1512 FastMathFlags FMF = FMFSource->getFastMathFlags();
1513 if (Value *V = Folder.FoldBinOpFMF(Instruction::FMul, L, R, FMF))
1514 return V;
1515 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, FMF);
1516 return Insert(I, Name);
1517 }
1518
1519 Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "",
1520 MDNode *FPMD = nullptr) {
1521 if (IsFPConstrained)
1522 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1523 L, R, nullptr, Name, FPMD);
1524
1525 if (Value *V = Folder.FoldBinOpFMF(Instruction::FDiv, L, R, FMF))
1526 return V;
1527 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF);
1528 return Insert(I, Name);
1529 }
1530
1531 /// Copy fast-math-flags from an instruction rather than using the builder's
1532 /// default FMF.
1533 Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource,
1534 const Twine &Name = "") {
1535 if (IsFPConstrained)
1536 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1537 L, R, FMFSource, Name);
1538
1539 if (Value *V = Folder.FoldBinOpFMF(Instruction::FDiv, L, R, FMF))
1540 return V;
1541 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, FMF);
1542 return Insert(I, Name);
1543 }
1544
1545 Value *CreateFRem(Value *L, Value *R, const Twine &Name = "",
1546 MDNode *FPMD = nullptr) {
1547 if (IsFPConstrained)
1548 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1549 L, R, nullptr, Name, FPMD);
1550
1551 if (Value *V = Folder.FoldBinOpFMF(Instruction::FRem, L, R, FMF)) return V;
1552 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF);
1553 return Insert(I, Name);
1554 }
1555
1556 /// Copy fast-math-flags from an instruction rather than using the builder's
1557 /// default FMF.
1558 Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource,
1559 const Twine &Name = "") {
1560 if (IsFPConstrained)
1561 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1562 L, R, FMFSource, Name);
1563
1564 FastMathFlags FMF = FMFSource->getFastMathFlags();
1565 if (Value *V = Folder.FoldBinOpFMF(Instruction::FRem, L, R, FMF)) return V;
1566 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, FMF);
1567 return Insert(I, Name);
1568 }
1569
1570 Value *CreateBinOp(Instruction::BinaryOps Opc,
1571 Value *LHS, Value *RHS, const Twine &Name = "",
1572 MDNode *FPMathTag = nullptr) {
1573 if (Value *V = Folder.FoldBinOp(Opc, LHS, RHS)) return V;
1574 Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS);
1575 if (isa<FPMathOperator>(BinOp))
1576 setFPAttrs(BinOp, FPMathTag, FMF);
1577 return Insert(BinOp, Name);
1578 }
1579
1580 Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1581 assert(Cond2->getType()->isIntOrIntVectorTy(1))(static_cast <bool> (Cond2->getType()->isIntOrIntVectorTy
(1)) ? void (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "llvm/include/llvm/IR/IRBuilder.h", 1581, __extension__ __PRETTY_FUNCTION__
))
;
1582 return CreateSelect(Cond1, Cond2,
1583 ConstantInt::getNullValue(Cond2->getType()), Name);
1584 }
1585
1586 Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1587 assert(Cond2->getType()->isIntOrIntVectorTy(1))(static_cast <bool> (Cond2->getType()->isIntOrIntVectorTy
(1)) ? void (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "llvm/include/llvm/IR/IRBuilder.h", 1587, __extension__ __PRETTY_FUNCTION__
))
;
1588 return CreateSelect(Cond1, ConstantInt::getAllOnesValue(Cond2->getType()),
1589 Cond2, Name);
1590 }
1591
1592 // NOTE: this is sequential, non-commutative, ordered reduction!
1593 Value *CreateLogicalOr(ArrayRef<Value *> Ops) {
1594 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1594, __extension__
__PRETTY_FUNCTION__))
;
1595 Value *Accum = Ops[0];
1596 for (unsigned i = 1; i < Ops.size(); i++)
1597 Accum = CreateLogicalOr(Accum, Ops[i]);
1598 return Accum;
1599 }
1600
1601 CallInst *CreateConstrainedFPBinOp(
1602 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr,
1603 const Twine &Name = "", MDNode *FPMathTag = nullptr,
1604 Optional<RoundingMode> Rounding = None,
1605 Optional<fp::ExceptionBehavior> Except = None);
1606
1607 Value *CreateNeg(Value *V, const Twine &Name = "", bool HasNUW = false,
1608 bool HasNSW = false) {
1609 return CreateSub(Constant::getNullValue(V->getType()), V, Name, HasNUW,
1610 HasNSW);
1611 }
1612
1613 Value *CreateNSWNeg(Value *V, const Twine &Name = "") {
1614 return CreateNeg(V, Name, false, true);
1615 }
1616
1617 Value *CreateNUWNeg(Value *V, const Twine &Name = "") {
1618 return CreateNeg(V, Name, true, false);
1619 }
1620
1621 Value *CreateFNeg(Value *V, const Twine &Name = "",
1622 MDNode *FPMathTag = nullptr) {
1623 if (Value *Res = Folder.FoldUnOpFMF(Instruction::FNeg, V, FMF))
1624 return Res;
1625 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF),
1626 Name);
1627 }
1628
1629 /// Copy fast-math-flags from an instruction rather than using the builder's
1630 /// default FMF.
1631 Value *CreateFNegFMF(Value *V, Instruction *FMFSource,
1632 const Twine &Name = "") {
1633 FastMathFlags FMF = FMFSource->getFastMathFlags();
1634 if (Value *Res = Folder.FoldUnOpFMF(Instruction::FNeg, V, FMF))
1635 return Res;
1636 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr, FMF),
1637 Name);
1638 }
1639
1640 Value *CreateNot(Value *V, const Twine &Name = "") {
1641 return CreateXor(V, Constant::getAllOnesValue(V->getType()), Name);
1642 }
1643
1644 Value *CreateUnOp(Instruction::UnaryOps Opc,
1645 Value *V, const Twine &Name = "",
1646 MDNode *FPMathTag = nullptr) {
1647 if (Value *Res = Folder.FoldUnOpFMF(Opc, V, FMF))
1648 return Res;
1649 Instruction *UnOp = UnaryOperator::Create(Opc, V);
1650 if (isa<FPMathOperator>(UnOp))
1651 setFPAttrs(UnOp, FPMathTag, FMF);
1652 return Insert(UnOp, Name);
1653 }
1654
1655 /// Create either a UnaryOperator or BinaryOperator depending on \p Opc.
1656 /// Correct number of operands must be passed accordingly.
1657 Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1658 const Twine &Name = "", MDNode *FPMathTag = nullptr);
1659
1660 //===--------------------------------------------------------------------===//
1661 // Instruction creation methods: Memory Instructions
1662 //===--------------------------------------------------------------------===//
1663
1664 AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace,
1665 Value *ArraySize = nullptr, const Twine &Name = "") {
1666 const DataLayout &DL = BB->getModule()->getDataLayout();
1667 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1668 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1669 }
1670
1671 AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr,
1672 const Twine &Name = "") {
1673 const DataLayout &DL = BB->getModule()->getDataLayout();
1674 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1675 unsigned AddrSpace = DL.getAllocaAddrSpace();
1676 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1677 }
1678
1679 /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of
1680 /// converting the string to 'bool' for the isVolatile parameter.
1681 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) {
1682 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1683 }
1684
1685 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
1686 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1687 }
1688
1689 LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile,
1690 const Twine &Name = "") {
1691 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), isVolatile, Name);
1692 }
1693
1694 StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
1695 return CreateAlignedStore(Val, Ptr, MaybeAlign(), isVolatile);
1696 }
1697
1698 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1699 const char *Name) {
1700 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1701 }
1702
1703 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1704 const Twine &Name = "") {
1705 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1706 }
1707
1708 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1709 bool isVolatile, const Twine &Name = "") {
1710 if (!Align) {
1711 const DataLayout &DL = BB->getModule()->getDataLayout();
1712 Align = DL.getABITypeAlign(Ty);
1713 }
1714 return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile, *Align), Name);
1715 }
1716
1717 StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align,
1718 bool isVolatile = false) {
1719 if (!Align) {
1720 const DataLayout &DL = BB->getModule()->getDataLayout();
1721 Align = DL.getABITypeAlign(Val->getType());
1722 }
1723 return Insert(new StoreInst(Val, Ptr, isVolatile, *Align));
1724 }
1725 FenceInst *CreateFence(AtomicOrdering Ordering,
1726 SyncScope::ID SSID = SyncScope::System,
1727 const Twine &Name = "") {
1728 return Insert(new FenceInst(Context, Ordering, SSID), Name);
1729 }
1730
1731 AtomicCmpXchgInst *
1732 CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
1733 AtomicOrdering SuccessOrdering,
1734 AtomicOrdering FailureOrdering,
1735 SyncScope::ID SSID = SyncScope::System) {
1736 if (!Align) {
1737 const DataLayout &DL = BB->getModule()->getDataLayout();
1738 Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
1739 }
1740
1741 return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
1742 FailureOrdering, SSID));
1743 }
1744
1745 AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
1746 Value *Val, MaybeAlign Align,
1747 AtomicOrdering Ordering,
1748 SyncScope::ID SSID = SyncScope::System) {
1749 if (!Align) {
1750 const DataLayout &DL = BB->getModule()->getDataLayout();
1751 Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
1752 }
1753
1754 return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
1755 }
1756
1757 Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1758 const Twine &Name = "", bool IsInBounds = false) {
1759 if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, IsInBounds))
1760 return V;
1761 return Insert(IsInBounds
1762 ? GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList)
1763 : GetElementPtrInst::Create(Ty, Ptr, IdxList),
1764 Name);
1765 }
1766
1767 Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1768 const Twine &Name = "") {
1769 return CreateGEP(Ty, Ptr, IdxList, Name, /* IsInBounds */ true);
1770 }
1771
1772 Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1773 const Twine &Name = "") {
1774 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1775
1776 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false))
1777 return V;
1778
1779 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1780 }
1781
1782 Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1783 const Twine &Name = "") {
1784 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1785
1786 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true))
1787 return V;
1788
1789 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1790 }
1791
1792 Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1,
1793 const Twine &Name = "") {
1794 Value *Idxs[] = {
1795 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1796 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1797 };
1798
1799 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false))
1800 return V;
1801
1802 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1803 }
1804
1805 Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0,
1806 unsigned Idx1, const Twine &Name = "") {
1807 Value *Idxs[] = {
1808 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1809 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1810 };
1811
1812 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true))
1813 return V;
1814
1815 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1816 }
1817
1818 Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1819 const Twine &Name = "") {
1820 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1821
1822 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false))
1823 return V;
1824
1825 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1826 }
1827
1828 Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1829 const Twine &Name = "") {
1830 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1831
1832 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true))
1833 return V;
1834
1835 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1836 }
1837
1838 Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1,
1839 const Twine &Name = "") {
1840 Value *Idxs[] = {
1841 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1842 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1843 };
1844
1845 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false))
1846 return V;
1847
1848 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1849 }
1850
1851 Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1852 uint64_t Idx1, const Twine &Name = "") {
1853 Value *Idxs[] = {
1854 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1855 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1856 };
1857
1858 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true))
1859 return V;
1860
1861 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1862 }
1863
1864 Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx,
1865 const Twine &Name = "") {
1866 return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name);
1867 }
1868
1869 /// Same as CreateGlobalString, but return a pointer with "i8*" type
1870 /// instead of a pointer to array of i8.
1871 ///
1872 /// If no module is given via \p M, it is take from the insertion point basic
1873 /// block.
1874 Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "",
1875 unsigned AddressSpace = 0,
1876 Module *M = nullptr) {
1877 GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace, M);
1878 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1879 Constant *Indices[] = {Zero, Zero};
1880 return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV,
1881 Indices);
1882 }
1883
1884 //===--------------------------------------------------------------------===//
1885 // Instruction creation methods: Cast/Conversion Operators
1886 //===--------------------------------------------------------------------===//
1887
1888 Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
1889 return CreateCast(Instruction::Trunc, V, DestTy, Name);
1890 }
1891
1892 Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") {
1893 return CreateCast(Instruction::ZExt, V, DestTy, Name);
1894 }
1895
1896 Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") {
1897 return CreateCast(Instruction::SExt, V, DestTy, Name);
1898 }
1899
1900 /// Create a ZExt or Trunc from the integer value V to DestTy. Return
1901 /// the value untouched if the type of V is already DestTy.
1902 Value *CreateZExtOrTrunc(Value *V, Type *DestTy,
1903 const Twine &Name = "") {
1904 assert(V->getType()->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1906, __extension__ __PRETTY_FUNCTION__
))
1905 DestTy->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1906, __extension__ __PRETTY_FUNCTION__
))
1906 "Can only zero extend/truncate integers!")(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1906, __extension__ __PRETTY_FUNCTION__
))
;
1907 Type *VTy = V->getType();
1908 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1909 return CreateZExt(V, DestTy, Name);
1910 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1911 return CreateTrunc(V, DestTy, Name);
1912 return V;
1913 }
1914
1915 /// Create a SExt or Trunc from the integer value V to DestTy. Return
1916 /// the value untouched if the type of V is already DestTy.
1917 Value *CreateSExtOrTrunc(Value *V, Type *DestTy,
1918 const Twine &Name = "") {
1919 assert(V->getType()->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1921, __extension__ __PRETTY_FUNCTION__
))
1920 DestTy->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1921, __extension__ __PRETTY_FUNCTION__
))
1921 "Can only sign extend/truncate integers!")(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1921, __extension__ __PRETTY_FUNCTION__
))
;
1922 Type *VTy = V->getType();
1923 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1924 return CreateSExt(V, DestTy, Name);
1925 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1926 return CreateTrunc(V, DestTy, Name);
1927 return V;
1928 }
1929
1930 Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") {
1931 if (IsFPConstrained)
1932 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui,
1933 V, DestTy, nullptr, Name);
1934 return CreateCast(Instruction::FPToUI, V, DestTy, Name);
1935 }
1936
1937 Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") {
1938 if (IsFPConstrained)
1939 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi,
1940 V, DestTy, nullptr, Name);
1941 return CreateCast(Instruction::FPToSI, V, DestTy, Name);
1942 }
1943
1944 Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
1945 if (IsFPConstrained)
1946 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp,
1947 V, DestTy, nullptr, Name);
1948 return CreateCast(Instruction::UIToFP, V, DestTy, Name);
1949 }
1950
1951 Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
1952 if (IsFPConstrained)
1953 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp,
1954 V, DestTy, nullptr, Name);
1955 return CreateCast(Instruction::SIToFP, V, DestTy, Name);
1956 }
1957
1958 Value *CreateFPTrunc(Value *V, Type *DestTy,
1959 const Twine &Name = "") {
1960 if (IsFPConstrained)
1961 return CreateConstrainedFPCast(
1962 Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr,
1963 Name);
1964 return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
1965 }
1966
1967 Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
1968 if (IsFPConstrained)
1969 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext,
1970 V, DestTy, nullptr, Name);
1971 return CreateCast(Instruction::FPExt, V, DestTy, Name);
1972 }
1973
1974 Value *CreatePtrToInt(Value *V, Type *DestTy,
1975 const Twine &Name = "") {
1976 return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
1977 }
1978
1979 Value *CreateIntToPtr(Value *V, Type *DestTy,
1980 const Twine &Name = "") {
1981 return CreateCast(Instruction::IntToPtr, V, DestTy, Name);
1982 }
1983
1984 Value *CreateBitCast(Value *V, Type *DestTy,
1985 const Twine &Name = "") {
1986 return CreateCast(Instruction::BitCast, V, DestTy, Name);
1987 }
1988
1989 Value *CreateAddrSpaceCast(Value *V, Type *DestTy,
1990 const Twine &Name = "") {
1991 return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name);
1992 }
1993
1994 Value *CreateZExtOrBitCast(Value *V, Type *DestTy,
1995 const Twine &Name = "") {
1996 if (V->getType() == DestTy)
1997 return V;
1998 if (auto *VC = dyn_cast<Constant>(V))
1999 return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name);
2000 return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name);
2001 }
2002
2003 Value *CreateSExtOrBitCast(Value *V, Type *DestTy,
2004 const Twine &Name = "") {
2005 if (V->getType() == DestTy)
2006 return V;
2007 if (auto *VC = dyn_cast<Constant>(V))
2008 return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name);
2009 return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name);
2010 }
2011
2012 Value *CreateTruncOrBitCast(Value *V, Type *DestTy,
2013 const Twine &Name = "") {
2014 if (V->getType() == DestTy)
2015 return V;
2016 if (auto *VC = dyn_cast<Constant>(V))
2017 return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name);
2018 return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name);
2019 }
2020
2021 Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
2022 const Twine &Name = "") {
2023 if (V->getType() == DestTy)
2024 return V;
2025 if (auto *VC = dyn_cast<Constant>(V))
2026 return Insert(Folder.CreateCast(Op, VC, DestTy), Name);
2027 return Insert(CastInst::Create(Op, V, DestTy), Name);
2028 }
2029
2030 Value *CreatePointerCast(Value *V, Type *DestTy,
2031 const Twine &Name = "") {
2032 if (V->getType() == DestTy)
2033 return V;
2034 if (auto *VC = dyn_cast<Constant>(V))
2035 return Insert(Folder.CreatePointerCast(VC, DestTy), Name);
2036 return Insert(CastInst::CreatePointerCast(V, DestTy), Name);
2037 }
2038
2039 Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy,
2040 const Twine &Name = "") {
2041 if (V->getType() == DestTy)
2042 return V;
2043
2044 if (auto *VC = dyn_cast<Constant>(V)) {
2045 return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy),
2046 Name);
2047 }
2048
2049 return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy),
2050 Name);
2051 }
2052
2053 Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned,
2054 const Twine &Name = "") {
2055 if (V->getType() == DestTy)
2056 return V;
2057 if (auto *VC = dyn_cast<Constant>(V))
2058 return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name);
2059 return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name);
2060 }
2061
2062 Value *CreateBitOrPointerCast(Value *V, Type *DestTy,
2063 const Twine &Name = "") {
2064 if (V->getType() == DestTy)
2065 return V;
2066 if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy())
2067 return CreatePtrToInt(V, DestTy, Name);
2068 if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy())
2069 return CreateIntToPtr(V, DestTy, Name);
2070
2071 return CreateBitCast(V, DestTy, Name);
2072 }
2073
2074 Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
2075 if (V->getType() == DestTy)
2076 return V;
2077 if (auto *VC = dyn_cast<Constant>(V))
2078 return Insert(Folder.CreateFPCast(VC, DestTy), Name);
2079 return Insert(CastInst::CreateFPCast(V, DestTy), Name);
2080 }
2081
2082 CallInst *CreateConstrainedFPCast(
2083 Intrinsic::ID ID, Value *V, Type *DestTy,
2084 Instruction *FMFSource = nullptr, const Twine &Name = "",
2085 MDNode *FPMathTag = nullptr,
2086 Optional<RoundingMode> Rounding = None,
2087 Optional<fp::ExceptionBehavior> Except = None);
2088
2089 // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a
2090 // compile time error, instead of converting the string to bool for the
2091 // isSigned parameter.
2092 Value *CreateIntCast(Value *, Type *, const char *) = delete;
2093
2094 //===--------------------------------------------------------------------===//
2095 // Instruction creation methods: Compare Instructions
2096 //===--------------------------------------------------------------------===//
2097
2098 Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
2099 return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name);
2100 }
2101
2102 Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") {
2103 return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name);
2104 }
2105
2106 Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2107 return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name);
2108 }
2109
2110 Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2111 return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name);
2112 }
2113
2114 Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") {
2115 return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name);
2116 }
2117
2118 Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") {
2119 return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name);
2120 }
2121
2122 Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2123 return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name);
2124 }
2125
2126 Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2127 return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name);
2128 }
2129
2130 Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") {
2131 return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name);
2132 }
2133
2134 Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") {
2135 return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name);
2136 }
2137
2138 Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2139 MDNode *FPMathTag = nullptr) {
2140 return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag);
2141 }
2142
2143 Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "",
2144 MDNode *FPMathTag = nullptr) {
2145 return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag);
2146 }
2147
2148 Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "",
2149 MDNode *FPMathTag = nullptr) {
2150 return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag);
2151 }
2152
2153 Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "",
2154 MDNode *FPMathTag = nullptr) {
2155 return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag);
2156 }
2157
2158 Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "",
2159 MDNode *FPMathTag = nullptr) {
2160 return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag);
2161 }
2162
2163 Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "",
2164 MDNode *FPMathTag = nullptr) {
2165 return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag);
2166 }
2167
2168 Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "",
2169 MDNode *FPMathTag = nullptr) {
2170 return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag);
2171 }
2172
2173 Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "",
2174 MDNode *FPMathTag = nullptr) {
2175 return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag);
2176 }
2177
2178 Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2179 MDNode *FPMathTag = nullptr) {
2180 return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag);
2181 }
2182
2183 Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "",
2184 MDNode *FPMathTag = nullptr) {
2185 return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag);
2186 }
2187
2188 Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "",
2189 MDNode *FPMathTag = nullptr) {
2190 return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag);
2191 }
2192
2193 Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "",
2194 MDNode *FPMathTag = nullptr) {
2195 return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag);
2196 }
2197
2198 Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "",
2199 MDNode *FPMathTag = nullptr) {
2200 return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag);
2201 }
2202
2203 Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "",
2204 MDNode *FPMathTag = nullptr) {
2205 return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag);
2206 }
2207
2208 Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2209 const Twine &Name = "") {
2210 if (auto *V = Folder.FoldICmp(P, LHS, RHS))
2211 return V;
2212 return Insert(new ICmpInst(P, LHS, RHS), Name);
2213 }
2214
2215 // Create a quiet floating-point comparison (i.e. one that raises an FP
2216 // exception only in the case where an input is a signaling NaN).
2217 // Note that this differs from CreateFCmpS only if IsFPConstrained is true.
2218 Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2219 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2220 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, false);
2221 }
2222
2223 Value *CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
2224 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2225 return CmpInst::isFPPredicate(Pred)
2226 ? CreateFCmp(Pred, LHS, RHS, Name, FPMathTag)
2227 : CreateICmp(Pred, LHS, RHS, Name);
2228 }
2229
2230 // Create a signaling floating-point comparison (i.e. one that raises an FP
2231 // exception whenever an input is any NaN, signaling or quiet).
2232 // Note that this differs from CreateFCmp only if IsFPConstrained is true.
2233 Value *CreateFCmpS(CmpInst::Predicate P, Value *LHS, Value *RHS,
2234 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2235 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, true);
2236 }
2237
2238private:
2239 // Helper routine to create either a signaling or a quiet FP comparison.
2240 Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
2241 const Twine &Name, MDNode *FPMathTag,
2242 bool IsSignaling);
2243
2244public:
2245 CallInst *CreateConstrainedFPCmp(
2246 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
2247 const Twine &Name = "", Optional<fp::ExceptionBehavior> Except = None);
2248
2249 //===--------------------------------------------------------------------===//
2250 // Instruction creation methods: Other Instructions
2251 //===--------------------------------------------------------------------===//
2252
2253 PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
2254 const Twine &Name = "") {
2255 PHINode *Phi = PHINode::Create(Ty, NumReservedValues);
2256 if (isa<FPMathOperator>(Phi))
2257 setFPAttrs(Phi, nullptr /* MDNode* */, FMF);
2258 return Insert(Phi, Name);
2259 }
2260
2261private:
2262 CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
2263 const Twine &Name = "",
2264 Instruction *FMFSource = nullptr,
2265 ArrayRef<OperandBundleDef> OpBundles = {});
2266
2267public:
2268 CallInst *CreateCall(FunctionType *FTy, Value *Callee,
2269 ArrayRef<Value *> Args = None, const Twine &Name = "",
2270 MDNode *FPMathTag = nullptr) {
2271 CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
2272 if (IsFPConstrained)
2273 setConstrainedFPCallAttr(CI);
2274 if (isa<FPMathOperator>(CI))
2275 setFPAttrs(CI, FPMathTag, FMF);
2276 return Insert(CI, Name);
2277 }
2278
2279 CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
2280 ArrayRef<OperandBundleDef> OpBundles,
2281 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2282 CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
2283 if (IsFPConstrained)
2284 setConstrainedFPCallAttr(CI);
2285 if (isa<FPMathOperator>(CI))
2286 setFPAttrs(CI, FPMathTag, FMF);
2287 return Insert(CI, Name);
2288 }
2289
2290 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None,
2291 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2292 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name,
2293 FPMathTag);
2294 }
2295
2296 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args,
2297 ArrayRef<OperandBundleDef> OpBundles,
2298 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2299 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args,
2300 OpBundles, Name, FPMathTag);
2301 }
2302
2303 CallInst *CreateConstrainedFPCall(
2304 Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
2305 Optional<RoundingMode> Rounding = None,
2306 Optional<fp::ExceptionBehavior> Except = None);
2307
2308 Value *CreateSelect(Value *C, Value *True, Value *False,
2309 const Twine &Name = "", Instruction *MDFrom = nullptr);
2310
2311 VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
2312 return Insert(new VAArgInst(List, Ty), Name);
2313 }
2314
2315 Value *CreateExtractElement(Value *Vec, Value *Idx,
2316 const Twine &Name = "") {
2317 if (Value *V = Folder.FoldExtractElement(Vec, Idx))
2318 return V;
2319 return Insert(ExtractElementInst::Create(Vec, Idx), Name);
2320 }
2321
2322 Value *CreateExtractElement(Value *Vec, uint64_t Idx,
2323 const Twine &Name = "") {
2324 return CreateExtractElement(Vec, getInt64(Idx), Name);
2325 }
2326
2327 Value *CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx,
2328 const Twine &Name = "") {
2329 return CreateInsertElement(PoisonValue::get(VecTy), NewElt, Idx, Name);
2330 }
2331
2332 Value *CreateInsertElement(Type *VecTy, Value *NewElt, uint64_t Idx,
2333 const Twine &Name = "") {
2334 return CreateInsertElement(PoisonValue::get(VecTy), NewElt, Idx, Name);
2335 }
2336
2337 Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx,
2338 const Twine &Name = "") {
2339 if (Value *V = Folder.FoldInsertElement(Vec, NewElt, Idx))
2340 return V;
2341 return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name);
2342 }
2343
2344 Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx,
2345 const Twine &Name = "") {
2346 return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
2347 }
2348
2349 Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
2350 const Twine &Name = "") {
2351 SmallVector<int, 16> IntMask;
2352 ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
2353 return CreateShuffleVector(V1, V2, IntMask, Name);
2354 }
2355
2356 /// See class ShuffleVectorInst for a description of the mask representation.
2357 Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
2358 const Twine &Name = "") {
2359 if (Value *V = Folder.FoldShuffleVector(V1, V2, Mask))
2360 return V;
2361 return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
2362 }
2363
2364 /// Create a unary shuffle. The second vector operand of the IR instruction
2365 /// is poison.
2366 Value *CreateShuffleVector(Value *V, ArrayRef<int> Mask,
2367 const Twine &Name = "") {
2368 return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name);
2369 }
2370
2371 Value *CreateExtractValue(Value *Agg, ArrayRef<unsigned> Idxs,
2372 const Twine &Name = "") {
2373 if (auto *V = Folder.FoldExtractValue(Agg, Idxs))
2374 return V;
2375 return Insert(ExtractValueInst::Create(Agg, Idxs), Name);
2376 }
2377
2378 Value *CreateInsertValue(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2379 const Twine &Name = "") {
2380 if (auto *V = Folder.FoldInsertValue(Agg, Val, Idxs))
2381 return V;
2382 return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
2383 }
2384
2385 LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
2386 const Twine &Name = "") {
2387 return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
2388 }
2389
2390 Value *CreateFreeze(Value *V, const Twine &Name = "") {
2391 return Insert(new FreezeInst(V), Name);
2392 }
2393
2394 //===--------------------------------------------------------------------===//
2395 // Utility creation methods
2396 //===--------------------------------------------------------------------===//
2397
2398 /// Return a boolean value testing if \p Arg == 0.
2399 Value *CreateIsNull(Value *Arg, const Twine &Name = "") {
2400 return CreateICmpEQ(Arg, ConstantInt::getNullValue(Arg->getType()), Name);
2401 }
2402
2403 /// Return a boolean value testing if \p Arg != 0.
2404 Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") {
2405 return CreateICmpNE(Arg, ConstantInt::getNullValue(Arg->getType()), Name);
2406 }
2407
2408 /// Return a boolean value testing if \p Arg < 0.
2409 Value *CreateIsNeg(Value *Arg, const Twine &Name = "") {
2410 return CreateICmpSLT(Arg, ConstantInt::getNullValue(Arg->getType()), Name);
2411 }
2412
2413 /// Return a boolean value testing if \p Arg > -1.
2414 Value *CreateIsNotNeg(Value *Arg, const Twine &Name = "") {
2415 return CreateICmpSGT(Arg, ConstantInt::getAllOnesValue(Arg->getType()),
2416 Name);
2417 }
2418
2419 /// Return the i64 difference between two pointer values, dividing out
2420 /// the size of the pointed-to objects.
2421 ///
2422 /// This is intended to implement C-style pointer subtraction. As such, the
2423 /// pointers must be appropriately aligned for their element types and
2424 /// pointing into the same object.
2425 Value *CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
2426 const Twine &Name = "");
2427
2428 /// Create a launder.invariant.group intrinsic call. If Ptr type is
2429 /// different from pointer to i8, it's casted to pointer to i8 in the same
2430 /// address space before call and casted back to Ptr type after call.
2431 Value *CreateLaunderInvariantGroup(Value *Ptr);
2432
2433 /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is
2434 /// different from pointer to i8, it's casted to pointer to i8 in the same
2435 /// address space before call and casted back to Ptr type after call.
2436 Value *CreateStripInvariantGroup(Value *Ptr);
2437
2438 /// Return a vector value that contains the vector V reversed
2439 Value *CreateVectorReverse(Value *V, const Twine &Name = "");
2440
2441 /// Return a vector splice intrinsic if using scalable vectors, otherwise
2442 /// return a shufflevector. If the immediate is positive, a vector is
2443 /// extracted from concat(V1, V2), starting at Imm. If the immediate
2444 /// is negative, we extract -Imm elements from V1 and the remaining
2445 /// elements from V2. Imm is a signed integer in the range
2446 /// -VL <= Imm < VL (where VL is the runtime vector length of the
2447 /// source/result vector)
2448 Value *CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
2449 const Twine &Name = "");
2450
2451 /// Return a vector value that contains \arg V broadcasted to \p
2452 /// NumElts elements.
2453 Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
2454
2455 /// Return a vector value that contains \arg V broadcasted to \p
2456 /// EC elements.
2457 Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");
2458
2459 /// Return a value that has been extracted from a larger integer type.
2460 Value *CreateExtractInteger(const DataLayout &DL, Value *From,
2461 IntegerType *ExtractedTy, uint64_t Offset,
2462 const Twine &Name);
2463
2464 Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
2465 unsigned Dimension, unsigned LastIndex,
2466 MDNode *DbgInfo);
2467
2468 Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex,
2469 MDNode *DbgInfo);
2470
2471 Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
2472 unsigned Index, unsigned FieldIndex,
2473 MDNode *DbgInfo);
2474
2475private:
2476 /// Helper function that creates an assume intrinsic call that
2477 /// represents an alignment assumption on the provided pointer \p PtrValue
2478 /// with offset \p OffsetValue and alignment value \p AlignValue.
2479 CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
2480 Value *PtrValue, Value *AlignValue,
2481 Value *OffsetValue);
2482
2483public:
2484 /// Create an assume intrinsic call that represents an alignment
2485 /// assumption on the provided pointer.
2486 ///
2487 /// An optional offset can be provided, and if it is provided, the offset
2488 /// must be subtracted from the provided pointer to get the pointer with the
2489 /// specified alignment.
2490 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2491 unsigned Alignment,
2492 Value *OffsetValue = nullptr);
2493
2494 /// Create an assume intrinsic call that represents an alignment
2495 /// assumption on the provided pointer.
2496 ///
2497 /// An optional offset can be provided, and if it is provided, the offset
2498 /// must be subtracted from the provided pointer to get the pointer with the
2499 /// specified alignment.
2500 ///
2501 /// This overload handles the condition where the Alignment is dependent
2502 /// on an existing value rather than a static value.
2503 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2504 Value *Alignment,
2505 Value *OffsetValue = nullptr);
2506};
2507
2508/// This provides a uniform API for creating instructions and inserting
2509/// them into a basic block: either at the end of a BasicBlock, or at a specific
2510/// iterator location in a block.
2511///
2512/// Note that the builder does not expose the full generality of LLVM
2513/// instructions. For access to extra instruction properties, use the mutators
2514/// (e.g. setVolatile) on the instructions after they have been
2515/// created. Convenience state exists to specify fast-math flags and fp-math
2516/// tags.
2517///
2518/// The first template argument specifies a class to use for creating constants.
2519/// This defaults to creating minimally folded constants. The second template
2520/// argument allows clients to specify custom insertion hooks that are called on
2521/// every newly created insertion.
2522template <typename FolderTy = ConstantFolder,
2523 typename InserterTy = IRBuilderDefaultInserter>
2524class IRBuilder : public IRBuilderBase {
2525private:
2526 FolderTy Folder;
2527 InserterTy Inserter;
2528
2529public:
2530 IRBuilder(LLVMContext &C, FolderTy Folder, InserterTy Inserter = InserterTy(),
2531 MDNode *FPMathTag = nullptr,
2532 ArrayRef<OperandBundleDef> OpBundles = None)
2533 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles),
2534 Folder(Folder), Inserter(Inserter) {}
2535
2536 explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr,
2537 ArrayRef<OperandBundleDef> OpBundles = None)
2538 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles) {}
2539
2540 explicit IRBuilder(BasicBlock *TheBB, FolderTy Folder,
2541 MDNode *FPMathTag = nullptr,
2542 ArrayRef<OperandBundleDef> OpBundles = None)
2543 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2544 FPMathTag, OpBundles), Folder(Folder) {
2545 SetInsertPoint(TheBB);
2546 }
2547
2548 explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr,
2549 ArrayRef<OperandBundleDef> OpBundles = None)
2550 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2551 FPMathTag, OpBundles) {
2552 SetInsertPoint(TheBB);
2553 }
2554
2555 explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr,
2556 ArrayRef<OperandBundleDef> OpBundles = None)
2557 : IRBuilderBase(IP->getContext(), this->Folder, this->Inserter,
22
Called C++ object pointer is null
2558 FPMathTag, OpBundles) {
2559 SetInsertPoint(IP);
2560 }
2561
2562 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, FolderTy Folder,
2563 MDNode *FPMathTag = nullptr,
2564 ArrayRef<OperandBundleDef> OpBundles = None)
2565 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2566 FPMathTag, OpBundles), Folder(Folder) {
2567 SetInsertPoint(TheBB, IP);
2568 }
2569
2570 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP,
2571 MDNode *FPMathTag = nullptr,
2572 ArrayRef<OperandBundleDef> OpBundles = None)
2573 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2574 FPMathTag, OpBundles) {
2575 SetInsertPoint(TheBB, IP);
2576 }
2577
2578 /// Avoid copying the full IRBuilder. Prefer using InsertPointGuard
2579 /// or FastMathFlagGuard instead.
2580 IRBuilder(const IRBuilder &) = delete;
2581
2582 InserterTy &getInserter() { return Inserter; }
2583};
2584
2585// Create wrappers for C Binding types (see CBindingWrapping.h).
2586DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast
<IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const
IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef
>(const_cast<IRBuilder<>*>(P)); }
2587
2588} // end namespace llvm
2589
2590#endif // LLVM_IR_IRBUILDER_H