Bug Summary

File:build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp
Warning:line 204, column 39
The result of the left shift is undefined due to shifting by '128', which is greater or equal to the width of type 'unsigned long long'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name MVEGatherScatterLowering.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/ARM -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/ARM -I include -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-10-03-140002-15933-1 -x c++ /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp

1//===- MVEGatherScatterLowering.cpp - Gather/Scatter lowering -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// This pass custom lowers llvm.gather and llvm.scatter instructions to
10/// arm.mve.gather and arm.mve.scatter intrinsics, optimising the code to
11/// produce a better final result as we go.
12//
13//===----------------------------------------------------------------------===//
14
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMSubtarget.h"
18#include "llvm/Analysis/LoopInfo.h"
19#include "llvm/Analysis/TargetTransformInfo.h"
20#include "llvm/Analysis/ValueTracking.h"
21#include "llvm/CodeGen/TargetLowering.h"
22#include "llvm/CodeGen/TargetPassConfig.h"
23#include "llvm/CodeGen/TargetSubtargetInfo.h"
24#include "llvm/InitializePasses.h"
25#include "llvm/IR/BasicBlock.h"
26#include "llvm/IR/Constant.h"
27#include "llvm/IR/Constants.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/InstrTypes.h"
31#include "llvm/IR/Instruction.h"
32#include "llvm/IR/Instructions.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsARM.h"
36#include "llvm/IR/IRBuilder.h"
37#include "llvm/IR/PatternMatch.h"
38#include "llvm/IR/Type.h"
39#include "llvm/IR/Value.h"
40#include "llvm/Pass.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Transforms/Utils/Local.h"
43#include <algorithm>
44#include <cassert>
45
46using namespace llvm;
47
48#define DEBUG_TYPE"arm-mve-gather-scatter-lowering" "arm-mve-gather-scatter-lowering"
49
50cl::opt<bool> EnableMaskedGatherScatters(
51 "enable-arm-maskedgatscat", cl::Hidden, cl::init(true),
52 cl::desc("Enable the generation of masked gathers and scatters"));
53
54namespace {
55
56class MVEGatherScatterLowering : public FunctionPass {
57public:
58 static char ID; // Pass identification, replacement for typeid
59
60 explicit MVEGatherScatterLowering() : FunctionPass(ID) {
61 initializeMVEGatherScatterLoweringPass(*PassRegistry::getPassRegistry());
62 }
63
64 bool runOnFunction(Function &F) override;
65
66 StringRef getPassName() const override {
67 return "MVE gather/scatter lowering";
68 }
69
70 void getAnalysisUsage(AnalysisUsage &AU) const override {
71 AU.setPreservesCFG();
72 AU.addRequired<TargetPassConfig>();
73 AU.addRequired<LoopInfoWrapperPass>();
74 FunctionPass::getAnalysisUsage(AU);
75 }
76
77private:
78 LoopInfo *LI = nullptr;
79 const DataLayout *DL;
80
81 // Check this is a valid gather with correct alignment
82 bool isLegalTypeAndAlignment(unsigned NumElements, unsigned ElemSize,
83 Align Alignment);
84 // Check whether Ptr is hidden behind a bitcast and look through it
85 void lookThroughBitcast(Value *&Ptr);
86 // Decompose a ptr into Base and Offsets, potentially using a GEP to return a
87 // scalar base and vector offsets, or else fallback to using a base of 0 and
88 // offset of Ptr where possible.
89 Value *decomposePtr(Value *Ptr, Value *&Offsets, int &Scale,
90 FixedVectorType *Ty, Type *MemoryTy,
91 IRBuilder<> &Builder);
92 // Check for a getelementptr and deduce base and offsets from it, on success
93 // returning the base directly and the offsets indirectly using the Offsets
94 // argument
95 Value *decomposeGEP(Value *&Offsets, FixedVectorType *Ty,
96 GetElementPtrInst *GEP, IRBuilder<> &Builder);
97 // Compute the scale of this gather/scatter instruction
98 int computeScale(unsigned GEPElemSize, unsigned MemoryElemSize);
99 // If the value is a constant, or derived from constants via additions
100 // and multilications, return its numeric value
101 Optional<int64_t> getIfConst(const Value *V);
102 // If Inst is an add instruction, check whether one summand is a
103 // constant. If so, scale this constant and return it together with
104 // the other summand.
105 std::pair<Value *, int64_t> getVarAndConst(Value *Inst, int TypeScale);
106
107 Instruction *lowerGather(IntrinsicInst *I);
108 // Create a gather from a base + vector of offsets
109 Instruction *tryCreateMaskedGatherOffset(IntrinsicInst *I, Value *Ptr,
110 Instruction *&Root,
111 IRBuilder<> &Builder);
112 // Create a gather from a vector of pointers
113 Instruction *tryCreateMaskedGatherBase(IntrinsicInst *I, Value *Ptr,
114 IRBuilder<> &Builder,
115 int64_t Increment = 0);
116 // Create an incrementing gather from a vector of pointers
117 Instruction *tryCreateMaskedGatherBaseWB(IntrinsicInst *I, Value *Ptr,
118 IRBuilder<> &Builder,
119 int64_t Increment = 0);
120
121 Instruction *lowerScatter(IntrinsicInst *I);
122 // Create a scatter to a base + vector of offsets
123 Instruction *tryCreateMaskedScatterOffset(IntrinsicInst *I, Value *Offsets,
124 IRBuilder<> &Builder);
125 // Create a scatter to a vector of pointers
126 Instruction *tryCreateMaskedScatterBase(IntrinsicInst *I, Value *Ptr,
127 IRBuilder<> &Builder,
128 int64_t Increment = 0);
129 // Create an incrementing scatter from a vector of pointers
130 Instruction *tryCreateMaskedScatterBaseWB(IntrinsicInst *I, Value *Ptr,
131 IRBuilder<> &Builder,
132 int64_t Increment = 0);
133
134 // QI gathers and scatters can increment their offsets on their own if
135 // the increment is a constant value (digit)
136 Instruction *tryCreateIncrementingGatScat(IntrinsicInst *I, Value *Ptr,
137 IRBuilder<> &Builder);
138 // QI gathers/scatters can increment their offsets on their own if the
139 // increment is a constant value (digit) - this creates a writeback QI
140 // gather/scatter
141 Instruction *tryCreateIncrementingWBGatScat(IntrinsicInst *I, Value *BasePtr,
142 Value *Ptr, unsigned TypeScale,
143 IRBuilder<> &Builder);
144
145 // Optimise the base and offsets of the given address
146 bool optimiseAddress(Value *Address, BasicBlock *BB, LoopInfo *LI);
147 // Try to fold consecutive geps together into one
148 Value *foldGEP(GetElementPtrInst *GEP, Value *&Offsets, unsigned &Scale,
149 IRBuilder<> &Builder);
150 // Check whether these offsets could be moved out of the loop they're in
151 bool optimiseOffsets(Value *Offsets, BasicBlock *BB, LoopInfo *LI);
152 // Pushes the given add out of the loop
153 void pushOutAdd(PHINode *&Phi, Value *OffsSecondOperand, unsigned StartIndex);
154 // Pushes the given mul or shl out of the loop
155 void pushOutMulShl(unsigned Opc, PHINode *&Phi, Value *IncrementPerRound,
156 Value *OffsSecondOperand, unsigned LoopIncrement,
157 IRBuilder<> &Builder);
158};
159
160} // end anonymous namespace
161
162char MVEGatherScatterLowering::ID = 0;
163
164INITIALIZE_PASS(MVEGatherScatterLowering, DEBUG_TYPE,static void *initializeMVEGatherScatterLoweringPassOnce(PassRegistry
&Registry) { PassInfo *PI = new PassInfo( "MVE gather/scattering lowering pass"
, "arm-mve-gather-scatter-lowering", &MVEGatherScatterLowering
::ID, PassInfo::NormalCtor_t(callDefaultCtor<MVEGatherScatterLowering
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeMVEGatherScatterLoweringPassFlag
; void llvm::initializeMVEGatherScatterLoweringPass(PassRegistry
&Registry) { llvm::call_once(InitializeMVEGatherScatterLoweringPassFlag
, initializeMVEGatherScatterLoweringPassOnce, std::ref(Registry
)); }
165 "MVE gather/scattering lowering pass", false, false)static void *initializeMVEGatherScatterLoweringPassOnce(PassRegistry
&Registry) { PassInfo *PI = new PassInfo( "MVE gather/scattering lowering pass"
, "arm-mve-gather-scatter-lowering", &MVEGatherScatterLowering
::ID, PassInfo::NormalCtor_t(callDefaultCtor<MVEGatherScatterLowering
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeMVEGatherScatterLoweringPassFlag
; void llvm::initializeMVEGatherScatterLoweringPass(PassRegistry
&Registry) { llvm::call_once(InitializeMVEGatherScatterLoweringPassFlag
, initializeMVEGatherScatterLoweringPassOnce, std::ref(Registry
)); }
166
167Pass *llvm::createMVEGatherScatterLoweringPass() {
168 return new MVEGatherScatterLowering();
169}
170
171bool MVEGatherScatterLowering::isLegalTypeAndAlignment(unsigned NumElements,
172 unsigned ElemSize,
173 Align Alignment) {
174 if (((NumElements == 4 &&
175 (ElemSize == 32 || ElemSize == 16 || ElemSize == 8)) ||
176 (NumElements == 8 && (ElemSize == 16 || ElemSize == 8)) ||
177 (NumElements == 16 && ElemSize == 8)) &&
178 Alignment >= ElemSize / 8)
179 return true;
180 LLVM_DEBUG(dbgs() << "masked gathers/scatters: instruction does not have "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: instruction does not have "
<< "valid alignment or vector type \n"; } } while (false
)
181 << "valid alignment or vector type \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: instruction does not have "
<< "valid alignment or vector type \n"; } } while (false
)
;
182 return false;
183}
184
185static bool checkOffsetSize(Value *Offsets, unsigned TargetElemCount) {
186 // Offsets that are not of type <N x i32> are sign extended by the
187 // getelementptr instruction, and MVE gathers/scatters treat the offset as
188 // unsigned. Thus, if the element size is smaller than 32, we can only allow
189 // positive offsets - i.e., the offsets are not allowed to be variables we
190 // can't look into.
191 // Additionally, <N x i32> offsets have to either originate from a zext of a
192 // vector with element types smaller or equal the type of the gather we're
193 // looking at, or consist of constants that we can check are small enough
194 // to fit into the gather type.
195 // Thus we check that 0 < value < 2^TargetElemSize.
196 unsigned TargetElemSize = 128 / TargetElemCount;
38
'TargetElemSize' initialized to 128
197 unsigned OffsetElemSize = cast<FixedVectorType>(Offsets->getType())
39
The object is a 'CastReturnType'
198 ->getElementType()
199 ->getScalarSizeInBits();
200 if (OffsetElemSize != TargetElemSize || OffsetElemSize != 32) {
40
Assuming 'OffsetElemSize' is not equal to 'TargetElemSize'
201 Constant *ConstOff = dyn_cast<Constant>(Offsets);
41
Assuming 'Offsets' is a 'CastReturnType'
202 if (!ConstOff
41.1
'ConstOff' is non-null
41.1
'ConstOff' is non-null
)
42
Taking false branch
203 return false;
204 int64_t TargetElemMaxSize = (1ULL << TargetElemSize);
43
The result of the left shift is undefined due to shifting by '128', which is greater or equal to the width of type 'unsigned long long'
205 auto CheckValueSize = [TargetElemMaxSize](Value *OffsetElem) {
206 ConstantInt *OConst = dyn_cast<ConstantInt>(OffsetElem);
207 if (!OConst)
208 return false;
209 int SExtValue = OConst->getSExtValue();
210 if (SExtValue >= TargetElemMaxSize || SExtValue < 0)
211 return false;
212 return true;
213 };
214 if (isa<FixedVectorType>(ConstOff->getType())) {
215 for (unsigned i = 0; i < TargetElemCount; i++) {
216 if (!CheckValueSize(ConstOff->getAggregateElement(i)))
217 return false;
218 }
219 } else {
220 if (!CheckValueSize(ConstOff))
221 return false;
222 }
223 }
224 return true;
225}
226
227Value *MVEGatherScatterLowering::decomposePtr(Value *Ptr, Value *&Offsets,
228 int &Scale, FixedVectorType *Ty,
229 Type *MemoryTy,
230 IRBuilder<> &Builder) {
231 if (auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
232 if (Value *V = decomposeGEP(Offsets, Ty, GEP, Builder)) {
233 Scale =
234 computeScale(GEP->getSourceElementType()->getPrimitiveSizeInBits(),
235 MemoryTy->getScalarSizeInBits());
236 return Scale == -1 ? nullptr : V;
237 }
238 }
239
240 // If we couldn't use the GEP (or it doesn't exist), attempt to use a
241 // BasePtr of 0 with Ptr as the Offsets, so long as there are only 4
242 // elements.
243 FixedVectorType *PtrTy = cast<FixedVectorType>(Ptr->getType());
244 if (PtrTy->getNumElements() != 4 || MemoryTy->getScalarSizeInBits() == 32)
245 return nullptr;
246 Value *Zero = ConstantInt::get(Builder.getInt32Ty(), 0);
247 Value *BasePtr = Builder.CreateIntToPtr(Zero, Builder.getInt8PtrTy());
248 Offsets = Builder.CreatePtrToInt(
249 Ptr, FixedVectorType::get(Builder.getInt32Ty(), 4));
250 Scale = 0;
251 return BasePtr;
252}
253
254Value *MVEGatherScatterLowering::decomposeGEP(Value *&Offsets,
255 FixedVectorType *Ty,
256 GetElementPtrInst *GEP,
257 IRBuilder<> &Builder) {
258 if (!GEP) {
259 LLVM_DEBUG(dbgs() << "masked gathers/scatters: no getelementpointer "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: no getelementpointer "
<< "found\n"; } } while (false)
260 << "found\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: no getelementpointer "
<< "found\n"; } } while (false)
;
261 return nullptr;
262 }
263 LLVM_DEBUG(dbgs() << "masked gathers/scatters: getelementpointer found."do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: getelementpointer found."
<< " Looking at intrinsic for base + vector of offsets\n"
; } } while (false)
264 << " Looking at intrinsic for base + vector of offsets\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: getelementpointer found."
<< " Looking at intrinsic for base + vector of offsets\n"
; } } while (false)
;
265 Value *GEPPtr = GEP->getPointerOperand();
266 Offsets = GEP->getOperand(1);
267 if (GEPPtr->getType()->isVectorTy() ||
268 !isa<FixedVectorType>(Offsets->getType()))
269 return nullptr;
270
271 if (GEP->getNumOperands() != 2) {
272 LLVM_DEBUG(dbgs() << "masked gathers/scatters: getelementptr with too many"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: getelementptr with too many"
<< " operands. Expanding.\n"; } } while (false)
273 << " operands. Expanding.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: getelementptr with too many"
<< " operands. Expanding.\n"; } } while (false)
;
274 return nullptr;
275 }
276 Offsets = GEP->getOperand(1);
277 unsigned OffsetsElemCount =
278 cast<FixedVectorType>(Offsets->getType())->getNumElements();
279 // Paranoid check whether the number of parallel lanes is the same
280 assert(Ty->getNumElements() == OffsetsElemCount)(static_cast <bool> (Ty->getNumElements() == OffsetsElemCount
) ? void (0) : __assert_fail ("Ty->getNumElements() == OffsetsElemCount"
, "llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp", 280, __extension__
__PRETTY_FUNCTION__))
;
281
282 ZExtInst *ZextOffs = dyn_cast<ZExtInst>(Offsets);
283 if (ZextOffs)
284 Offsets = ZextOffs->getOperand(0);
285 FixedVectorType *OffsetType = cast<FixedVectorType>(Offsets->getType());
286
287 // If the offsets are already being zext-ed to <N x i32>, that relieves us of
288 // having to make sure that they won't overflow.
289 if (!ZextOffs || cast<FixedVectorType>(ZextOffs->getDestTy())
290 ->getElementType()
291 ->getScalarSizeInBits() != 32)
292 if (!checkOffsetSize(Offsets, OffsetsElemCount))
293 return nullptr;
294
295 // The offset sizes have been checked; if any truncating or zext-ing is
296 // required to fix them, do that now
297 if (Ty != Offsets->getType()) {
298 if ((Ty->getElementType()->getScalarSizeInBits() <
299 OffsetType->getElementType()->getScalarSizeInBits())) {
300 Offsets = Builder.CreateTrunc(Offsets, Ty);
301 } else {
302 Offsets = Builder.CreateZExt(Offsets, VectorType::getInteger(Ty));
303 }
304 }
305 // If none of the checks failed, return the gep's base pointer
306 LLVM_DEBUG(dbgs() << "masked gathers/scatters: found correct offsets\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: found correct offsets\n"
; } } while (false)
;
307 return GEPPtr;
308}
309
310void MVEGatherScatterLowering::lookThroughBitcast(Value *&Ptr) {
311 // Look through bitcast instruction if #elements is the same
312 if (auto *BitCast = dyn_cast<BitCastInst>(Ptr)) {
313 auto *BCTy = cast<FixedVectorType>(BitCast->getType());
314 auto *BCSrcTy = cast<FixedVectorType>(BitCast->getOperand(0)->getType());
315 if (BCTy->getNumElements() == BCSrcTy->getNumElements()) {
316 LLVM_DEBUG(dbgs() << "masked gathers/scatters: looking through "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: looking through "
<< "bitcast\n"; } } while (false)
317 << "bitcast\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: looking through "
<< "bitcast\n"; } } while (false)
;
318 Ptr = BitCast->getOperand(0);
319 }
320 }
321}
322
323int MVEGatherScatterLowering::computeScale(unsigned GEPElemSize,
324 unsigned MemoryElemSize) {
325 // This can be a 32bit load/store scaled by 4, a 16bit load/store scaled by 2,
326 // or a 8bit, 16bit or 32bit load/store scaled by 1
327 if (GEPElemSize == 32 && MemoryElemSize == 32)
328 return 2;
329 else if (GEPElemSize == 16 && MemoryElemSize == 16)
330 return 1;
331 else if (GEPElemSize == 8)
332 return 0;
333 LLVM_DEBUG(dbgs() << "masked gathers/scatters: incorrect scale. Can't "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: incorrect scale. Can't "
<< "create intrinsic\n"; } } while (false)
334 << "create intrinsic\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: incorrect scale. Can't "
<< "create intrinsic\n"; } } while (false)
;
335 return -1;
336}
337
338Optional<int64_t> MVEGatherScatterLowering::getIfConst(const Value *V) {
339 const Constant *C = dyn_cast<Constant>(V);
340 if (C && C->getSplatValue())
341 return Optional<int64_t>{C->getUniqueInteger().getSExtValue()};
342 if (!isa<Instruction>(V))
343 return Optional<int64_t>{};
344
345 const Instruction *I = cast<Instruction>(V);
346 if (I->getOpcode() == Instruction::Add || I->getOpcode() == Instruction::Or ||
347 I->getOpcode() == Instruction::Mul ||
348 I->getOpcode() == Instruction::Shl) {
349 Optional<int64_t> Op0 = getIfConst(I->getOperand(0));
350 Optional<int64_t> Op1 = getIfConst(I->getOperand(1));
351 if (!Op0 || !Op1)
352 return Optional<int64_t>{};
353 if (I->getOpcode() == Instruction::Add)
354 return Optional<int64_t>{Op0.value() + Op1.value()};
355 if (I->getOpcode() == Instruction::Mul)
356 return Optional<int64_t>{Op0.value() * Op1.value()};
357 if (I->getOpcode() == Instruction::Shl)
358 return Optional<int64_t>{Op0.value() << Op1.value()};
359 if (I->getOpcode() == Instruction::Or)
360 return Optional<int64_t>{Op0.value() | Op1.value()};
361 }
362 return Optional<int64_t>{};
363}
364
365// Return true if I is an Or instruction that is equivalent to an add, due to
366// the operands having no common bits set.
367static bool isAddLikeOr(Instruction *I, const DataLayout &DL) {
368 return I->getOpcode() == Instruction::Or &&
369 haveNoCommonBitsSet(I->getOperand(0), I->getOperand(1), DL);
370}
371
372std::pair<Value *, int64_t>
373MVEGatherScatterLowering::getVarAndConst(Value *Inst, int TypeScale) {
374 std::pair<Value *, int64_t> ReturnFalse =
375 std::pair<Value *, int64_t>(nullptr, 0);
376 // At this point, the instruction we're looking at must be an add or an
377 // add-like-or.
378 Instruction *Add = dyn_cast<Instruction>(Inst);
379 if (Add == nullptr ||
380 (Add->getOpcode() != Instruction::Add && !isAddLikeOr(Add, *DL)))
381 return ReturnFalse;
382
383 Value *Summand;
384 Optional<int64_t> Const;
385 // Find out which operand the value that is increased is
386 if ((Const = getIfConst(Add->getOperand(0))))
387 Summand = Add->getOperand(1);
388 else if ((Const = getIfConst(Add->getOperand(1))))
389 Summand = Add->getOperand(0);
390 else
391 return ReturnFalse;
392
393 // Check that the constant is small enough for an incrementing gather
394 int64_t Immediate = *Const << TypeScale;
395 if (Immediate > 512 || Immediate < -512 || Immediate % 4 != 0)
396 return ReturnFalse;
397
398 return std::pair<Value *, int64_t>(Summand, Immediate);
399}
400
401Instruction *MVEGatherScatterLowering::lowerGather(IntrinsicInst *I) {
402 using namespace PatternMatch;
403 LLVM_DEBUG(dbgs() << "masked gathers: checking transform preconditions\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: checking transform preconditions\n"
<< *I << "\n"; } } while (false)
404 << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: checking transform preconditions\n"
<< *I << "\n"; } } while (false)
;
405
406 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
407 // Attempt to turn the masked gather in I into a MVE intrinsic
408 // Potentially optimising the addressing modes as we do so.
409 auto *Ty = cast<FixedVectorType>(I->getType());
410 Value *Ptr = I->getArgOperand(0);
411 Align Alignment = cast<ConstantInt>(I->getArgOperand(1))->getAlignValue();
412 Value *Mask = I->getArgOperand(2);
413 Value *PassThru = I->getArgOperand(3);
414
415 if (!isLegalTypeAndAlignment(Ty->getNumElements(), Ty->getScalarSizeInBits(),
416 Alignment))
417 return nullptr;
418 lookThroughBitcast(Ptr);
419 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Unexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Unexpected pointer type\""
, "llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp", 419, __extension__
__PRETTY_FUNCTION__))
;
420
421 IRBuilder<> Builder(I->getContext());
422 Builder.SetInsertPoint(I);
423 Builder.SetCurrentDebugLocation(I->getDebugLoc());
424
425 Instruction *Root = I;
426
427 Instruction *Load = tryCreateIncrementingGatScat(I, Ptr, Builder);
428 if (!Load)
429 Load = tryCreateMaskedGatherOffset(I, Ptr, Root, Builder);
430 if (!Load)
431 Load = tryCreateMaskedGatherBase(I, Ptr, Builder);
432 if (!Load)
433 return nullptr;
434
435 if (!isa<UndefValue>(PassThru) && !match(PassThru, m_Zero())) {
436 LLVM_DEBUG(dbgs() << "masked gathers: found non-trivial passthru - "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: found non-trivial passthru - "
<< "creating select\n"; } } while (false)
437 << "creating select\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: found non-trivial passthru - "
<< "creating select\n"; } } while (false)
;
438 Load = SelectInst::Create(Mask, Load, PassThru);
439 Builder.Insert(Load);
440 }
441
442 Root->replaceAllUsesWith(Load);
443 Root->eraseFromParent();
444 if (Root != I)
445 // If this was an extending gather, we need to get rid of the sext/zext
446 // sext/zext as well as of the gather itself
447 I->eraseFromParent();
448
449 LLVM_DEBUG(dbgs() << "masked gathers: successfully built masked gather\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: successfully built masked gather\n"
<< *Load << "\n"; } } while (false)
450 << *Load << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: successfully built masked gather\n"
<< *Load << "\n"; } } while (false)
;
451 return Load;
452}
453
454Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBase(
455 IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
456 using namespace PatternMatch;
457 auto *Ty = cast<FixedVectorType>(I->getType());
458 LLVM_DEBUG(dbgs() << "masked gathers: loading from vector of pointers\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: loading from vector of pointers\n"
; } } while (false)
;
459 if (Ty->getNumElements() != 4 || Ty->getScalarSizeInBits() != 32)
460 // Can't build an intrinsic for this
461 return nullptr;
462 Value *Mask = I->getArgOperand(2);
463 if (match(Mask, m_One()))
464 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base,
465 {Ty, Ptr->getType()},
466 {Ptr, Builder.getInt32(Increment)});
467 else
468 return Builder.CreateIntrinsic(
469 Intrinsic::arm_mve_vldr_gather_base_predicated,
470 {Ty, Ptr->getType(), Mask->getType()},
471 {Ptr, Builder.getInt32(Increment), Mask});
472}
473
474Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherBaseWB(
475 IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
476 using namespace PatternMatch;
477 auto *Ty = cast<FixedVectorType>(I->getType());
478 LLVM_DEBUG(dbgs() << "masked gathers: loading from vector of pointers with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: loading from vector of pointers with "
<< "writeback\n"; } } while (false)
479 << "writeback\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: loading from vector of pointers with "
<< "writeback\n"; } } while (false)
;
480 if (Ty->getNumElements() != 4 || Ty->getScalarSizeInBits() != 32)
481 // Can't build an intrinsic for this
482 return nullptr;
483 Value *Mask = I->getArgOperand(2);
484 if (match(Mask, m_One()))
485 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vldr_gather_base_wb,
486 {Ty, Ptr->getType()},
487 {Ptr, Builder.getInt32(Increment)});
488 else
489 return Builder.CreateIntrinsic(
490 Intrinsic::arm_mve_vldr_gather_base_wb_predicated,
491 {Ty, Ptr->getType(), Mask->getType()},
492 {Ptr, Builder.getInt32(Increment), Mask});
493}
494
495Instruction *MVEGatherScatterLowering::tryCreateMaskedGatherOffset(
496 IntrinsicInst *I, Value *Ptr, Instruction *&Root, IRBuilder<> &Builder) {
497 using namespace PatternMatch;
498
499 Type *MemoryTy = I->getType();
500 Type *ResultTy = MemoryTy;
501
502 unsigned Unsigned = 1;
503 // The size of the gather was already checked in isLegalTypeAndAlignment;
504 // if it was not a full vector width an appropriate extend should follow.
505 auto *Extend = Root;
506 bool TruncResult = false;
507 if (MemoryTy->getPrimitiveSizeInBits() < 128) {
508 if (I->hasOneUse()) {
509 // If the gather has a single extend of the correct type, use an extending
510 // gather and replace the ext. In which case the correct root to replace
511 // is not the CallInst itself, but the instruction which extends it.
512 Instruction* User = cast<Instruction>(*I->users().begin());
513 if (isa<SExtInst>(User) &&
514 User->getType()->getPrimitiveSizeInBits() == 128) {
515 LLVM_DEBUG(dbgs() << "masked gathers: Incorporating extend: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Incorporating extend: "
<< *User << "\n"; } } while (false)
516 << *User << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Incorporating extend: "
<< *User << "\n"; } } while (false)
;
517 Extend = User;
518 ResultTy = User->getType();
519 Unsigned = 0;
520 } else if (isa<ZExtInst>(User) &&
521 User->getType()->getPrimitiveSizeInBits() == 128) {
522 LLVM_DEBUG(dbgs() << "masked gathers: Incorporating extend: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Incorporating extend: "
<< *ResultTy << "\n"; } } while (false)
523 << *ResultTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Incorporating extend: "
<< *ResultTy << "\n"; } } while (false)
;
524 Extend = User;
525 ResultTy = User->getType();
526 }
527 }
528
529 // If an extend hasn't been found and the type is an integer, create an
530 // extending gather and truncate back to the original type.
531 if (ResultTy->getPrimitiveSizeInBits() < 128 &&
532 ResultTy->isIntOrIntVectorTy()) {
533 ResultTy = ResultTy->getWithNewBitWidth(
534 128 / cast<FixedVectorType>(ResultTy)->getNumElements());
535 TruncResult = true;
536 LLVM_DEBUG(dbgs() << "masked gathers: Small input type, truncing to: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Small input type, truncing to: "
<< *ResultTy << "\n"; } } while (false)
537 << *ResultTy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Small input type, truncing to: "
<< *ResultTy << "\n"; } } while (false)
;
538 }
539
540 // The final size of the gather must be a full vector width
541 if (ResultTy->getPrimitiveSizeInBits() != 128) {
542 LLVM_DEBUG(dbgs() << "masked gathers: Extend needed but not provided "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Extend needed but not provided "
"from the correct type. Expanding\n"; } } while (false)
543 "from the correct type. Expanding\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers: Extend needed but not provided "
"from the correct type. Expanding\n"; } } while (false)
;
544 return nullptr;
545 }
546 }
547
548 Value *Offsets;
549 int Scale;
550 Value *BasePtr = decomposePtr(
551 Ptr, Offsets, Scale, cast<FixedVectorType>(ResultTy), MemoryTy, Builder);
552 if (!BasePtr)
553 return nullptr;
554
555 Root = Extend;
556 Value *Mask = I->getArgOperand(2);
557 Instruction *Load = nullptr;
558 if (!match(Mask, m_One()))
559 Load = Builder.CreateIntrinsic(
560 Intrinsic::arm_mve_vldr_gather_offset_predicated,
561 {ResultTy, BasePtr->getType(), Offsets->getType(), Mask->getType()},
562 {BasePtr, Offsets, Builder.getInt32(MemoryTy->getScalarSizeInBits()),
563 Builder.getInt32(Scale), Builder.getInt32(Unsigned), Mask});
564 else
565 Load = Builder.CreateIntrinsic(
566 Intrinsic::arm_mve_vldr_gather_offset,
567 {ResultTy, BasePtr->getType(), Offsets->getType()},
568 {BasePtr, Offsets, Builder.getInt32(MemoryTy->getScalarSizeInBits()),
569 Builder.getInt32(Scale), Builder.getInt32(Unsigned)});
570
571 if (TruncResult) {
572 Load = TruncInst::Create(Instruction::Trunc, Load, MemoryTy);
573 Builder.Insert(Load);
574 }
575 return Load;
576}
577
578Instruction *MVEGatherScatterLowering::lowerScatter(IntrinsicInst *I) {
579 using namespace PatternMatch;
580 LLVM_DEBUG(dbgs() << "masked scatters: checking transform preconditions\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: checking transform preconditions\n"
<< *I << "\n"; } } while (false)
581 << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: checking transform preconditions\n"
<< *I << "\n"; } } while (false)
;
582
583 // @llvm.masked.scatter.*(data, ptrs, alignment, mask)
584 // Attempt to turn the masked scatter in I into a MVE intrinsic
585 // Potentially optimising the addressing modes as we do so.
586 Value *Input = I->getArgOperand(0);
587 Value *Ptr = I->getArgOperand(1);
588 Align Alignment = cast<ConstantInt>(I->getArgOperand(2))->getAlignValue();
589 auto *Ty = cast<FixedVectorType>(Input->getType());
590
591 if (!isLegalTypeAndAlignment(Ty->getNumElements(), Ty->getScalarSizeInBits(),
592 Alignment))
593 return nullptr;
594
595 lookThroughBitcast(Ptr);
596 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Unexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Unexpected pointer type\""
, "llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp", 596, __extension__
__PRETTY_FUNCTION__))
;
597
598 IRBuilder<> Builder(I->getContext());
599 Builder.SetInsertPoint(I);
600 Builder.SetCurrentDebugLocation(I->getDebugLoc());
601
602 Instruction *Store = tryCreateIncrementingGatScat(I, Ptr, Builder);
603 if (!Store)
604 Store = tryCreateMaskedScatterOffset(I, Ptr, Builder);
605 if (!Store)
606 Store = tryCreateMaskedScatterBase(I, Ptr, Builder);
607 if (!Store)
608 return nullptr;
609
610 LLVM_DEBUG(dbgs() << "masked scatters: successfully built masked scatter\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: successfully built masked scatter\n"
<< *Store << "\n"; } } while (false)
611 << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: successfully built masked scatter\n"
<< *Store << "\n"; } } while (false)
;
612 I->eraseFromParent();
613 return Store;
614}
615
616Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBase(
617 IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
618 using namespace PatternMatch;
619 Value *Input = I->getArgOperand(0);
620 auto *Ty = cast<FixedVectorType>(Input->getType());
621 // Only QR variants allow truncating
622 if (!(Ty->getNumElements() == 4 && Ty->getScalarSizeInBits() == 32)) {
623 // Can't build an intrinsic for this
624 return nullptr;
625 }
626 Value *Mask = I->getArgOperand(3);
627 // int_arm_mve_vstr_scatter_base(_predicated) addr, offset, data(, mask)
628 LLVM_DEBUG(dbgs() << "masked scatters: storing to a vector of pointers\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: storing to a vector of pointers\n"
; } } while (false)
;
629 if (match(Mask, m_One()))
630 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base,
631 {Ptr->getType(), Input->getType()},
632 {Ptr, Builder.getInt32(Increment), Input});
633 else
634 return Builder.CreateIntrinsic(
635 Intrinsic::arm_mve_vstr_scatter_base_predicated,
636 {Ptr->getType(), Input->getType(), Mask->getType()},
637 {Ptr, Builder.getInt32(Increment), Input, Mask});
638}
639
640Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterBaseWB(
641 IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder, int64_t Increment) {
642 using namespace PatternMatch;
643 Value *Input = I->getArgOperand(0);
644 auto *Ty = cast<FixedVectorType>(Input->getType());
645 LLVM_DEBUG(dbgs() << "masked scatters: storing to a vector of pointers "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: storing to a vector of pointers "
<< "with writeback\n"; } } while (false)
646 << "with writeback\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: storing to a vector of pointers "
<< "with writeback\n"; } } while (false)
;
647 if (Ty->getNumElements() != 4 || Ty->getScalarSizeInBits() != 32)
648 // Can't build an intrinsic for this
649 return nullptr;
650 Value *Mask = I->getArgOperand(3);
651 if (match(Mask, m_One()))
652 return Builder.CreateIntrinsic(Intrinsic::arm_mve_vstr_scatter_base_wb,
653 {Ptr->getType(), Input->getType()},
654 {Ptr, Builder.getInt32(Increment), Input});
655 else
656 return Builder.CreateIntrinsic(
657 Intrinsic::arm_mve_vstr_scatter_base_wb_predicated,
658 {Ptr->getType(), Input->getType(), Mask->getType()},
659 {Ptr, Builder.getInt32(Increment), Input, Mask});
660}
661
662Instruction *MVEGatherScatterLowering::tryCreateMaskedScatterOffset(
663 IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
664 using namespace PatternMatch;
665 Value *Input = I->getArgOperand(0);
666 Value *Mask = I->getArgOperand(3);
667 Type *InputTy = Input->getType();
668 Type *MemoryTy = InputTy;
669
670 LLVM_DEBUG(dbgs() << "masked scatters: getelementpointer found. Storing"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: getelementpointer found. Storing"
<< " to base + vector of offsets\n"; } } while (false)
671 << " to base + vector of offsets\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: getelementpointer found. Storing"
<< " to base + vector of offsets\n"; } } while (false)
;
672 // If the input has been truncated, try to integrate that trunc into the
673 // scatter instruction (we don't care about alignment here)
674 if (TruncInst *Trunc = dyn_cast<TruncInst>(Input)) {
675 Value *PreTrunc = Trunc->getOperand(0);
676 Type *PreTruncTy = PreTrunc->getType();
677 if (PreTruncTy->getPrimitiveSizeInBits() == 128) {
678 Input = PreTrunc;
679 InputTy = PreTruncTy;
680 }
681 }
682 bool ExtendInput = false;
683 if (InputTy->getPrimitiveSizeInBits() < 128 &&
684 InputTy->isIntOrIntVectorTy()) {
685 // If we can't find a trunc to incorporate into the instruction, create an
686 // implicit one with a zext, so that we can still create a scatter. We know
687 // that the input type is 4x/8x/16x and of type i8/i16/i32, so any type
688 // smaller than 128 bits will divide evenly into a 128bit vector.
689 InputTy = InputTy->getWithNewBitWidth(
690 128 / cast<FixedVectorType>(InputTy)->getNumElements());
691 ExtendInput = true;
692 LLVM_DEBUG(dbgs() << "masked scatters: Small input type, will extend:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: Small input type, will extend:\n"
<< *Input << "\n"; } } while (false)
693 << *Input << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: Small input type, will extend:\n"
<< *Input << "\n"; } } while (false)
;
694 }
695 if (InputTy->getPrimitiveSizeInBits() != 128) {
696 LLVM_DEBUG(dbgs() << "masked scatters: cannot create scatters for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: cannot create scatters for "
"non-standard input types. Expanding.\n"; } } while (false)
697 "non-standard input types. Expanding.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked scatters: cannot create scatters for "
"non-standard input types. Expanding.\n"; } } while (false)
;
698 return nullptr;
699 }
700
701 Value *Offsets;
702 int Scale;
703 Value *BasePtr = decomposePtr(
704 Ptr, Offsets, Scale, cast<FixedVectorType>(InputTy), MemoryTy, Builder);
705 if (!BasePtr)
706 return nullptr;
707
708 if (ExtendInput)
709 Input = Builder.CreateZExt(Input, InputTy);
710 if (!match(Mask, m_One()))
711 return Builder.CreateIntrinsic(
712 Intrinsic::arm_mve_vstr_scatter_offset_predicated,
713 {BasePtr->getType(), Offsets->getType(), Input->getType(),
714 Mask->getType()},
715 {BasePtr, Offsets, Input,
716 Builder.getInt32(MemoryTy->getScalarSizeInBits()),
717 Builder.getInt32(Scale), Mask});
718 else
719 return Builder.CreateIntrinsic(
720 Intrinsic::arm_mve_vstr_scatter_offset,
721 {BasePtr->getType(), Offsets->getType(), Input->getType()},
722 {BasePtr, Offsets, Input,
723 Builder.getInt32(MemoryTy->getScalarSizeInBits()),
724 Builder.getInt32(Scale)});
725}
726
727Instruction *MVEGatherScatterLowering::tryCreateIncrementingGatScat(
728 IntrinsicInst *I, Value *Ptr, IRBuilder<> &Builder) {
729 FixedVectorType *Ty;
730 if (I->getIntrinsicID() == Intrinsic::masked_gather)
731 Ty = cast<FixedVectorType>(I->getType());
732 else
733 Ty = cast<FixedVectorType>(I->getArgOperand(0)->getType());
734
735 // Incrementing gathers only exist for v4i32
736 if (Ty->getNumElements() != 4 || Ty->getScalarSizeInBits() != 32)
737 return nullptr;
738 // Incrementing gathers are not beneficial outside of a loop
739 Loop *L = LI->getLoopFor(I->getParent());
740 if (L == nullptr)
741 return nullptr;
742
743 // Decompose the GEP into Base and Offsets
744 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
745 Value *Offsets;
746 Value *BasePtr = decomposeGEP(Offsets, Ty, GEP, Builder);
747 if (!BasePtr)
748 return nullptr;
749
750 LLVM_DEBUG(dbgs() << "masked gathers/scatters: trying to build incrementing "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: trying to build incrementing "
"wb gather/scatter\n"; } } while (false)
751 "wb gather/scatter\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: trying to build incrementing "
"wb gather/scatter\n"; } } while (false)
;
752
753 // The gep was in charge of making sure the offsets are scaled correctly
754 // - calculate that factor so it can be applied by hand
755 int TypeScale =
756 computeScale(DL->getTypeSizeInBits(GEP->getOperand(0)->getType()),
757 DL->getTypeSizeInBits(GEP->getType()) /
758 cast<FixedVectorType>(GEP->getType())->getNumElements());
759 if (TypeScale == -1)
760 return nullptr;
761
762 if (GEP->hasOneUse()) {
763 // Only in this case do we want to build a wb gather, because the wb will
764 // change the phi which does affect other users of the gep (which will still
765 // be using the phi in the old way)
766 if (auto *Load = tryCreateIncrementingWBGatScat(I, BasePtr, Offsets,
767 TypeScale, Builder))
768 return Load;
769 }
770
771 LLVM_DEBUG(dbgs() << "masked gathers/scatters: trying to build incrementing "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: trying to build incrementing "
"non-wb gather/scatter\n"; } } while (false)
772 "non-wb gather/scatter\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: trying to build incrementing "
"non-wb gather/scatter\n"; } } while (false)
;
773
774 std::pair<Value *, int64_t> Add = getVarAndConst(Offsets, TypeScale);
775 if (Add.first == nullptr)
776 return nullptr;
777 Value *OffsetsIncoming = Add.first;
778 int64_t Immediate = Add.second;
779
780 // Make sure the offsets are scaled correctly
781 Instruction *ScaledOffsets = BinaryOperator::Create(
782 Instruction::Shl, OffsetsIncoming,
783 Builder.CreateVectorSplat(Ty->getNumElements(), Builder.getInt32(TypeScale)),
784 "ScaledIndex", I);
785 // Add the base to the offsets
786 OffsetsIncoming = BinaryOperator::Create(
787 Instruction::Add, ScaledOffsets,
788 Builder.CreateVectorSplat(
789 Ty->getNumElements(),
790 Builder.CreatePtrToInt(
791 BasePtr,
792 cast<VectorType>(ScaledOffsets->getType())->getElementType())),
793 "StartIndex", I);
794
795 if (I->getIntrinsicID() == Intrinsic::masked_gather)
796 return tryCreateMaskedGatherBase(I, OffsetsIncoming, Builder, Immediate);
797 else
798 return tryCreateMaskedScatterBase(I, OffsetsIncoming, Builder, Immediate);
799}
800
801Instruction *MVEGatherScatterLowering::tryCreateIncrementingWBGatScat(
802 IntrinsicInst *I, Value *BasePtr, Value *Offsets, unsigned TypeScale,
803 IRBuilder<> &Builder) {
804 // Check whether this gather's offset is incremented by a constant - if so,
805 // and the load is of the right type, we can merge this into a QI gather
806 Loop *L = LI->getLoopFor(I->getParent());
807 // Offsets that are worth merging into this instruction will be incremented
808 // by a constant, thus we're looking for an add of a phi and a constant
809 PHINode *Phi = dyn_cast<PHINode>(Offsets);
810 if (Phi == nullptr || Phi->getNumIncomingValues() != 2 ||
811 Phi->getParent() != L->getHeader() || Phi->getNumUses() != 2)
812 // No phi means no IV to write back to; if there is a phi, we expect it
813 // to have exactly two incoming values; the only phis we are interested in
814 // will be loop IV's and have exactly two uses, one in their increment and
815 // one in the gather's gep
816 return nullptr;
817
818 unsigned IncrementIndex =
819 Phi->getIncomingBlock(0) == L->getLoopLatch() ? 0 : 1;
820 // Look through the phi to the phi increment
821 Offsets = Phi->getIncomingValue(IncrementIndex);
822
823 std::pair<Value *, int64_t> Add = getVarAndConst(Offsets, TypeScale);
824 if (Add.first == nullptr)
825 return nullptr;
826 Value *OffsetsIncoming = Add.first;
827 int64_t Immediate = Add.second;
828 if (OffsetsIncoming != Phi)
829 // Then the increment we are looking at is not an increment of the
830 // induction variable, and we don't want to do a writeback
831 return nullptr;
832
833 Builder.SetInsertPoint(&Phi->getIncomingBlock(1 - IncrementIndex)->back());
834 unsigned NumElems =
835 cast<FixedVectorType>(OffsetsIncoming->getType())->getNumElements();
836
837 // Make sure the offsets are scaled correctly
838 Instruction *ScaledOffsets = BinaryOperator::Create(
839 Instruction::Shl, Phi->getIncomingValue(1 - IncrementIndex),
840 Builder.CreateVectorSplat(NumElems, Builder.getInt32(TypeScale)),
841 "ScaledIndex", &Phi->getIncomingBlock(1 - IncrementIndex)->back());
842 // Add the base to the offsets
843 OffsetsIncoming = BinaryOperator::Create(
844 Instruction::Add, ScaledOffsets,
845 Builder.CreateVectorSplat(
846 NumElems,
847 Builder.CreatePtrToInt(
848 BasePtr,
849 cast<VectorType>(ScaledOffsets->getType())->getElementType())),
850 "StartIndex", &Phi->getIncomingBlock(1 - IncrementIndex)->back());
851 // The gather is pre-incrementing
852 OffsetsIncoming = BinaryOperator::Create(
853 Instruction::Sub, OffsetsIncoming,
854 Builder.CreateVectorSplat(NumElems, Builder.getInt32(Immediate)),
855 "PreIncrementStartIndex",
856 &Phi->getIncomingBlock(1 - IncrementIndex)->back());
857 Phi->setIncomingValue(1 - IncrementIndex, OffsetsIncoming);
858
859 Builder.SetInsertPoint(I);
860
861 Instruction *EndResult;
862 Instruction *NewInduction;
863 if (I->getIntrinsicID() == Intrinsic::masked_gather) {
864 // Build the incrementing gather
865 Value *Load = tryCreateMaskedGatherBaseWB(I, Phi, Builder, Immediate);
866 // One value to be handed to whoever uses the gather, one is the loop
867 // increment
868 EndResult = ExtractValueInst::Create(Load, 0, "Gather");
869 NewInduction = ExtractValueInst::Create(Load, 1, "GatherIncrement");
870 Builder.Insert(EndResult);
871 Builder.Insert(NewInduction);
872 } else {
873 // Build the incrementing scatter
874 EndResult = NewInduction =
875 tryCreateMaskedScatterBaseWB(I, Phi, Builder, Immediate);
876 }
877 Instruction *AddInst = cast<Instruction>(Offsets);
878 AddInst->replaceAllUsesWith(NewInduction);
879 AddInst->eraseFromParent();
880 Phi->setIncomingValue(IncrementIndex, NewInduction);
881
882 return EndResult;
883}
884
885void MVEGatherScatterLowering::pushOutAdd(PHINode *&Phi,
886 Value *OffsSecondOperand,
887 unsigned StartIndex) {
888 LLVM_DEBUG(dbgs() << "masked gathers/scatters: optimising add instruction\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: optimising add instruction\n"
; } } while (false)
;
889 Instruction *InsertionPoint =
890 &cast<Instruction>(Phi->getIncomingBlock(StartIndex)->back());
891 // Initialize the phi with a vector that contains a sum of the constants
892 Instruction *NewIndex = BinaryOperator::Create(
893 Instruction::Add, Phi->getIncomingValue(StartIndex), OffsSecondOperand,
894 "PushedOutAdd", InsertionPoint);
895 unsigned IncrementIndex = StartIndex == 0 ? 1 : 0;
896
897 // Order such that start index comes first (this reduces mov's)
898 Phi->addIncoming(NewIndex, Phi->getIncomingBlock(StartIndex));
899 Phi->addIncoming(Phi->getIncomingValue(IncrementIndex),
900 Phi->getIncomingBlock(IncrementIndex));
901 Phi->removeIncomingValue(IncrementIndex);
902 Phi->removeIncomingValue(StartIndex);
903}
904
905void MVEGatherScatterLowering::pushOutMulShl(unsigned Opcode, PHINode *&Phi,
906 Value *IncrementPerRound,
907 Value *OffsSecondOperand,
908 unsigned LoopIncrement,
909 IRBuilder<> &Builder) {
910 LLVM_DEBUG(dbgs() << "masked gathers/scatters: optimising mul instruction\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: optimising mul instruction\n"
; } } while (false)
;
911
912 // Create a new scalar add outside of the loop and transform it to a splat
913 // by which loop variable can be incremented
914 Instruction *InsertionPoint = &cast<Instruction>(
915 Phi->getIncomingBlock(LoopIncrement == 1 ? 0 : 1)->back());
916
917 // Create a new index
918 Value *StartIndex =
919 BinaryOperator::Create((Instruction::BinaryOps)Opcode,
920 Phi->getIncomingValue(LoopIncrement == 1 ? 0 : 1),
921 OffsSecondOperand, "PushedOutMul", InsertionPoint);
922
923 Instruction *Product =
924 BinaryOperator::Create((Instruction::BinaryOps)Opcode, IncrementPerRound,
925 OffsSecondOperand, "Product", InsertionPoint);
926 // Increment NewIndex by Product instead of the multiplication
927 Instruction *NewIncrement = BinaryOperator::Create(
928 Instruction::Add, Phi, Product, "IncrementPushedOutMul",
929 cast<Instruction>(Phi->getIncomingBlock(LoopIncrement)->back())
930 .getPrevNode());
931
932 Phi->addIncoming(StartIndex,
933 Phi->getIncomingBlock(LoopIncrement == 1 ? 0 : 1));
934 Phi->addIncoming(NewIncrement, Phi->getIncomingBlock(LoopIncrement));
935 Phi->removeIncomingValue((unsigned)0);
936 Phi->removeIncomingValue((unsigned)0);
937}
938
939// Check whether all usages of this instruction are as offsets of
940// gathers/scatters or simple arithmetics only used by gathers/scatters
941static bool hasAllGatScatUsers(Instruction *I, const DataLayout &DL) {
942 if (I->hasNUses(0)) {
943 return false;
944 }
945 bool Gatscat = true;
946 for (User *U : I->users()) {
947 if (!isa<Instruction>(U))
948 return false;
949 if (isa<GetElementPtrInst>(U) ||
950 isGatherScatter(dyn_cast<IntrinsicInst>(U))) {
951 return Gatscat;
952 } else {
953 unsigned OpCode = cast<Instruction>(U)->getOpcode();
954 if ((OpCode == Instruction::Add || OpCode == Instruction::Mul ||
955 OpCode == Instruction::Shl ||
956 isAddLikeOr(cast<Instruction>(U), DL)) &&
957 hasAllGatScatUsers(cast<Instruction>(U), DL)) {
958 continue;
959 }
960 return false;
961 }
962 }
963 return Gatscat;
964}
965
966bool MVEGatherScatterLowering::optimiseOffsets(Value *Offsets, BasicBlock *BB,
967 LoopInfo *LI) {
968 LLVM_DEBUG(dbgs() << "masked gathers/scatters: trying to optimize: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: trying to optimize: "
<< *Offsets << "\n"; } } while (false)
969 << *Offsets << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: trying to optimize: "
<< *Offsets << "\n"; } } while (false)
;
970 // Optimise the addresses of gathers/scatters by moving invariant
971 // calculations out of the loop
972 if (!isa<Instruction>(Offsets))
973 return false;
974 Instruction *Offs = cast<Instruction>(Offsets);
975 if (Offs->getOpcode() != Instruction::Add && !isAddLikeOr(Offs, *DL) &&
976 Offs->getOpcode() != Instruction::Mul &&
977 Offs->getOpcode() != Instruction::Shl)
978 return false;
979 Loop *L = LI->getLoopFor(BB);
980 if (L == nullptr)
981 return false;
982 if (!Offs->hasOneUse()) {
983 if (!hasAllGatScatUsers(Offs, *DL))
984 return false;
985 }
986
987 // Find out which, if any, operand of the instruction
988 // is a phi node
989 PHINode *Phi;
990 int OffsSecondOp;
991 if (isa<PHINode>(Offs->getOperand(0))) {
992 Phi = cast<PHINode>(Offs->getOperand(0));
993 OffsSecondOp = 1;
994 } else if (isa<PHINode>(Offs->getOperand(1))) {
995 Phi = cast<PHINode>(Offs->getOperand(1));
996 OffsSecondOp = 0;
997 } else {
998 bool Changed = false;
999 if (isa<Instruction>(Offs->getOperand(0)) &&
1000 L->contains(cast<Instruction>(Offs->getOperand(0))))
1001 Changed |= optimiseOffsets(Offs->getOperand(0), BB, LI);
1002 if (isa<Instruction>(Offs->getOperand(1)) &&
1003 L->contains(cast<Instruction>(Offs->getOperand(1))))
1004 Changed |= optimiseOffsets(Offs->getOperand(1), BB, LI);
1005 if (!Changed)
1006 return false;
1007 if (isa<PHINode>(Offs->getOperand(0))) {
1008 Phi = cast<PHINode>(Offs->getOperand(0));
1009 OffsSecondOp = 1;
1010 } else if (isa<PHINode>(Offs->getOperand(1))) {
1011 Phi = cast<PHINode>(Offs->getOperand(1));
1012 OffsSecondOp = 0;
1013 } else {
1014 return false;
1015 }
1016 }
1017 // A phi node we want to perform this function on should be from the
1018 // loop header.
1019 if (Phi->getParent() != L->getHeader())
1020 return false;
1021
1022 // We're looking for a simple add recurrence.
1023 BinaryOperator *IncInstruction;
1024 Value *Start, *IncrementPerRound;
1025 if (!matchSimpleRecurrence(Phi, IncInstruction, Start, IncrementPerRound) ||
1026 IncInstruction->getOpcode() != Instruction::Add)
1027 return false;
1028
1029 int IncrementingBlock = Phi->getIncomingValue(0) == IncInstruction ? 0 : 1;
1030
1031 // Get the value that is added to/multiplied with the phi
1032 Value *OffsSecondOperand = Offs->getOperand(OffsSecondOp);
1033
1034 if (IncrementPerRound->getType() != OffsSecondOperand->getType() ||
1035 !L->isLoopInvariant(OffsSecondOperand))
1036 // Something has gone wrong, abort
1037 return false;
1038
1039 // Only proceed if the increment per round is a constant or an instruction
1040 // which does not originate from within the loop
1041 if (!isa<Constant>(IncrementPerRound) &&
1042 !(isa<Instruction>(IncrementPerRound) &&
1043 !L->contains(cast<Instruction>(IncrementPerRound))))
1044 return false;
1045
1046 // If the phi is not used by anything else, we can just adapt it when
1047 // replacing the instruction; if it is, we'll have to duplicate it
1048 PHINode *NewPhi;
1049 if (Phi->getNumUses() == 2) {
1050 // No other users -> reuse existing phi (One user is the instruction
1051 // we're looking at, the other is the phi increment)
1052 if (IncInstruction->getNumUses() != 1) {
1053 // If the incrementing instruction does have more users than
1054 // our phi, we need to copy it
1055 IncInstruction = BinaryOperator::Create(
1056 Instruction::BinaryOps(IncInstruction->getOpcode()), Phi,
1057 IncrementPerRound, "LoopIncrement", IncInstruction);
1058 Phi->setIncomingValue(IncrementingBlock, IncInstruction);
1059 }
1060 NewPhi = Phi;
1061 } else {
1062 // There are other users -> create a new phi
1063 NewPhi = PHINode::Create(Phi->getType(), 2, "NewPhi", Phi);
1064 // Copy the incoming values of the old phi
1065 NewPhi->addIncoming(Phi->getIncomingValue(IncrementingBlock == 1 ? 0 : 1),
1066 Phi->getIncomingBlock(IncrementingBlock == 1 ? 0 : 1));
1067 IncInstruction = BinaryOperator::Create(
1068 Instruction::BinaryOps(IncInstruction->getOpcode()), NewPhi,
1069 IncrementPerRound, "LoopIncrement", IncInstruction);
1070 NewPhi->addIncoming(IncInstruction,
1071 Phi->getIncomingBlock(IncrementingBlock));
1072 IncrementingBlock = 1;
1073 }
1074
1075 IRBuilder<> Builder(BB->getContext());
1076 Builder.SetInsertPoint(Phi);
1077 Builder.SetCurrentDebugLocation(Offs->getDebugLoc());
1078
1079 switch (Offs->getOpcode()) {
1080 case Instruction::Add:
1081 case Instruction::Or:
1082 pushOutAdd(NewPhi, OffsSecondOperand, IncrementingBlock == 1 ? 0 : 1);
1083 break;
1084 case Instruction::Mul:
1085 case Instruction::Shl:
1086 pushOutMulShl(Offs->getOpcode(), NewPhi, IncrementPerRound,
1087 OffsSecondOperand, IncrementingBlock, Builder);
1088 break;
1089 default:
1090 return false;
1091 }
1092 LLVM_DEBUG(dbgs() << "masked gathers/scatters: simplified loop variable "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: simplified loop variable "
<< "add/mul\n"; } } while (false)
1093 << "add/mul\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: simplified loop variable "
<< "add/mul\n"; } } while (false)
;
1094
1095 // The instruction has now been "absorbed" into the phi value
1096 Offs->replaceAllUsesWith(NewPhi);
1097 if (Offs->hasNUses(0))
1098 Offs->eraseFromParent();
1099 // Clean up the old increment in case it's unused because we built a new
1100 // one
1101 if (IncInstruction->hasNUses(0))
1102 IncInstruction->eraseFromParent();
1103
1104 return true;
1105}
1106
1107static Value *CheckAndCreateOffsetAdd(Value *X, unsigned ScaleX, Value *Y,
1108 unsigned ScaleY, IRBuilder<> &Builder) {
1109 // Splat the non-vector value to a vector of the given type - if the value is
1110 // a constant (and its value isn't too big), we can even use this opportunity
1111 // to scale it to the size of the vector elements
1112 auto FixSummands = [&Builder](FixedVectorType *&VT, Value *&NonVectorVal) {
1113 ConstantInt *Const;
1114 if ((Const = dyn_cast<ConstantInt>(NonVectorVal)) &&
1115 VT->getElementType() != NonVectorVal->getType()) {
1116 unsigned TargetElemSize = VT->getElementType()->getPrimitiveSizeInBits();
1117 uint64_t N = Const->getZExtValue();
1118 if (N < (unsigned)(1 << (TargetElemSize - 1))) {
1119 NonVectorVal = Builder.CreateVectorSplat(
1120 VT->getNumElements(), Builder.getIntN(TargetElemSize, N));
1121 return;
1122 }
1123 }
1124 NonVectorVal =
1125 Builder.CreateVectorSplat(VT->getNumElements(), NonVectorVal);
1126 };
1127
1128 FixedVectorType *XElType = dyn_cast<FixedVectorType>(X->getType());
14
Assuming the object is a 'CastReturnType'
1129 FixedVectorType *YElType = dyn_cast<FixedVectorType>(Y->getType());
15
Assuming the object is not a 'CastReturnType'
1130 // If one of X, Y is not a vector, we have to splat it in order
1131 // to add the two of them.
1132 if (XElType
15.1
'XElType' is non-null
15.1
'XElType' is non-null
&& !YElType
15.2
'YElType' is null
15.2
'YElType' is null
) {
16
Taking true branch
1133 FixSummands(XElType, Y);
1134 YElType = cast<FixedVectorType>(Y->getType());
17
The object is a 'CastReturnType'
1135 } else if (YElType && !XElType) {
1136 FixSummands(YElType, X);
1137 XElType = cast<FixedVectorType>(X->getType());
1138 }
1139 assert(XElType && YElType && "Unknown vector types")(static_cast <bool> (XElType && YElType &&
"Unknown vector types") ? void (0) : __assert_fail ("XElType && YElType && \"Unknown vector types\""
, "llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp", 1139, __extension__
__PRETTY_FUNCTION__))
;
18
'?' condition is true
1140 // Check that the summands are of compatible types
1141 if (XElType != YElType) {
19
Assuming 'XElType' is equal to 'YElType'
20
Taking false branch
1142 LLVM_DEBUG(dbgs() << "masked gathers/scatters: incompatible gep offsets\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "masked gathers/scatters: incompatible gep offsets\n"
; } } while (false)
;
1143 return nullptr;
1144 }
1145
1146 if (XElType->getElementType()->getScalarSizeInBits() != 32) {
21
Assuming the condition is true
22
Taking true branch
1147 // Check that by adding the vectors we do not accidentally
1148 // create an overflow
1149 Constant *ConstX = dyn_cast<Constant>(X);
23
'X' is a 'Constant'
1150 Constant *ConstY = dyn_cast<Constant>(Y);
24
Assuming 'Y' is a 'CastReturnType'
1151 if (!ConstX
24.1
'ConstX' is non-null
24.1
'ConstX' is non-null
|| !ConstY
24.2
'ConstY' is non-null
24.2
'ConstY' is non-null
)
25
Taking false branch
1152 return nullptr;
1153 unsigned TargetElemSize = 128 / XElType->getNumElements();
1154 for (unsigned i = 0; i < XElType->getNumElements(); i++) {
26
Loop condition is true. Entering loop body
31
Assuming the condition is false
32
Loop condition is false. Execution continues on line 1167
1155 ConstantInt *ConstXEl =
1156 dyn_cast<ConstantInt>(ConstX->getAggregateElement(i));
27
Assuming the object is a 'CastReturnType'
1157 ConstantInt *ConstYEl =
1158 dyn_cast<ConstantInt>(ConstY->getAggregateElement(i));
28
Assuming the object is a 'CastReturnType'
1159 if (!ConstXEl
28.1
'ConstXEl' is non-null
28.1
'ConstXEl' is non-null
|| !ConstYEl
28.2
'ConstYEl' is non-null
28.2
'ConstYEl' is non-null
||
30
Taking false branch
1160 ConstXEl->getZExtValue() * ScaleX +
29
Assuming the condition is false
1161 ConstYEl->getZExtValue() * ScaleY >=
1162 (unsigned)(1 << (TargetElemSize - 1)))
1163 return nullptr;
1164 }
1165 }
1166
1167 Value *XScale = Builder.CreateVectorSplat(
1168 XElType->getNumElements(),
1169 Builder.getIntN(XElType->getScalarSizeInBits(), ScaleX));
1170 Value *YScale = Builder.CreateVectorSplat(
1171 YElType->getNumElements(),
1172 Builder.getIntN(YElType->getScalarSizeInBits(), ScaleY));
1173 Value *Add = Builder.CreateAdd(Builder.CreateMul(X, XScale),
33
Calling 'IRBuilderBase::CreateAdd'
36
Returning from 'IRBuilderBase::CreateAdd'
1174 Builder.CreateMul(Y, YScale));
1175
1176 if (checkOffsetSize(Add, XElType->getNumElements()))
37
Calling 'checkOffsetSize'
1177 return Add;
1178 else
1179 return nullptr;
1180}
1181
1182Value *MVEGatherScatterLowering::foldGEP(GetElementPtrInst *GEP,
1183 Value *&Offsets, unsigned &Scale,
1184 IRBuilder<> &Builder) {
1185 Value *GEPPtr = GEP->getPointerOperand();
1186 Offsets = GEP->getOperand(1);
1187 Scale = DL->getTypeAllocSize(GEP->getSourceElementType());
1188 // We only merge geps with constant offsets, because only for those
1189 // we can make sure that we do not cause an overflow
1190 if (GEP->getNumIndices() != 1 || !isa<Constant>(Offsets))
1
Assuming the condition is false
2
Assuming 'Offsets' is a 'class llvm::Constant &'
3
Taking false branch
7
Assuming the condition is false
8
Assuming 'Offsets' is a 'class llvm::Constant &'
9
Taking false branch
1191 return nullptr;
1192 if (GetElementPtrInst *BaseGEP = dyn_cast<GetElementPtrInst>(GEPPtr)) {
4
Assuming 'BaseGEP' is non-null
5
Taking true branch
10
Assuming 'BaseGEP' is non-null
11
Taking true branch
1193 // Merge the two geps into one
1194 Value *BaseBasePtr = foldGEP(BaseGEP, Offsets, Scale, Builder);
6
Calling 'MVEGatherScatterLowering::foldGEP'
1195 if (!BaseBasePtr
11.1
'BaseBasePtr' is non-null
11.1
'BaseBasePtr' is non-null
)
12
Taking false branch
1196 return nullptr;
1197 Offsets = CheckAndCreateOffsetAdd(
13
Calling 'CheckAndCreateOffsetAdd'
1198 Offsets, Scale, GEP->getOperand(1),
1199 DL->getTypeAllocSize(GEP->getSourceElementType()), Builder);
1200 if (Offsets == nullptr)
1201 return nullptr;
1202 Scale = 1; // Scale is always an i8 at this point.
1203 return BaseBasePtr;
1204 }
1205 return GEPPtr;
1206}
1207
1208bool MVEGatherScatterLowering::optimiseAddress(Value *Address, BasicBlock *BB,
1209 LoopInfo *LI) {
1210 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Address);
1211 if (!GEP)
1212 return false;
1213 bool Changed = false;
1214 if (GEP->hasOneUse() && isa<GetElementPtrInst>(GEP->getPointerOperand())) {
1215 IRBuilder<> Builder(GEP->getContext());
1216 Builder.SetInsertPoint(GEP);
1217 Builder.SetCurrentDebugLocation(GEP->getDebugLoc());
1218 Value *Offsets;
1219 unsigned Scale;
1220 Value *Base = foldGEP(GEP, Offsets, Scale, Builder);
1221 // We only want to merge the geps if there is a real chance that they can be
1222 // used by an MVE gather; thus the offset has to have the correct size
1223 // (always i32 if it is not of vector type) and the base has to be a
1224 // pointer.
1225 if (Offsets && Base && Base != GEP) {
1226 assert(Scale == 1 && "Expected to fold GEP to a scale of 1")(static_cast <bool> (Scale == 1 && "Expected to fold GEP to a scale of 1"
) ? void (0) : __assert_fail ("Scale == 1 && \"Expected to fold GEP to a scale of 1\""
, "llvm/lib/Target/ARM/MVEGatherScatterLowering.cpp", 1226, __extension__
__PRETTY_FUNCTION__))
;
1227 Type *BaseTy = Builder.getInt8PtrTy();
1228 if (auto *VecTy = dyn_cast<FixedVectorType>(Base->getType()))
1229 BaseTy = FixedVectorType::get(BaseTy, VecTy);
1230 GetElementPtrInst *NewAddress = GetElementPtrInst::Create(
1231 Builder.getInt8Ty(), Builder.CreateBitCast(Base, BaseTy), Offsets,
1232 "gep.merged", GEP);
1233 LLVM_DEBUG(dbgs() << "Folded GEP: " << *GEPdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "Folded GEP: "
<< *GEP << "\n new : " << *NewAddress
<< "\n"; } } while (false)
1234 << "\n new : " << *NewAddress << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-mve-gather-scatter-lowering")) { dbgs() << "Folded GEP: "
<< *GEP << "\n new : " << *NewAddress
<< "\n"; } } while (false)
;
1235 GEP->replaceAllUsesWith(
1236 Builder.CreateBitCast(NewAddress, GEP->getType()));
1237 GEP = NewAddress;
1238 Changed = true;
1239 }
1240 }
1241 Changed |= optimiseOffsets(GEP->getOperand(1), GEP->getParent(), LI);
1242 return Changed;
1243}
1244
1245bool MVEGatherScatterLowering::runOnFunction(Function &F) {
1246 if (!EnableMaskedGatherScatters)
1247 return false;
1248 auto &TPC = getAnalysis<TargetPassConfig>();
1249 auto &TM = TPC.getTM<TargetMachine>();
1250 auto *ST = &TM.getSubtarget<ARMSubtarget>(F);
1251 if (!ST->hasMVEIntegerOps())
1252 return false;
1253 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1254 DL = &F.getParent()->getDataLayout();
1255 SmallVector<IntrinsicInst *, 4> Gathers;
1256 SmallVector<IntrinsicInst *, 4> Scatters;
1257
1258 bool Changed = false;
1259
1260 for (BasicBlock &BB : F) {
1261 Changed |= SimplifyInstructionsInBlock(&BB);
1262
1263 for (Instruction &I : BB) {
1264 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
1265 if (II && II->getIntrinsicID() == Intrinsic::masked_gather &&
1266 isa<FixedVectorType>(II->getType())) {
1267 Gathers.push_back(II);
1268 Changed |= optimiseAddress(II->getArgOperand(0), II->getParent(), LI);
1269 } else if (II && II->getIntrinsicID() == Intrinsic::masked_scatter &&
1270 isa<FixedVectorType>(II->getArgOperand(0)->getType())) {
1271 Scatters.push_back(II);
1272 Changed |= optimiseAddress(II->getArgOperand(1), II->getParent(), LI);
1273 }
1274 }
1275 }
1276 for (unsigned i = 0; i < Gathers.size(); i++) {
1277 IntrinsicInst *I = Gathers[i];
1278 Instruction *L = lowerGather(I);
1279 if (L == nullptr)
1280 continue;
1281
1282 // Get rid of any now dead instructions
1283 SimplifyInstructionsInBlock(L->getParent());
1284 Changed = true;
1285 }
1286
1287 for (unsigned i = 0; i < Scatters.size(); i++) {
1288 IntrinsicInst *I = Scatters[i];
1289 Instruction *S = lowerScatter(I);
1290 if (S == nullptr)
1291 continue;
1292
1293 // Get rid of any now dead instructions
1294 SimplifyInstructionsInBlock(S->getParent());
1295 Changed = true;
1296 }
1297 return Changed;
1298}

/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantFolder.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DebugLoc.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/FPEnv.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/GlobalVariable.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/LLVMContext.h"
38#include "llvm/IR/Module.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/IR/Type.h"
41#include "llvm/IR/Value.h"
42#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/CBindingWrapping.h"
45#include "llvm/Support/Casting.h"
46#include <cassert>
47#include <cstdint>
48#include <functional>
49#include <utility>
50
51namespace llvm {
52
53class APInt;
54class Use;
55
56/// This provides the default implementation of the IRBuilder
57/// 'InsertHelper' method that is called whenever an instruction is created by
58/// IRBuilder and needs to be inserted.
59///
60/// By default, this inserts the instruction at the insertion point.
61class IRBuilderDefaultInserter {
62public:
63 virtual ~IRBuilderDefaultInserter();
64
65 virtual void InsertHelper(Instruction *I, const Twine &Name,
66 BasicBlock *BB,
67 BasicBlock::iterator InsertPt) const {
68 if (BB) BB->getInstList().insert(InsertPt, I);
69 I->setName(Name);
70 }
71};
72
73/// Provides an 'InsertHelper' that calls a user-provided callback after
74/// performing the default insertion.
75class IRBuilderCallbackInserter : public IRBuilderDefaultInserter {
76 std::function<void(Instruction *)> Callback;
77
78public:
79 ~IRBuilderCallbackInserter() override;
80
81 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
82 : Callback(std::move(Callback)) {}
83
84 void InsertHelper(Instruction *I, const Twine &Name,
85 BasicBlock *BB,
86 BasicBlock::iterator InsertPt) const override {
87 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
88 Callback(I);
89 }
90};
91
92/// Common base class shared among various IRBuilders.
93class IRBuilderBase {
94 /// Pairs of (metadata kind, MDNode *) that should be added to all newly
95 /// created instructions, like !dbg metadata.
96 SmallVector<std::pair<unsigned, MDNode *>, 2> MetadataToCopy;
97
98 /// Add or update the an entry (Kind, MD) to MetadataToCopy, if \p MD is not
99 /// null. If \p MD is null, remove the entry with \p Kind.
100 void AddOrRemoveMetadataToCopy(unsigned Kind, MDNode *MD) {
101 if (!MD) {
102 erase_if(MetadataToCopy, [Kind](const std::pair<unsigned, MDNode *> &KV) {
103 return KV.first == Kind;
104 });
105 return;
106 }
107
108 for (auto &KV : MetadataToCopy)
109 if (KV.first == Kind) {
110 KV.second = MD;
111 return;
112 }
113
114 MetadataToCopy.emplace_back(Kind, MD);
115 }
116
117protected:
118 BasicBlock *BB;
119 BasicBlock::iterator InsertPt;
120 LLVMContext &Context;
121 const IRBuilderFolder &Folder;
122 const IRBuilderDefaultInserter &Inserter;
123
124 MDNode *DefaultFPMathTag;
125 FastMathFlags FMF;
126
127 bool IsFPConstrained = false;
128 fp::ExceptionBehavior DefaultConstrainedExcept = fp::ebStrict;
129 RoundingMode DefaultConstrainedRounding = RoundingMode::Dynamic;
130
131 ArrayRef<OperandBundleDef> DefaultOperandBundles;
132
133public:
134 IRBuilderBase(LLVMContext &context, const IRBuilderFolder &Folder,
135 const IRBuilderDefaultInserter &Inserter, MDNode *FPMathTag,
136 ArrayRef<OperandBundleDef> OpBundles)
137 : Context(context), Folder(Folder), Inserter(Inserter),
138 DefaultFPMathTag(FPMathTag), DefaultOperandBundles(OpBundles) {
139 ClearInsertionPoint();
140 }
141
142 /// Insert and return the specified instruction.
143 template<typename InstTy>
144 InstTy *Insert(InstTy *I, const Twine &Name = "") const {
145 Inserter.InsertHelper(I, Name, BB, InsertPt);
146 AddMetadataToInst(I);
147 return I;
148 }
149
150 /// No-op overload to handle constants.
151 Constant *Insert(Constant *C, const Twine& = "") const {
152 return C;
153 }
154
155 Value *Insert(Value *V, const Twine &Name = "") const {
156 if (Instruction *I = dyn_cast<Instruction>(V))
157 return Insert(I, Name);
158 assert(isa<Constant>(V))(static_cast <bool> (isa<Constant>(V)) ? void (0)
: __assert_fail ("isa<Constant>(V)", "llvm/include/llvm/IR/IRBuilder.h"
, 158, __extension__ __PRETTY_FUNCTION__))
;
159 return V;
160 }
161
162 //===--------------------------------------------------------------------===//
163 // Builder configuration methods
164 //===--------------------------------------------------------------------===//
165
166 /// Clear the insertion point: created instructions will not be
167 /// inserted into a block.
168 void ClearInsertionPoint() {
169 BB = nullptr;
170 InsertPt = BasicBlock::iterator();
171 }
172
173 BasicBlock *GetInsertBlock() const { return BB; }
174 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
175 LLVMContext &getContext() const { return Context; }
176
177 /// This specifies that created instructions should be appended to the
178 /// end of the specified block.
179 void SetInsertPoint(BasicBlock *TheBB) {
180 BB = TheBB;
181 InsertPt = BB->end();
182 }
183
184 /// This specifies that created instructions should be inserted before
185 /// the specified instruction.
186 void SetInsertPoint(Instruction *I) {
187 BB = I->getParent();
188 InsertPt = I->getIterator();
189 assert(InsertPt != BB->end() && "Can't read debug loc from end()")(static_cast <bool> (InsertPt != BB->end() &&
"Can't read debug loc from end()") ? void (0) : __assert_fail
("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "llvm/include/llvm/IR/IRBuilder.h", 189, __extension__ __PRETTY_FUNCTION__
))
;
190 SetCurrentDebugLocation(I->getDebugLoc());
191 }
192
193 /// This specifies that created instructions should be inserted at the
194 /// specified point.
195 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
196 BB = TheBB;
197 InsertPt = IP;
198 if (IP != TheBB->end())
199 SetCurrentDebugLocation(IP->getDebugLoc());
200 }
201
202 /// This specifies that created instructions should inserted at the beginning
203 /// end of the specified function, but after already existing static alloca
204 /// instructions that are at the start.
205 void SetInsertPointPastAllocas(Function *F) {
206 BB = &F->getEntryBlock();
207 InsertPt = BB->getFirstNonPHIOrDbgOrAlloca();
208 }
209
210 /// Set location information used by debugging information.
211 void SetCurrentDebugLocation(DebugLoc L) {
212 AddOrRemoveMetadataToCopy(LLVMContext::MD_dbg, L.getAsMDNode());
213 }
214
215 /// Collect metadata with IDs \p MetadataKinds from \p Src which should be
216 /// added to all created instructions. Entries present in MedataDataToCopy but
217 /// not on \p Src will be dropped from MetadataToCopy.
218 void CollectMetadataToCopy(Instruction *Src,
219 ArrayRef<unsigned> MetadataKinds) {
220 for (unsigned K : MetadataKinds)
221 AddOrRemoveMetadataToCopy(K, Src->getMetadata(K));
222 }
223
224 /// Get location information used by debugging information.
225 DebugLoc getCurrentDebugLocation() const;
226
227 /// If this builder has a current debug location, set it on the
228 /// specified instruction.
229 void SetInstDebugLocation(Instruction *I) const;
230
231 /// Add all entries in MetadataToCopy to \p I.
232 void AddMetadataToInst(Instruction *I) const {
233 for (const auto &KV : MetadataToCopy)
234 I->setMetadata(KV.first, KV.second);
235 }
236
237 /// Get the return type of the current function that we're emitting
238 /// into.
239 Type *getCurrentFunctionReturnType() const;
240
241 /// InsertPoint - A saved insertion point.
242 class InsertPoint {
243 BasicBlock *Block = nullptr;
244 BasicBlock::iterator Point;
245
246 public:
247 /// Creates a new insertion point which doesn't point to anything.
248 InsertPoint() = default;
249
250 /// Creates a new insertion point at the given location.
251 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
252 : Block(InsertBlock), Point(InsertPoint) {}
253
254 /// Returns true if this insert point is set.
255 bool isSet() const { return (Block != nullptr); }
256
257 BasicBlock *getBlock() const { return Block; }
258 BasicBlock::iterator getPoint() const { return Point; }
259 };
260
261 /// Returns the current insert point.
262 InsertPoint saveIP() const {
263 return InsertPoint(GetInsertBlock(), GetInsertPoint());
264 }
265
266 /// Returns the current insert point, clearing it in the process.
267 InsertPoint saveAndClearIP() {
268 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
269 ClearInsertionPoint();
270 return IP;
271 }
272
273 /// Sets the current insert point to a previously-saved location.
274 void restoreIP(InsertPoint IP) {
275 if (IP.isSet())
276 SetInsertPoint(IP.getBlock(), IP.getPoint());
277 else
278 ClearInsertionPoint();
279 }
280
281 /// Get the floating point math metadata being used.
282 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
283
284 /// Get the flags to be applied to created floating point ops
285 FastMathFlags getFastMathFlags() const { return FMF; }
286
287 FastMathFlags &getFastMathFlags() { return FMF; }
288
289 /// Clear the fast-math flags.
290 void clearFastMathFlags() { FMF.clear(); }
291
292 /// Set the floating point math metadata to be used.
293 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
294
295 /// Set the fast-math flags to be used with generated fp-math operators
296 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
297
298 /// Enable/Disable use of constrained floating point math. When
299 /// enabled the CreateF<op>() calls instead create constrained
300 /// floating point intrinsic calls. Fast math flags are unaffected
301 /// by this setting.
302 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
303
304 /// Query for the use of constrained floating point math
305 bool getIsFPConstrained() { return IsFPConstrained; }
306
307 /// Set the exception handling to be used with constrained floating point
308 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
309#ifndef NDEBUG
310 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(NewExcept);
311 assert(ExceptStr && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 311, __extension__ __PRETTY_FUNCTION__
))
;
312#endif
313 DefaultConstrainedExcept = NewExcept;
314 }
315
316 /// Set the rounding mode handling to be used with constrained floating point
317 void setDefaultConstrainedRounding(RoundingMode NewRounding) {
318#ifndef NDEBUG
319 Optional<StringRef> RoundingStr = convertRoundingModeToStr(NewRounding);
320 assert(RoundingStr && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 320, __extension__ __PRETTY_FUNCTION__
))
;
321#endif
322 DefaultConstrainedRounding = NewRounding;
323 }
324
325 /// Get the exception handling used with constrained floating point
326 fp::ExceptionBehavior getDefaultConstrainedExcept() {
327 return DefaultConstrainedExcept;
328 }
329
330 /// Get the rounding mode handling used with constrained floating point
331 RoundingMode getDefaultConstrainedRounding() {
332 return DefaultConstrainedRounding;
333 }
334
335 void setConstrainedFPFunctionAttr() {
336 assert(BB && "Must have a basic block to set any function attributes!")(static_cast <bool> (BB && "Must have a basic block to set any function attributes!"
) ? void (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "llvm/include/llvm/IR/IRBuilder.h", 336, __extension__ __PRETTY_FUNCTION__
))
;
337
338 Function *F = BB->getParent();
339 if (!F->hasFnAttribute(Attribute::StrictFP)) {
340 F->addFnAttr(Attribute::StrictFP);
341 }
342 }
343
344 void setConstrainedFPCallAttr(CallBase *I) {
345 I->addFnAttr(Attribute::StrictFP);
346 }
347
348 void setDefaultOperandBundles(ArrayRef<OperandBundleDef> OpBundles) {
349 DefaultOperandBundles = OpBundles;
350 }
351
352 //===--------------------------------------------------------------------===//
353 // RAII helpers.
354 //===--------------------------------------------------------------------===//
355
356 // RAII object that stores the current insertion point and restores it
357 // when the object is destroyed. This includes the debug location.
358 class InsertPointGuard {
359 IRBuilderBase &Builder;
360 AssertingVH<BasicBlock> Block;
361 BasicBlock::iterator Point;
362 DebugLoc DbgLoc;
363
364 public:
365 InsertPointGuard(IRBuilderBase &B)
366 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
367 DbgLoc(B.getCurrentDebugLocation()) {}
368
369 InsertPointGuard(const InsertPointGuard &) = delete;
370 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
371
372 ~InsertPointGuard() {
373 Builder.restoreIP(InsertPoint(Block, Point));
374 Builder.SetCurrentDebugLocation(DbgLoc);
375 }
376 };
377
378 // RAII object that stores the current fast math settings and restores
379 // them when the object is destroyed.
380 class FastMathFlagGuard {
381 IRBuilderBase &Builder;
382 FastMathFlags FMF;
383 MDNode *FPMathTag;
384 bool IsFPConstrained;
385 fp::ExceptionBehavior DefaultConstrainedExcept;
386 RoundingMode DefaultConstrainedRounding;
387
388 public:
389 FastMathFlagGuard(IRBuilderBase &B)
390 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag),
391 IsFPConstrained(B.IsFPConstrained),
392 DefaultConstrainedExcept(B.DefaultConstrainedExcept),
393 DefaultConstrainedRounding(B.DefaultConstrainedRounding) {}
394
395 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
396 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
397
398 ~FastMathFlagGuard() {
399 Builder.FMF = FMF;
400 Builder.DefaultFPMathTag = FPMathTag;
401 Builder.IsFPConstrained = IsFPConstrained;
402 Builder.DefaultConstrainedExcept = DefaultConstrainedExcept;
403 Builder.DefaultConstrainedRounding = DefaultConstrainedRounding;
404 }
405 };
406
407 // RAII object that stores the current default operand bundles and restores
408 // them when the object is destroyed.
409 class OperandBundlesGuard {
410 IRBuilderBase &Builder;
411 ArrayRef<OperandBundleDef> DefaultOperandBundles;
412
413 public:
414 OperandBundlesGuard(IRBuilderBase &B)
415 : Builder(B), DefaultOperandBundles(B.DefaultOperandBundles) {}
416
417 OperandBundlesGuard(const OperandBundlesGuard &) = delete;
418 OperandBundlesGuard &operator=(const OperandBundlesGuard &) = delete;
419
420 ~OperandBundlesGuard() {
421 Builder.DefaultOperandBundles = DefaultOperandBundles;
422 }
423 };
424
425
426 //===--------------------------------------------------------------------===//
427 // Miscellaneous creation methods.
428 //===--------------------------------------------------------------------===//
429
430 /// Make a new global variable with initializer type i8*
431 ///
432 /// Make a new global variable with an initializer that has array of i8 type
433 /// filled in with the null terminated string value specified. The new global
434 /// variable will be marked mergable with any others of the same contents. If
435 /// Name is specified, it is the name of the global variable created.
436 ///
437 /// If no module is given via \p M, it is take from the insertion point basic
438 /// block.
439 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
440 unsigned AddressSpace = 0,
441 Module *M = nullptr);
442
443 /// Get a constant value representing either true or false.
444 ConstantInt *getInt1(bool V) {
445 return ConstantInt::get(getInt1Ty(), V);
446 }
447
448 /// Get the constant value for i1 true.
449 ConstantInt *getTrue() {
450 return ConstantInt::getTrue(Context);
451 }
452
453 /// Get the constant value for i1 false.
454 ConstantInt *getFalse() {
455 return ConstantInt::getFalse(Context);
456 }
457
458 /// Get a constant 8-bit value.
459 ConstantInt *getInt8(uint8_t C) {
460 return ConstantInt::get(getInt8Ty(), C);
461 }
462
463 /// Get a constant 16-bit value.
464 ConstantInt *getInt16(uint16_t C) {
465 return ConstantInt::get(getInt16Ty(), C);
466 }
467
468 /// Get a constant 32-bit value.
469 ConstantInt *getInt32(uint32_t C) {
470 return ConstantInt::get(getInt32Ty(), C);
471 }
472
473 /// Get a constant 64-bit value.
474 ConstantInt *getInt64(uint64_t C) {
475 return ConstantInt::get(getInt64Ty(), C);
476 }
477
478 /// Get a constant N-bit value, zero extended or truncated from
479 /// a 64-bit value.
480 ConstantInt *getIntN(unsigned N, uint64_t C) {
481 return ConstantInt::get(getIntNTy(N), C);
482 }
483
484 /// Get a constant integer value.
485 ConstantInt *getInt(const APInt &AI) {
486 return ConstantInt::get(Context, AI);
487 }
488
489 //===--------------------------------------------------------------------===//
490 // Type creation methods
491 //===--------------------------------------------------------------------===//
492
493 /// Fetch the type representing a single bit
494 IntegerType *getInt1Ty() {
495 return Type::getInt1Ty(Context);
496 }
497
498 /// Fetch the type representing an 8-bit integer.
499 IntegerType *getInt8Ty() {
500 return Type::getInt8Ty(Context);
501 }
502
503 /// Fetch the type representing a 16-bit integer.
504 IntegerType *getInt16Ty() {
505 return Type::getInt16Ty(Context);
506 }
507
508 /// Fetch the type representing a 32-bit integer.
509 IntegerType *getInt32Ty() {
510 return Type::getInt32Ty(Context);
511 }
512
513 /// Fetch the type representing a 64-bit integer.
514 IntegerType *getInt64Ty() {
515 return Type::getInt64Ty(Context);
516 }
517
518 /// Fetch the type representing a 128-bit integer.
519 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
520
521 /// Fetch the type representing an N-bit integer.
522 IntegerType *getIntNTy(unsigned N) {
523 return Type::getIntNTy(Context, N);
524 }
525
526 /// Fetch the type representing a 16-bit floating point value.
527 Type *getHalfTy() {
528 return Type::getHalfTy(Context);
529 }
530
531 /// Fetch the type representing a 16-bit brain floating point value.
532 Type *getBFloatTy() {
533 return Type::getBFloatTy(Context);
534 }
535
536 /// Fetch the type representing a 32-bit floating point value.
537 Type *getFloatTy() {
538 return Type::getFloatTy(Context);
539 }
540
541 /// Fetch the type representing a 64-bit floating point value.
542 Type *getDoubleTy() {
543 return Type::getDoubleTy(Context);
544 }
545
546 /// Fetch the type representing void.
547 Type *getVoidTy() {
548 return Type::getVoidTy(Context);
549 }
550
551 /// Fetch the type representing a pointer.
552 PointerType *getPtrTy(unsigned AddrSpace = 0) {
553 return PointerType::get(Context, AddrSpace);
554 }
555
556 /// Fetch the type representing a pointer to an 8-bit integer value.
557 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
558 return Type::getInt8PtrTy(Context, AddrSpace);
559 }
560
561 /// Fetch the type representing a pointer to an integer value.
562 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
563 return DL.getIntPtrType(Context, AddrSpace);
564 }
565
566 //===--------------------------------------------------------------------===//
567 // Intrinsic creation methods
568 //===--------------------------------------------------------------------===//
569
570 /// Create and insert a memset to the specified pointer and the
571 /// specified value.
572 ///
573 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
574 /// specified, it will be added to the instruction. Likewise with alias.scope
575 /// and noalias tags.
576 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
577 MaybeAlign Align, bool isVolatile = false,
578 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
579 MDNode *NoAliasTag = nullptr) {
580 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
581 TBAATag, ScopeTag, NoAliasTag);
582 }
583
584 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
585 bool isVolatile = false, MDNode *TBAATag = nullptr,
586 MDNode *ScopeTag = nullptr,
587 MDNode *NoAliasTag = nullptr);
588
589 CallInst *CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, Value *Val,
590 Value *Size, bool IsVolatile = false,
591 MDNode *TBAATag = nullptr,
592 MDNode *ScopeTag = nullptr,
593 MDNode *NoAliasTag = nullptr);
594
595 /// Create and insert an element unordered-atomic memset of the region of
596 /// memory starting at the given pointer to the given value.
597 ///
598 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
599 /// specified, it will be added to the instruction. Likewise with alias.scope
600 /// and noalias tags.
601 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
602 uint64_t Size, Align Alignment,
603 uint32_t ElementSize,
604 MDNode *TBAATag = nullptr,
605 MDNode *ScopeTag = nullptr,
606 MDNode *NoAliasTag = nullptr) {
607 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
608 Align(Alignment), ElementSize,
609 TBAATag, ScopeTag, NoAliasTag);
610 }
611
612 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
613 Value *Size, Align Alignment,
614 uint32_t ElementSize,
615 MDNode *TBAATag = nullptr,
616 MDNode *ScopeTag = nullptr,
617 MDNode *NoAliasTag = nullptr);
618
619 /// Create and insert a memcpy between the specified pointers.
620 ///
621 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
622 /// specified, it will be added to the instruction. Likewise with alias.scope
623 /// and noalias tags.
624 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
625 MaybeAlign SrcAlign, uint64_t Size,
626 bool isVolatile = false, MDNode *TBAATag = nullptr,
627 MDNode *TBAAStructTag = nullptr,
628 MDNode *ScopeTag = nullptr,
629 MDNode *NoAliasTag = nullptr) {
630 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
631 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
632 NoAliasTag);
633 }
634
635 CallInst *CreateMemTransferInst(
636 Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
637 MaybeAlign SrcAlign, Value *Size, bool isVolatile = false,
638 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
639 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
640
641 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
642 MaybeAlign SrcAlign, Value *Size,
643 bool isVolatile = false, MDNode *TBAATag = nullptr,
644 MDNode *TBAAStructTag = nullptr,
645 MDNode *ScopeTag = nullptr,
646 MDNode *NoAliasTag = nullptr) {
647 return CreateMemTransferInst(Intrinsic::memcpy, Dst, DstAlign, Src,
648 SrcAlign, Size, isVolatile, TBAATag,
649 TBAAStructTag, ScopeTag, NoAliasTag);
650 }
651
652 CallInst *
653 CreateMemCpyInline(Value *Dst, MaybeAlign DstAlign, Value *Src,
654 MaybeAlign SrcAlign, Value *Size, bool IsVolatile = false,
655 MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
656 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr);
657
658 /// Create and insert an element unordered-atomic memcpy between the
659 /// specified pointers.
660 ///
661 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
662 ///
663 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
664 /// specified, it will be added to the instruction. Likewise with alias.scope
665 /// and noalias tags.
666 CallInst *CreateElementUnorderedAtomicMemCpy(
667 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
668 uint32_t ElementSize, MDNode *TBAATag = nullptr,
669 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
670 MDNode *NoAliasTag = nullptr);
671
672 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
673 MaybeAlign SrcAlign, uint64_t Size,
674 bool isVolatile = false, MDNode *TBAATag = nullptr,
675 MDNode *ScopeTag = nullptr,
676 MDNode *NoAliasTag = nullptr) {
677 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
678 isVolatile, TBAATag, ScopeTag, NoAliasTag);
679 }
680
681 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
682 MaybeAlign SrcAlign, Value *Size,
683 bool isVolatile = false, MDNode *TBAATag = nullptr,
684 MDNode *ScopeTag = nullptr,
685 MDNode *NoAliasTag = nullptr);
686
687 /// \brief Create and insert an element unordered-atomic memmove between the
688 /// specified pointers.
689 ///
690 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
691 /// respectively.
692 ///
693 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
694 /// specified, it will be added to the instruction. Likewise with alias.scope
695 /// and noalias tags.
696 CallInst *CreateElementUnorderedAtomicMemMove(
697 Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
698 uint32_t ElementSize, MDNode *TBAATag = nullptr,
699 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
700 MDNode *NoAliasTag = nullptr);
701
702private:
703 CallInst *getReductionIntrinsic(Intrinsic::ID ID, Value *Src);
704
705public:
706 /// Create a sequential vector fadd reduction intrinsic of the source vector.
707 /// The first parameter is a scalar accumulator value. An unordered reduction
708 /// can be created by adding the reassoc fast-math flag to the resulting
709 /// sequential reduction.
710 CallInst *CreateFAddReduce(Value *Acc, Value *Src);
711
712 /// Create a sequential vector fmul reduction intrinsic of the source vector.
713 /// The first parameter is a scalar accumulator value. An unordered reduction
714 /// can be created by adding the reassoc fast-math flag to the resulting
715 /// sequential reduction.
716 CallInst *CreateFMulReduce(Value *Acc, Value *Src);
717
718 /// Create a vector int add reduction intrinsic of the source vector.
719 CallInst *CreateAddReduce(Value *Src);
720
721 /// Create a vector int mul reduction intrinsic of the source vector.
722 CallInst *CreateMulReduce(Value *Src);
723
724 /// Create a vector int AND reduction intrinsic of the source vector.
725 CallInst *CreateAndReduce(Value *Src);
726
727 /// Create a vector int OR reduction intrinsic of the source vector.
728 CallInst *CreateOrReduce(Value *Src);
729
730 /// Create a vector int XOR reduction intrinsic of the source vector.
731 CallInst *CreateXorReduce(Value *Src);
732
733 /// Create a vector integer max reduction intrinsic of the source
734 /// vector.
735 CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
736
737 /// Create a vector integer min reduction intrinsic of the source
738 /// vector.
739 CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
740
741 /// Create a vector float max reduction intrinsic of the source
742 /// vector.
743 CallInst *CreateFPMaxReduce(Value *Src);
744
745 /// Create a vector float min reduction intrinsic of the source
746 /// vector.
747 CallInst *CreateFPMinReduce(Value *Src);
748
749 /// Create a lifetime.start intrinsic.
750 ///
751 /// If the pointer isn't i8* it will be converted.
752 CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
753
754 /// Create a lifetime.end intrinsic.
755 ///
756 /// If the pointer isn't i8* it will be converted.
757 CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
758
759 /// Create a call to invariant.start intrinsic.
760 ///
761 /// If the pointer isn't i8* it will be converted.
762 CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
763
764 /// Create a call to llvm.threadlocal.address intrinsic.
765 CallInst *CreateThreadLocalAddress(Value *Ptr);
766
767 /// Create a call to Masked Load intrinsic
768 CallInst *CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask,
769 Value *PassThru = nullptr, const Twine &Name = "");
770
771 /// Create a call to Masked Store intrinsic
772 CallInst *CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment,
773 Value *Mask);
774
775 /// Create a call to Masked Gather intrinsic
776 CallInst *CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment,
777 Value *Mask = nullptr, Value *PassThru = nullptr,
778 const Twine &Name = "");
779
780 /// Create a call to Masked Scatter intrinsic
781 CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment,
782 Value *Mask = nullptr);
783
784 /// Create a call to Masked Expand Load intrinsic
785 CallInst *CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask = nullptr,
786 Value *PassThru = nullptr,
787 const Twine &Name = "");
788
789 /// Create a call to Masked Compress Store intrinsic
790 CallInst *CreateMaskedCompressStore(Value *Val, Value *Ptr,
791 Value *Mask = nullptr);
792
793 /// Create an assume intrinsic call that allows the optimizer to
794 /// assume that the provided condition will be true.
795 ///
796 /// The optional argument \p OpBundles specifies operand bundles that are
797 /// added to the call instruction.
798 CallInst *CreateAssumption(Value *Cond,
799 ArrayRef<OperandBundleDef> OpBundles = llvm::None);
800
801 /// Create a llvm.experimental.noalias.scope.decl intrinsic call.
802 Instruction *CreateNoAliasScopeDeclaration(Value *Scope);
803 Instruction *CreateNoAliasScopeDeclaration(MDNode *ScopeTag) {
804 return CreateNoAliasScopeDeclaration(
805 MetadataAsValue::get(Context, ScopeTag));
806 }
807
808 /// Create a call to the experimental.gc.statepoint intrinsic to
809 /// start a new statepoint sequence.
810 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
811 FunctionCallee ActualCallee,
812 ArrayRef<Value *> CallArgs,
813 Optional<ArrayRef<Value *>> DeoptArgs,
814 ArrayRef<Value *> GCArgs,
815 const Twine &Name = "");
816
817 /// Create a call to the experimental.gc.statepoint intrinsic to
818 /// start a new statepoint sequence.
819 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
820 FunctionCallee ActualCallee, uint32_t Flags,
821 ArrayRef<Value *> CallArgs,
822 Optional<ArrayRef<Use>> TransitionArgs,
823 Optional<ArrayRef<Use>> DeoptArgs,
824 ArrayRef<Value *> GCArgs,
825 const Twine &Name = "");
826
827 /// Conveninence function for the common case when CallArgs are filled
828 /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be
829 /// .get()'ed to get the Value pointer.
830 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
831 FunctionCallee ActualCallee,
832 ArrayRef<Use> CallArgs,
833 Optional<ArrayRef<Value *>> DeoptArgs,
834 ArrayRef<Value *> GCArgs,
835 const Twine &Name = "");
836
837 /// Create an invoke to the experimental.gc.statepoint intrinsic to
838 /// start a new statepoint sequence.
839 InvokeInst *
840 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
841 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
842 BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs,
843 Optional<ArrayRef<Value *>> DeoptArgs,
844 ArrayRef<Value *> GCArgs, const Twine &Name = "");
845
846 /// Create an invoke to the experimental.gc.statepoint intrinsic to
847 /// start a new statepoint sequence.
848 InvokeInst *CreateGCStatepointInvoke(
849 uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
850 BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
851 ArrayRef<Value *> InvokeArgs, Optional<ArrayRef<Use>> TransitionArgs,
852 Optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
853 const Twine &Name = "");
854
855 // Convenience function for the common case when CallArgs are filled in using
856 // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to
857 // get the Value *.
858 InvokeInst *
859 CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes,
860 FunctionCallee ActualInvokee, BasicBlock *NormalDest,
861 BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
862 Optional<ArrayRef<Value *>> DeoptArgs,
863 ArrayRef<Value *> GCArgs, const Twine &Name = "");
864
865 /// Create a call to the experimental.gc.result intrinsic to extract
866 /// the result from a call wrapped in a statepoint.
867 CallInst *CreateGCResult(Instruction *Statepoint,
868 Type *ResultType,
869 const Twine &Name = "");
870
871 /// Create a call to the experimental.gc.relocate intrinsics to
872 /// project the relocated value of one pointer from the statepoint.
873 CallInst *CreateGCRelocate(Instruction *Statepoint,
874 int BaseOffset,
875 int DerivedOffset,
876 Type *ResultType,
877 const Twine &Name = "");
878
879 /// Create a call to the experimental.gc.pointer.base intrinsic to get the
880 /// base pointer for the specified derived pointer.
881 CallInst *CreateGCGetPointerBase(Value *DerivedPtr, const Twine &Name = "");
882
883 /// Create a call to the experimental.gc.get.pointer.offset intrinsic to get
884 /// the offset of the specified derived pointer from its base.
885 CallInst *CreateGCGetPointerOffset(Value *DerivedPtr, const Twine &Name = "");
886
887 /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
888 /// will be the same type as that of \p Scaling.
889 Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
890
891 /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
892 Value *CreateStepVector(Type *DstType, const Twine &Name = "");
893
894 /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
895 /// type.
896 CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
897 Instruction *FMFSource = nullptr,
898 const Twine &Name = "");
899
900 /// Create a call to intrinsic \p ID with 2 operands which is mangled on the
901 /// first type.
902 CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
903 Instruction *FMFSource = nullptr,
904 const Twine &Name = "");
905
906 /// Create a call to intrinsic \p ID with \p Args, mangled using \p Types. If
907 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
908 /// the intrinsic.
909 CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types,
910 ArrayRef<Value *> Args,
911 Instruction *FMFSource = nullptr,
912 const Twine &Name = "");
913
914 /// Create a call to intrinsic \p ID with \p RetTy and \p Args. If
915 /// \p FMFSource is provided, copy fast-math-flags from that instruction to
916 /// the intrinsic.
917 CallInst *CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
918 ArrayRef<Value *> Args,
919 Instruction *FMFSource = nullptr,
920 const Twine &Name = "");
921
922 /// Create call to the minnum intrinsic.
923 CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
924 return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name);
925 }
926
927 /// Create call to the maxnum intrinsic.
928 CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
929 return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name);
930 }
931
932 /// Create call to the minimum intrinsic.
933 CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
934 return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
935 }
936
937 /// Create call to the maximum intrinsic.
938 CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
939 return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
940 }
941
942 /// Create a call to the arithmetic_fence intrinsic.
943 CallInst *CreateArithmeticFence(Value *Val, Type *DstType,
944 const Twine &Name = "") {
945 return CreateIntrinsic(Intrinsic::arithmetic_fence, DstType, Val, nullptr,
946 Name);
947 }
948
949 /// Create a call to the vector.extract intrinsic.
950 CallInst *CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx,
951 const Twine &Name = "") {
952 return CreateIntrinsic(Intrinsic::vector_extract,
953 {DstType, SrcVec->getType()}, {SrcVec, Idx}, nullptr,
954 Name);
955 }
956
957 /// Create a call to the vector.insert intrinsic.
958 CallInst *CreateInsertVector(Type *DstType, Value *SrcVec, Value *SubVec,
959 Value *Idx, const Twine &Name = "") {
960 return CreateIntrinsic(Intrinsic::vector_insert,
961 {DstType, SubVec->getType()}, {SrcVec, SubVec, Idx},
962 nullptr, Name);
963 }
964
965private:
966 /// Create a call to a masked intrinsic with given Id.
967 CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
968 ArrayRef<Type *> OverloadedTypes,
969 const Twine &Name = "");
970
971 Value *getCastedInt8PtrValue(Value *Ptr);
972
973 //===--------------------------------------------------------------------===//
974 // Instruction creation methods: Terminators
975 //===--------------------------------------------------------------------===//
976
977private:
978 /// Helper to add branch weight and unpredictable metadata onto an
979 /// instruction.
980 /// \returns The annotated instruction.
981 template <typename InstTy>
982 InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) {
983 if (Weights)
984 I->setMetadata(LLVMContext::MD_prof, Weights);
985 if (Unpredictable)
986 I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable);
987 return I;
988 }
989
990public:
991 /// Create a 'ret void' instruction.
992 ReturnInst *CreateRetVoid() {
993 return Insert(ReturnInst::Create(Context));
994 }
995
996 /// Create a 'ret <val>' instruction.
997 ReturnInst *CreateRet(Value *V) {
998 return Insert(ReturnInst::Create(Context, V));
999 }
1000
1001 /// Create a sequence of N insertvalue instructions,
1002 /// with one Value from the retVals array each, that build a aggregate
1003 /// return value one value at a time, and a ret instruction to return
1004 /// the resulting aggregate value.
1005 ///
1006 /// This is a convenience function for code that uses aggregate return values
1007 /// as a vehicle for having multiple return values.
1008 ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) {
1009 Value *V = PoisonValue::get(getCurrentFunctionReturnType());
1010 for (unsigned i = 0; i != N; ++i)
1011 V = CreateInsertValue(V, retVals[i], i, "mrv");
1012 return Insert(ReturnInst::Create(Context, V));
1013 }
1014
1015 /// Create an unconditional 'br label X' instruction.
1016 BranchInst *CreateBr(BasicBlock *Dest) {
1017 return Insert(BranchInst::Create(Dest));
1018 }
1019
1020 /// Create a conditional 'br Cond, TrueDest, FalseDest'
1021 /// instruction.
1022 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
1023 MDNode *BranchWeights = nullptr,
1024 MDNode *Unpredictable = nullptr) {
1025 return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond),
1026 BranchWeights, Unpredictable));
1027 }
1028
1029 /// Create a conditional 'br Cond, TrueDest, FalseDest'
1030 /// instruction. Copy branch meta data if available.
1031 BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False,
1032 Instruction *MDSrc) {
1033 BranchInst *Br = BranchInst::Create(True, False, Cond);
1034 if (MDSrc) {
1035 unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable,
1036 LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};
1037 Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4));
1038 }
1039 return Insert(Br);
1040 }
1041
1042 /// Create a switch instruction with the specified value, default dest,
1043 /// and with a hint for the number of cases that will be added (for efficient
1044 /// allocation).
1045 SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10,
1046 MDNode *BranchWeights = nullptr,
1047 MDNode *Unpredictable = nullptr) {
1048 return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases),
1049 BranchWeights, Unpredictable));
1050 }
1051
1052 /// Create an indirect branch instruction with the specified address
1053 /// operand, with an optional hint for the number of destinations that will be
1054 /// added (for efficient allocation).
1055 IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) {
1056 return Insert(IndirectBrInst::Create(Addr, NumDests));
1057 }
1058
1059 /// Create an invoke instruction.
1060 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1061 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1062 ArrayRef<Value *> Args,
1063 ArrayRef<OperandBundleDef> OpBundles,
1064 const Twine &Name = "") {
1065 InvokeInst *II =
1066 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles);
1067 if (IsFPConstrained)
1068 setConstrainedFPCallAttr(II);
1069 return Insert(II, Name);
1070 }
1071 InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
1072 BasicBlock *NormalDest, BasicBlock *UnwindDest,
1073 ArrayRef<Value *> Args = None,
1074 const Twine &Name = "") {
1075 InvokeInst *II =
1076 InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args);
1077 if (IsFPConstrained)
1078 setConstrainedFPCallAttr(II);
1079 return Insert(II, Name);
1080 }
1081
1082 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1083 BasicBlock *UnwindDest, ArrayRef<Value *> Args,
1084 ArrayRef<OperandBundleDef> OpBundles,
1085 const Twine &Name = "") {
1086 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1087 NormalDest, UnwindDest, Args, OpBundles, Name);
1088 }
1089
1090 InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
1091 BasicBlock *UnwindDest,
1092 ArrayRef<Value *> Args = None,
1093 const Twine &Name = "") {
1094 return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
1095 NormalDest, UnwindDest, Args, Name);
1096 }
1097
1098 /// \brief Create a callbr instruction.
1099 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1100 BasicBlock *DefaultDest,
1101 ArrayRef<BasicBlock *> IndirectDests,
1102 ArrayRef<Value *> Args = None,
1103 const Twine &Name = "") {
1104 return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
1105 Args), Name);
1106 }
1107 CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
1108 BasicBlock *DefaultDest,
1109 ArrayRef<BasicBlock *> IndirectDests,
1110 ArrayRef<Value *> Args,
1111 ArrayRef<OperandBundleDef> OpBundles,
1112 const Twine &Name = "") {
1113 return Insert(
1114 CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
1115 OpBundles), Name);
1116 }
1117
1118 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1119 ArrayRef<BasicBlock *> IndirectDests,
1120 ArrayRef<Value *> Args = None,
1121 const Twine &Name = "") {
1122 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1123 DefaultDest, IndirectDests, Args, Name);
1124 }
1125 CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
1126 ArrayRef<BasicBlock *> IndirectDests,
1127 ArrayRef<Value *> Args,
1128 ArrayRef<OperandBundleDef> OpBundles,
1129 const Twine &Name = "") {
1130 return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
1131 DefaultDest, IndirectDests, Args, Name);
1132 }
1133
1134 ResumeInst *CreateResume(Value *Exn) {
1135 return Insert(ResumeInst::Create(Exn));
1136 }
1137
1138 CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad,
1139 BasicBlock *UnwindBB = nullptr) {
1140 return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB));
1141 }
1142
1143 CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB,
1144 unsigned NumHandlers,
1145 const Twine &Name = "") {
1146 return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers),
1147 Name);
1148 }
1149
1150 CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args,
1151 const Twine &Name = "") {
1152 return Insert(CatchPadInst::Create(ParentPad, Args), Name);
1153 }
1154
1155 CleanupPadInst *CreateCleanupPad(Value *ParentPad,
1156 ArrayRef<Value *> Args = None,
1157 const Twine &Name = "") {
1158 return Insert(CleanupPadInst::Create(ParentPad, Args), Name);
1159 }
1160
1161 CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) {
1162 return Insert(CatchReturnInst::Create(CatchPad, BB));
1163 }
1164
1165 UnreachableInst *CreateUnreachable() {
1166 return Insert(new UnreachableInst(Context));
1167 }
1168
1169 //===--------------------------------------------------------------------===//
1170 // Instruction creation methods: Binary Operators
1171 //===--------------------------------------------------------------------===//
1172private:
1173 BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc,
1174 Value *LHS, Value *RHS,
1175 const Twine &Name,
1176 bool HasNUW, bool HasNSW) {
1177 BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name);
1178 if (HasNUW) BO->setHasNoUnsignedWrap();
1179 if (HasNSW) BO->setHasNoSignedWrap();
1180 return BO;
1181 }
1182
1183 Instruction *setFPAttrs(Instruction *I, MDNode *FPMD,
1184 FastMathFlags FMF) const {
1185 if (!FPMD)
1186 FPMD = DefaultFPMathTag;
1187 if (FPMD)
1188 I->setMetadata(LLVMContext::MD_fpmath, FPMD);
1189 I->setFastMathFlags(FMF);
1190 return I;
1191 }
1192
1193 Value *getConstrainedFPRounding(Optional<RoundingMode> Rounding) {
1194 RoundingMode UseRounding = DefaultConstrainedRounding;
1195
1196 if (Rounding)
1197 UseRounding = Rounding.value();
1198
1199 Optional<StringRef> RoundingStr = convertRoundingModeToStr(UseRounding);
1200 assert(RoundingStr && "Garbage strict rounding mode!")(static_cast <bool> (RoundingStr && "Garbage strict rounding mode!"
) ? void (0) : __assert_fail ("RoundingStr && \"Garbage strict rounding mode!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1200, __extension__ __PRETTY_FUNCTION__
))
;
1201 auto *RoundingMDS = MDString::get(Context, RoundingStr.value());
1202
1203 return MetadataAsValue::get(Context, RoundingMDS);
1204 }
1205
1206 Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) {
1207 fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept;
1208
1209 if (Except)
1210 UseExcept = Except.value();
1211
1212 Optional<StringRef> ExceptStr = convertExceptionBehaviorToStr(UseExcept);
1213 assert(ExceptStr && "Garbage strict exception behavior!")(static_cast <bool> (ExceptStr && "Garbage strict exception behavior!"
) ? void (0) : __assert_fail ("ExceptStr && \"Garbage strict exception behavior!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1213, __extension__ __PRETTY_FUNCTION__
))
;
1214 auto *ExceptMDS = MDString::get(Context, ExceptStr.value());
1215
1216 return MetadataAsValue::get(Context, ExceptMDS);
1217 }
1218
1219 Value *getConstrainedFPPredicate(CmpInst::Predicate Predicate) {
1220 assert(CmpInst::isFPPredicate(Predicate) &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
1221 Predicate != CmpInst::FCMP_FALSE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
1222 Predicate != CmpInst::FCMP_TRUE &&(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
1223 "Invalid constrained FP comparison predicate!")(static_cast <bool> (CmpInst::isFPPredicate(Predicate) &&
Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst
::FCMP_TRUE && "Invalid constrained FP comparison predicate!"
) ? void (0) : __assert_fail ("CmpInst::isFPPredicate(Predicate) && Predicate != CmpInst::FCMP_FALSE && Predicate != CmpInst::FCMP_TRUE && \"Invalid constrained FP comparison predicate!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1223, __extension__ __PRETTY_FUNCTION__
))
;
1224
1225 StringRef PredicateStr = CmpInst::getPredicateName(Predicate);
1226 auto *PredicateMDS = MDString::get(Context, PredicateStr);
1227
1228 return MetadataAsValue::get(Context, PredicateMDS);
1229 }
1230
1231public:
1232 Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "",
1233 bool HasNUW = false, bool HasNSW = false) {
1234 if (Value *V =
34
Assuming 'V' is non-null
35
Taking true branch
1235 Folder.FoldNoWrapBinOp(Instruction::Add, LHS, RHS, HasNUW, HasNSW))
1236 return V;
1237 return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, HasNUW,
1238 HasNSW);
1239 }
1240
1241 Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1242 return CreateAdd(LHS, RHS, Name, false, true);
1243 }
1244
1245 Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") {
1246 return CreateAdd(LHS, RHS, Name, true, false);
1247 }
1248
1249 Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "",
1250 bool HasNUW = false, bool HasNSW = false) {
1251 if (Value *V =
1252 Folder.FoldNoWrapBinOp(Instruction::Sub, LHS, RHS, HasNUW, HasNSW))
1253 return V;
1254 return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name, HasNUW,
1255 HasNSW);
1256 }
1257
1258 Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1259 return CreateSub(LHS, RHS, Name, false, true);
1260 }
1261
1262 Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") {
1263 return CreateSub(LHS, RHS, Name, true, false);
1264 }
1265
1266 Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "",
1267 bool HasNUW = false, bool HasNSW = false) {
1268 if (Value *V =
1269 Folder.FoldNoWrapBinOp(Instruction::Mul, LHS, RHS, HasNUW, HasNSW))
1270 return V;
1271 return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name, HasNUW,
1272 HasNSW);
1273 }
1274
1275 Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1276 return CreateMul(LHS, RHS, Name, false, true);
1277 }
1278
1279 Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") {
1280 return CreateMul(LHS, RHS, Name, true, false);
1281 }
1282
1283 Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1284 bool isExact = false) {
1285 if (Value *V = Folder.FoldExactBinOp(Instruction::UDiv, LHS, RHS, isExact))
1286 return V;
1287 if (!isExact)
1288 return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name);
1289 return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name);
1290 }
1291
1292 Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1293 return CreateUDiv(LHS, RHS, Name, true);
1294 }
1295
1296 Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "",
1297 bool isExact = false) {
1298 if (Value *V = Folder.FoldExactBinOp(Instruction::SDiv, LHS, RHS, isExact))
1299 return V;
1300 if (!isExact)
1301 return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name);
1302 return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name);
1303 }
1304
1305 Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") {
1306 return CreateSDiv(LHS, RHS, Name, true);
1307 }
1308
1309 Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") {
1310 if (Value *V = Folder.FoldBinOp(Instruction::URem, LHS, RHS))
1311 return V;
1312 return Insert(BinaryOperator::CreateURem(LHS, RHS), Name);
1313 }
1314
1315 Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") {
1316 if (Value *V = Folder.FoldBinOp(Instruction::SRem, LHS, RHS))
1317 return V;
1318 return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name);
1319 }
1320
1321 Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "",
1322 bool HasNUW = false, bool HasNSW = false) {
1323 if (Value *V =
1324 Folder.FoldNoWrapBinOp(Instruction::Shl, LHS, RHS, HasNUW, HasNSW))
1325 return V;
1326 return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name,
1327 HasNUW, HasNSW);
1328 }
1329
1330 Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "",
1331 bool HasNUW = false, bool HasNSW = false) {
1332 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1333 HasNUW, HasNSW);
1334 }
1335
1336 Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "",
1337 bool HasNUW = false, bool HasNSW = false) {
1338 return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name,
1339 HasNUW, HasNSW);
1340 }
1341
1342 Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "",
1343 bool isExact = false) {
1344 if (Value *V = Folder.FoldExactBinOp(Instruction::LShr, LHS, RHS, isExact))
1345 return V;
1346 if (!isExact)
1347 return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name);
1348 return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name);
1349 }
1350
1351 Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1352 bool isExact = false) {
1353 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1354 }
1355
1356 Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1357 bool isExact = false) {
1358 return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1359 }
1360
1361 Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "",
1362 bool isExact = false) {
1363 if (Value *V = Folder.FoldExactBinOp(Instruction::AShr, LHS, RHS, isExact))
1364 return V;
1365 if (!isExact)
1366 return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name);
1367 return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name);
1368 }
1369
1370 Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "",
1371 bool isExact = false) {
1372 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1373 }
1374
1375 Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "",
1376 bool isExact = false) {
1377 return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact);
1378 }
1379
1380 Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") {
1381 if (auto *V = Folder.FoldBinOp(Instruction::And, LHS, RHS))
1382 return V;
1383 return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name);
1384 }
1385
1386 Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1387 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1388 }
1389
1390 Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1391 return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1392 }
1393
1394 Value *CreateAnd(ArrayRef<Value*> Ops) {
1395 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1395, __extension__
__PRETTY_FUNCTION__))
;
1396 Value *Accum = Ops[0];
1397 for (unsigned i = 1; i < Ops.size(); i++)
1398 Accum = CreateAnd(Accum, Ops[i]);
1399 return Accum;
1400 }
1401
1402 Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
1403 if (auto *V = Folder.FoldBinOp(Instruction::Or, LHS, RHS))
1404 return V;
1405 return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
1406 }
1407
1408 Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1409 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1410 }
1411
1412 Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1413 return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1414 }
1415
1416 Value *CreateOr(ArrayRef<Value*> Ops) {
1417 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1417, __extension__
__PRETTY_FUNCTION__))
;
1418 Value *Accum = Ops[0];
1419 for (unsigned i = 1; i < Ops.size(); i++)
1420 Accum = CreateOr(Accum, Ops[i]);
1421 return Accum;
1422 }
1423
1424 Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") {
1425 if (Value *V = Folder.FoldBinOp(Instruction::Xor, LHS, RHS))
1426 return V;
1427 return Insert(BinaryOperator::CreateXor(LHS, RHS), Name);
1428 }
1429
1430 Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") {
1431 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1432 }
1433
1434 Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") {
1435 return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name);
1436 }
1437
1438 Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "",
1439 MDNode *FPMD = nullptr) {
1440 if (IsFPConstrained)
1441 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1442 L, R, nullptr, Name, FPMD);
1443
1444 if (Value *V = Folder.FoldBinOpFMF(Instruction::FAdd, L, R, FMF))
1445 return V;
1446 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF);
1447 return Insert(I, Name);
1448 }
1449
1450 /// Copy fast-math-flags from an instruction rather than using the builder's
1451 /// default FMF.
1452 Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource,
1453 const Twine &Name = "") {
1454 if (IsFPConstrained)
1455 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd,
1456 L, R, FMFSource, Name);
1457
1458 FastMathFlags FMF = FMFSource->getFastMathFlags();
1459 if (Value *V = Folder.FoldBinOpFMF(Instruction::FAdd, L, R, FMF))
1460 return V;
1461 Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, FMF);
1462 return Insert(I, Name);
1463 }
1464
1465 Value *CreateFSub(Value *L, Value *R, const Twine &Name = "",
1466 MDNode *FPMD = nullptr) {
1467 if (IsFPConstrained)
1468 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1469 L, R, nullptr, Name, FPMD);
1470
1471 if (Value *V = Folder.FoldBinOpFMF(Instruction::FSub, L, R, FMF))
1472 return V;
1473 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF);
1474 return Insert(I, Name);
1475 }
1476
1477 /// Copy fast-math-flags from an instruction rather than using the builder's
1478 /// default FMF.
1479 Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource,
1480 const Twine &Name = "") {
1481 if (IsFPConstrained)
1482 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub,
1483 L, R, FMFSource, Name);
1484
1485 FastMathFlags FMF = FMFSource->getFastMathFlags();
1486 if (Value *V = Folder.FoldBinOpFMF(Instruction::FSub, L, R, FMF))
1487 return V;
1488 Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, FMF);
1489 return Insert(I, Name);
1490 }
1491
1492 Value *CreateFMul(Value *L, Value *R, const Twine &Name = "",
1493 MDNode *FPMD = nullptr) {
1494 if (IsFPConstrained)
1495 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1496 L, R, nullptr, Name, FPMD);
1497
1498 if (Value *V = Folder.FoldBinOpFMF(Instruction::FMul, L, R, FMF))
1499 return V;
1500 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF);
1501 return Insert(I, Name);
1502 }
1503
1504 /// Copy fast-math-flags from an instruction rather than using the builder's
1505 /// default FMF.
1506 Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource,
1507 const Twine &Name = "") {
1508 if (IsFPConstrained)
1509 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul,
1510 L, R, FMFSource, Name);
1511
1512 FastMathFlags FMF = FMFSource->getFastMathFlags();
1513 if (Value *V = Folder.FoldBinOpFMF(Instruction::FMul, L, R, FMF))
1514 return V;
1515 Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, FMF);
1516 return Insert(I, Name);
1517 }
1518
1519 Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "",
1520 MDNode *FPMD = nullptr) {
1521 if (IsFPConstrained)
1522 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1523 L, R, nullptr, Name, FPMD);
1524
1525 if (Value *V = Folder.FoldBinOpFMF(Instruction::FDiv, L, R, FMF))
1526 return V;
1527 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF);
1528 return Insert(I, Name);
1529 }
1530
1531 /// Copy fast-math-flags from an instruction rather than using the builder's
1532 /// default FMF.
1533 Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource,
1534 const Twine &Name = "") {
1535 if (IsFPConstrained)
1536 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv,
1537 L, R, FMFSource, Name);
1538
1539 if (Value *V = Folder.FoldBinOpFMF(Instruction::FDiv, L, R, FMF))
1540 return V;
1541 Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, FMF);
1542 return Insert(I, Name);
1543 }
1544
1545 Value *CreateFRem(Value *L, Value *R, const Twine &Name = "",
1546 MDNode *FPMD = nullptr) {
1547 if (IsFPConstrained)
1548 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1549 L, R, nullptr, Name, FPMD);
1550
1551 if (Value *V = Folder.FoldBinOpFMF(Instruction::FRem, L, R, FMF)) return V;
1552 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF);
1553 return Insert(I, Name);
1554 }
1555
1556 /// Copy fast-math-flags from an instruction rather than using the builder's
1557 /// default FMF.
1558 Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource,
1559 const Twine &Name = "") {
1560 if (IsFPConstrained)
1561 return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem,
1562 L, R, FMFSource, Name);
1563
1564 FastMathFlags FMF = FMFSource->getFastMathFlags();
1565 if (Value *V = Folder.FoldBinOpFMF(Instruction::FRem, L, R, FMF)) return V;
1566 Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, FMF);
1567 return Insert(I, Name);
1568 }
1569
1570 Value *CreateBinOp(Instruction::BinaryOps Opc,
1571 Value *LHS, Value *RHS, const Twine &Name = "",
1572 MDNode *FPMathTag = nullptr) {
1573 if (Value *V = Folder.FoldBinOp(Opc, LHS, RHS)) return V;
1574 Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS);
1575 if (isa<FPMathOperator>(BinOp))
1576 setFPAttrs(BinOp, FPMathTag, FMF);
1577 return Insert(BinOp, Name);
1578 }
1579
1580 Value *CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1581 assert(Cond2->getType()->isIntOrIntVectorTy(1))(static_cast <bool> (Cond2->getType()->isIntOrIntVectorTy
(1)) ? void (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "llvm/include/llvm/IR/IRBuilder.h", 1581, __extension__ __PRETTY_FUNCTION__
))
;
1582 return CreateSelect(Cond1, Cond2,
1583 ConstantInt::getNullValue(Cond2->getType()), Name);
1584 }
1585
1586 Value *CreateLogicalOr(Value *Cond1, Value *Cond2, const Twine &Name = "") {
1587 assert(Cond2->getType()->isIntOrIntVectorTy(1))(static_cast <bool> (Cond2->getType()->isIntOrIntVectorTy
(1)) ? void (0) : __assert_fail ("Cond2->getType()->isIntOrIntVectorTy(1)"
, "llvm/include/llvm/IR/IRBuilder.h", 1587, __extension__ __PRETTY_FUNCTION__
))
;
1588 return CreateSelect(Cond1, ConstantInt::getAllOnesValue(Cond2->getType()),
1589 Cond2, Name);
1590 }
1591
1592 // NOTE: this is sequential, non-commutative, ordered reduction!
1593 Value *CreateLogicalOr(ArrayRef<Value *> Ops) {
1594 assert(!Ops.empty())(static_cast <bool> (!Ops.empty()) ? void (0) : __assert_fail
("!Ops.empty()", "llvm/include/llvm/IR/IRBuilder.h", 1594, __extension__
__PRETTY_FUNCTION__))
;
1595 Value *Accum = Ops[0];
1596 for (unsigned i = 1; i < Ops.size(); i++)
1597 Accum = CreateLogicalOr(Accum, Ops[i]);
1598 return Accum;
1599 }
1600
1601 CallInst *CreateConstrainedFPBinOp(
1602 Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr,
1603 const Twine &Name = "", MDNode *FPMathTag = nullptr,
1604 Optional<RoundingMode> Rounding = None,
1605 Optional<fp::ExceptionBehavior> Except = None);
1606
1607 Value *CreateNeg(Value *V, const Twine &Name = "", bool HasNUW = false,
1608 bool HasNSW = false) {
1609 return CreateSub(Constant::getNullValue(V->getType()), V, Name, HasNUW,
1610 HasNSW);
1611 }
1612
1613 Value *CreateNSWNeg(Value *V, const Twine &Name = "") {
1614 return CreateNeg(V, Name, false, true);
1615 }
1616
1617 Value *CreateNUWNeg(Value *V, const Twine &Name = "") {
1618 return CreateNeg(V, Name, true, false);
1619 }
1620
1621 Value *CreateFNeg(Value *V, const Twine &Name = "",
1622 MDNode *FPMathTag = nullptr) {
1623 if (Value *Res = Folder.FoldUnOpFMF(Instruction::FNeg, V, FMF))
1624 return Res;
1625 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF),
1626 Name);
1627 }
1628
1629 /// Copy fast-math-flags from an instruction rather than using the builder's
1630 /// default FMF.
1631 Value *CreateFNegFMF(Value *V, Instruction *FMFSource,
1632 const Twine &Name = "") {
1633 FastMathFlags FMF = FMFSource->getFastMathFlags();
1634 if (Value *Res = Folder.FoldUnOpFMF(Instruction::FNeg, V, FMF))
1635 return Res;
1636 return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr, FMF),
1637 Name);
1638 }
1639
1640 Value *CreateNot(Value *V, const Twine &Name = "") {
1641 return CreateXor(V, Constant::getAllOnesValue(V->getType()), Name);
1642 }
1643
1644 Value *CreateUnOp(Instruction::UnaryOps Opc,
1645 Value *V, const Twine &Name = "",
1646 MDNode *FPMathTag = nullptr) {
1647 if (Value *Res = Folder.FoldUnOpFMF(Opc, V, FMF))
1648 return Res;
1649 Instruction *UnOp = UnaryOperator::Create(Opc, V);
1650 if (isa<FPMathOperator>(UnOp))
1651 setFPAttrs(UnOp, FPMathTag, FMF);
1652 return Insert(UnOp, Name);
1653 }
1654
1655 /// Create either a UnaryOperator or BinaryOperator depending on \p Opc.
1656 /// Correct number of operands must be passed accordingly.
1657 Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
1658 const Twine &Name = "", MDNode *FPMathTag = nullptr);
1659
1660 //===--------------------------------------------------------------------===//
1661 // Instruction creation methods: Memory Instructions
1662 //===--------------------------------------------------------------------===//
1663
1664 AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace,
1665 Value *ArraySize = nullptr, const Twine &Name = "") {
1666 const DataLayout &DL = BB->getModule()->getDataLayout();
1667 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1668 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1669 }
1670
1671 AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr,
1672 const Twine &Name = "") {
1673 const DataLayout &DL = BB->getModule()->getDataLayout();
1674 Align AllocaAlign = DL.getPrefTypeAlign(Ty);
1675 unsigned AddrSpace = DL.getAllocaAddrSpace();
1676 return Insert(new AllocaInst(Ty, AddrSpace, ArraySize, AllocaAlign), Name);
1677 }
1678
1679 /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of
1680 /// converting the string to 'bool' for the isVolatile parameter.
1681 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) {
1682 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1683 }
1684
1685 LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
1686 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), Name);
1687 }
1688
1689 LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile,
1690 const Twine &Name = "") {
1691 return CreateAlignedLoad(Ty, Ptr, MaybeAlign(), isVolatile, Name);
1692 }
1693
1694 StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
1695 return CreateAlignedStore(Val, Ptr, MaybeAlign(), isVolatile);
1696 }
1697
1698 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1699 const char *Name) {
1700 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1701 }
1702
1703 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1704 const Twine &Name = "") {
1705 return CreateAlignedLoad(Ty, Ptr, Align, /*isVolatile*/false, Name);
1706 }
1707
1708 LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align,
1709 bool isVolatile, const Twine &Name = "") {
1710 if (!Align) {
1711 const DataLayout &DL = BB->getModule()->getDataLayout();
1712 Align = DL.getABITypeAlign(Ty);
1713 }
1714 return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile, *Align), Name);
1715 }
1716
1717 StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align,
1718 bool isVolatile = false) {
1719 if (!Align) {
1720 const DataLayout &DL = BB->getModule()->getDataLayout();
1721 Align = DL.getABITypeAlign(Val->getType());
1722 }
1723 return Insert(new StoreInst(Val, Ptr, isVolatile, *Align));
1724 }
1725 FenceInst *CreateFence(AtomicOrdering Ordering,
1726 SyncScope::ID SSID = SyncScope::System,
1727 const Twine &Name = "") {
1728 return Insert(new FenceInst(Context, Ordering, SSID), Name);
1729 }
1730
1731 AtomicCmpXchgInst *
1732 CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
1733 AtomicOrdering SuccessOrdering,
1734 AtomicOrdering FailureOrdering,
1735 SyncScope::ID SSID = SyncScope::System) {
1736 if (!Align) {
1737 const DataLayout &DL = BB->getModule()->getDataLayout();
1738 Align = llvm::Align(DL.getTypeStoreSize(New->getType()));
1739 }
1740
1741 return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, *Align, SuccessOrdering,
1742 FailureOrdering, SSID));
1743 }
1744
1745 AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr,
1746 Value *Val, MaybeAlign Align,
1747 AtomicOrdering Ordering,
1748 SyncScope::ID SSID = SyncScope::System) {
1749 if (!Align) {
1750 const DataLayout &DL = BB->getModule()->getDataLayout();
1751 Align = llvm::Align(DL.getTypeStoreSize(Val->getType()));
1752 }
1753
1754 return Insert(new AtomicRMWInst(Op, Ptr, Val, *Align, Ordering, SSID));
1755 }
1756
1757 Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1758 const Twine &Name = "", bool IsInBounds = false) {
1759 if (auto *V = Folder.FoldGEP(Ty, Ptr, IdxList, IsInBounds))
1760 return V;
1761 return Insert(IsInBounds
1762 ? GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList)
1763 : GetElementPtrInst::Create(Ty, Ptr, IdxList),
1764 Name);
1765 }
1766
1767 Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
1768 const Twine &Name = "") {
1769 return CreateGEP(Ty, Ptr, IdxList, Name, /* IsInBounds */ true);
1770 }
1771
1772 Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1773 const Twine &Name = "") {
1774 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1775
1776 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false))
1777 return V;
1778
1779 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1780 }
1781
1782 Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0,
1783 const Twine &Name = "") {
1784 Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0);
1785
1786 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true))
1787 return V;
1788
1789 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1790 }
1791
1792 Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1,
1793 const Twine &Name = "") {
1794 Value *Idxs[] = {
1795 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1796 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1797 };
1798
1799 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false))
1800 return V;
1801
1802 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1803 }
1804
1805 Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0,
1806 unsigned Idx1, const Twine &Name = "") {
1807 Value *Idxs[] = {
1808 ConstantInt::get(Type::getInt32Ty(Context), Idx0),
1809 ConstantInt::get(Type::getInt32Ty(Context), Idx1)
1810 };
1811
1812 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true))
1813 return V;
1814
1815 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1816 }
1817
1818 Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1819 const Twine &Name = "") {
1820 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1821
1822 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/false))
1823 return V;
1824
1825 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
1826 }
1827
1828 Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1829 const Twine &Name = "") {
1830 Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
1831
1832 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idx, /*IsInBounds=*/true))
1833 return V;
1834
1835 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
1836 }
1837
1838 Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1,
1839 const Twine &Name = "") {
1840 Value *Idxs[] = {
1841 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1842 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1843 };
1844
1845 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/false))
1846 return V;
1847
1848 return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
1849 }
1850
1851 Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0,
1852 uint64_t Idx1, const Twine &Name = "") {
1853 Value *Idxs[] = {
1854 ConstantInt::get(Type::getInt64Ty(Context), Idx0),
1855 ConstantInt::get(Type::getInt64Ty(Context), Idx1)
1856 };
1857
1858 if (auto *V = Folder.FoldGEP(Ty, Ptr, Idxs, /*IsInBounds=*/true))
1859 return V;
1860
1861 return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
1862 }
1863
1864 Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx,
1865 const Twine &Name = "") {
1866 return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name);
1867 }
1868
1869 /// Same as CreateGlobalString, but return a pointer with "i8*" type
1870 /// instead of a pointer to array of i8.
1871 ///
1872 /// If no module is given via \p M, it is take from the insertion point basic
1873 /// block.
1874 Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "",
1875 unsigned AddressSpace = 0,
1876 Module *M = nullptr) {
1877 GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace, M);
1878 Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1879 Constant *Indices[] = {Zero, Zero};
1880 return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV,
1881 Indices);
1882 }
1883
1884 //===--------------------------------------------------------------------===//
1885 // Instruction creation methods: Cast/Conversion Operators
1886 //===--------------------------------------------------------------------===//
1887
1888 Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
1889 return CreateCast(Instruction::Trunc, V, DestTy, Name);
1890 }
1891
1892 Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") {
1893 return CreateCast(Instruction::ZExt, V, DestTy, Name);
1894 }
1895
1896 Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") {
1897 return CreateCast(Instruction::SExt, V, DestTy, Name);
1898 }
1899
1900 /// Create a ZExt or Trunc from the integer value V to DestTy. Return
1901 /// the value untouched if the type of V is already DestTy.
1902 Value *CreateZExtOrTrunc(Value *V, Type *DestTy,
1903 const Twine &Name = "") {
1904 assert(V->getType()->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1906, __extension__ __PRETTY_FUNCTION__
))
1905 DestTy->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1906, __extension__ __PRETTY_FUNCTION__
))
1906 "Can only zero extend/truncate integers!")(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only zero extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1906, __extension__ __PRETTY_FUNCTION__
))
;
1907 Type *VTy = V->getType();
1908 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1909 return CreateZExt(V, DestTy, Name);
1910 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1911 return CreateTrunc(V, DestTy, Name);
1912 return V;
1913 }
1914
1915 /// Create a SExt or Trunc from the integer value V to DestTy. Return
1916 /// the value untouched if the type of V is already DestTy.
1917 Value *CreateSExtOrTrunc(Value *V, Type *DestTy,
1918 const Twine &Name = "") {
1919 assert(V->getType()->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1921, __extension__ __PRETTY_FUNCTION__
))
1920 DestTy->isIntOrIntVectorTy() &&(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1921, __extension__ __PRETTY_FUNCTION__
))
1921 "Can only sign extend/truncate integers!")(static_cast <bool> (V->getType()->isIntOrIntVectorTy
() && DestTy->isIntOrIntVectorTy() && "Can only sign extend/truncate integers!"
) ? void (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\""
, "llvm/include/llvm/IR/IRBuilder.h", 1921, __extension__ __PRETTY_FUNCTION__
))
;
1922 Type *VTy = V->getType();
1923 if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits())
1924 return CreateSExt(V, DestTy, Name);
1925 if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1926 return CreateTrunc(V, DestTy, Name);
1927 return V;
1928 }
1929
1930 Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") {
1931 if (IsFPConstrained)
1932 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui,
1933 V, DestTy, nullptr, Name);
1934 return CreateCast(Instruction::FPToUI, V, DestTy, Name);
1935 }
1936
1937 Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") {
1938 if (IsFPConstrained)
1939 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi,
1940 V, DestTy, nullptr, Name);
1941 return CreateCast(Instruction::FPToSI, V, DestTy, Name);
1942 }
1943
1944 Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
1945 if (IsFPConstrained)
1946 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp,
1947 V, DestTy, nullptr, Name);
1948 return CreateCast(Instruction::UIToFP, V, DestTy, Name);
1949 }
1950
1951 Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){
1952 if (IsFPConstrained)
1953 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp,
1954 V, DestTy, nullptr, Name);
1955 return CreateCast(Instruction::SIToFP, V, DestTy, Name);
1956 }
1957
1958 Value *CreateFPTrunc(Value *V, Type *DestTy,
1959 const Twine &Name = "") {
1960 if (IsFPConstrained)
1961 return CreateConstrainedFPCast(
1962 Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr,
1963 Name);
1964 return CreateCast(Instruction::FPTrunc, V, DestTy, Name);
1965 }
1966
1967 Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") {
1968 if (IsFPConstrained)
1969 return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext,
1970 V, DestTy, nullptr, Name);
1971 return CreateCast(Instruction::FPExt, V, DestTy, Name);
1972 }
1973
1974 Value *CreatePtrToInt(Value *V, Type *DestTy,
1975 const Twine &Name = "") {
1976 return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
1977 }
1978
1979 Value *CreateIntToPtr(Value *V, Type *DestTy,
1980 const Twine &Name = "") {
1981 return CreateCast(Instruction::IntToPtr, V, DestTy, Name);
1982 }
1983
1984 Value *CreateBitCast(Value *V, Type *DestTy,
1985 const Twine &Name = "") {
1986 return CreateCast(Instruction::BitCast, V, DestTy, Name);
1987 }
1988
1989 Value *CreateAddrSpaceCast(Value *V, Type *DestTy,
1990 const Twine &Name = "") {
1991 return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name);
1992 }
1993
1994 Value *CreateZExtOrBitCast(Value *V, Type *DestTy,
1995 const Twine &Name = "") {
1996 if (V->getType() == DestTy)
1997 return V;
1998 if (auto *VC = dyn_cast<Constant>(V))
1999 return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name);
2000 return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name);
2001 }
2002
2003 Value *CreateSExtOrBitCast(Value *V, Type *DestTy,
2004 const Twine &Name = "") {
2005 if (V->getType() == DestTy)
2006 return V;
2007 if (auto *VC = dyn_cast<Constant>(V))
2008 return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name);
2009 return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name);
2010 }
2011
2012 Value *CreateTruncOrBitCast(Value *V, Type *DestTy,
2013 const Twine &Name = "") {
2014 if (V->getType() == DestTy)
2015 return V;
2016 if (auto *VC = dyn_cast<Constant>(V))
2017 return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name);
2018 return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name);
2019 }
2020
2021 Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy,
2022 const Twine &Name = "") {
2023 if (V->getType() == DestTy)
2024 return V;
2025 if (auto *VC = dyn_cast<Constant>(V))
2026 return Insert(Folder.CreateCast(Op, VC, DestTy), Name);
2027 return Insert(CastInst::Create(Op, V, DestTy), Name);
2028 }
2029
2030 Value *CreatePointerCast(Value *V, Type *DestTy,
2031 const Twine &Name = "") {
2032 if (V->getType() == DestTy)
2033 return V;
2034 if (auto *VC = dyn_cast<Constant>(V))
2035 return Insert(Folder.CreatePointerCast(VC, DestTy), Name);
2036 return Insert(CastInst::CreatePointerCast(V, DestTy), Name);
2037 }
2038
2039 Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy,
2040 const Twine &Name = "") {
2041 if (V->getType() == DestTy)
2042 return V;
2043
2044 if (auto *VC = dyn_cast<Constant>(V)) {
2045 return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy),
2046 Name);
2047 }
2048
2049 return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy),
2050 Name);
2051 }
2052
2053 Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned,
2054 const Twine &Name = "") {
2055 if (V->getType() == DestTy)
2056 return V;
2057 if (auto *VC = dyn_cast<Constant>(V))
2058 return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name);
2059 return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name);
2060 }
2061
2062 Value *CreateBitOrPointerCast(Value *V, Type *DestTy,
2063 const Twine &Name = "") {
2064 if (V->getType() == DestTy)
2065 return V;
2066 if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy())
2067 return CreatePtrToInt(V, DestTy, Name);
2068 if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy())
2069 return CreateIntToPtr(V, DestTy, Name);
2070
2071 return CreateBitCast(V, DestTy, Name);
2072 }
2073
2074 Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") {
2075 if (V->getType() == DestTy)
2076 return V;
2077 if (auto *VC = dyn_cast<Constant>(V))
2078 return Insert(Folder.CreateFPCast(VC, DestTy), Name);
2079 return Insert(CastInst::CreateFPCast(V, DestTy), Name);
2080 }
2081
2082 CallInst *CreateConstrainedFPCast(
2083 Intrinsic::ID ID, Value *V, Type *DestTy,
2084 Instruction *FMFSource = nullptr, const Twine &Name = "",
2085 MDNode *FPMathTag = nullptr,
2086 Optional<RoundingMode> Rounding = None,
2087 Optional<fp::ExceptionBehavior> Except = None);
2088
2089 // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a
2090 // compile time error, instead of converting the string to bool for the
2091 // isSigned parameter.
2092 Value *CreateIntCast(Value *, Type *, const char *) = delete;
2093
2094 //===--------------------------------------------------------------------===//
2095 // Instruction creation methods: Compare Instructions
2096 //===--------------------------------------------------------------------===//
2097
2098 Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") {
2099 return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name);
2100 }
2101
2102 Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") {
2103 return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name);
2104 }
2105
2106 Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2107 return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name);
2108 }
2109
2110 Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2111 return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name);
2112 }
2113
2114 Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") {
2115 return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name);
2116 }
2117
2118 Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") {
2119 return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name);
2120 }
2121
2122 Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") {
2123 return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name);
2124 }
2125
2126 Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") {
2127 return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name);
2128 }
2129
2130 Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") {
2131 return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name);
2132 }
2133
2134 Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") {
2135 return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name);
2136 }
2137
2138 Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2139 MDNode *FPMathTag = nullptr) {
2140 return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag);
2141 }
2142
2143 Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "",
2144 MDNode *FPMathTag = nullptr) {
2145 return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag);
2146 }
2147
2148 Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "",
2149 MDNode *FPMathTag = nullptr) {
2150 return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag);
2151 }
2152
2153 Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "",
2154 MDNode *FPMathTag = nullptr) {
2155 return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag);
2156 }
2157
2158 Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "",
2159 MDNode *FPMathTag = nullptr) {
2160 return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag);
2161 }
2162
2163 Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "",
2164 MDNode *FPMathTag = nullptr) {
2165 return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag);
2166 }
2167
2168 Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "",
2169 MDNode *FPMathTag = nullptr) {
2170 return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag);
2171 }
2172
2173 Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "",
2174 MDNode *FPMathTag = nullptr) {
2175 return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag);
2176 }
2177
2178 Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "",
2179 MDNode *FPMathTag = nullptr) {
2180 return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag);
2181 }
2182
2183 Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "",
2184 MDNode *FPMathTag = nullptr) {
2185 return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag);
2186 }
2187
2188 Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "",
2189 MDNode *FPMathTag = nullptr) {
2190 return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag);
2191 }
2192
2193 Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "",
2194 MDNode *FPMathTag = nullptr) {
2195 return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag);
2196 }
2197
2198 Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "",
2199 MDNode *FPMathTag = nullptr) {
2200 return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag);
2201 }
2202
2203 Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "",
2204 MDNode *FPMathTag = nullptr) {
2205 return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag);
2206 }
2207
2208 Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2209 const Twine &Name = "") {
2210 if (auto *V = Folder.FoldICmp(P, LHS, RHS))
2211 return V;
2212 return Insert(new ICmpInst(P, LHS, RHS), Name);
2213 }
2214
2215 // Create a quiet floating-point comparison (i.e. one that raises an FP
2216 // exception only in the case where an input is a signaling NaN).
2217 // Note that this differs from CreateFCmpS only if IsFPConstrained is true.
2218 Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS,
2219 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2220 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, false);
2221 }
2222
2223 Value *CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS,
2224 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2225 return CmpInst::isFPPredicate(Pred)
2226 ? CreateFCmp(Pred, LHS, RHS, Name, FPMathTag)
2227 : CreateICmp(Pred, LHS, RHS, Name);
2228 }
2229
2230 // Create a signaling floating-point comparison (i.e. one that raises an FP
2231 // exception whenever an input is any NaN, signaling or quiet).
2232 // Note that this differs from CreateFCmp only if IsFPConstrained is true.
2233 Value *CreateFCmpS(CmpInst::Predicate P, Value *LHS, Value *RHS,
2234 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2235 return CreateFCmpHelper(P, LHS, RHS, Name, FPMathTag, true);
2236 }
2237
2238private:
2239 // Helper routine to create either a signaling or a quiet FP comparison.
2240 Value *CreateFCmpHelper(CmpInst::Predicate P, Value *LHS, Value *RHS,
2241 const Twine &Name, MDNode *FPMathTag,
2242 bool IsSignaling);
2243
2244public:
2245 CallInst *CreateConstrainedFPCmp(
2246 Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
2247 const Twine &Name = "", Optional<fp::ExceptionBehavior> Except = None);
2248
2249 //===--------------------------------------------------------------------===//
2250 // Instruction creation methods: Other Instructions
2251 //===--------------------------------------------------------------------===//
2252
2253 PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
2254 const Twine &Name = "") {
2255 PHINode *Phi = PHINode::Create(Ty, NumReservedValues);
2256 if (isa<FPMathOperator>(Phi))
2257 setFPAttrs(Phi, nullptr /* MDNode* */, FMF);
2258 return Insert(Phi, Name);
2259 }
2260
2261private:
2262 CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
2263 const Twine &Name = "",
2264 Instruction *FMFSource = nullptr,
2265 ArrayRef<OperandBundleDef> OpBundles = {});
2266
2267public:
2268 CallInst *CreateCall(FunctionType *FTy, Value *Callee,
2269 ArrayRef<Value *> Args = None, const Twine &Name = "",
2270 MDNode *FPMathTag = nullptr) {
2271 CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
2272 if (IsFPConstrained)
2273 setConstrainedFPCallAttr(CI);
2274 if (isa<FPMathOperator>(CI))
2275 setFPAttrs(CI, FPMathTag, FMF);
2276 return Insert(CI, Name);
2277 }
2278
2279 CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
2280 ArrayRef<OperandBundleDef> OpBundles,
2281 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2282 CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
2283 if (IsFPConstrained)
2284 setConstrainedFPCallAttr(CI);
2285 if (isa<FPMathOperator>(CI))
2286 setFPAttrs(CI, FPMathTag, FMF);
2287 return Insert(CI, Name);
2288 }
2289
2290 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None,
2291 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2292 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name,
2293 FPMathTag);
2294 }
2295
2296 CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args,
2297 ArrayRef<OperandBundleDef> OpBundles,
2298 const Twine &Name = "", MDNode *FPMathTag = nullptr) {
2299 return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args,
2300 OpBundles, Name, FPMathTag);
2301 }
2302
2303 CallInst *CreateConstrainedFPCall(
2304 Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "",
2305 Optional<RoundingMode> Rounding = None,
2306 Optional<fp::ExceptionBehavior> Except = None);
2307
2308 Value *CreateSelect(Value *C, Value *True, Value *False,
2309 const Twine &Name = "", Instruction *MDFrom = nullptr);
2310
2311 VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") {
2312 return Insert(new VAArgInst(List, Ty), Name);
2313 }
2314
2315 Value *CreateExtractElement(Value *Vec, Value *Idx,
2316 const Twine &Name = "") {
2317 if (Value *V = Folder.FoldExtractElement(Vec, Idx))
2318 return V;
2319 return Insert(ExtractElementInst::Create(Vec, Idx), Name);
2320 }
2321
2322 Value *CreateExtractElement(Value *Vec, uint64_t Idx,
2323 const Twine &Name = "") {
2324 return CreateExtractElement(Vec, getInt64(Idx), Name);
2325 }
2326
2327 Value *CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx,
2328 const Twine &Name = "") {
2329 return CreateInsertElement(PoisonValue::get(VecTy), NewElt, Idx, Name);
2330 }
2331
2332 Value *CreateInsertElement(Type *VecTy, Value *NewElt, uint64_t Idx,
2333 const Twine &Name = "") {
2334 return CreateInsertElement(PoisonValue::get(VecTy), NewElt, Idx, Name);
2335 }
2336
2337 Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx,
2338 const Twine &Name = "") {
2339 if (Value *V = Folder.FoldInsertElement(Vec, NewElt, Idx))
2340 return V;
2341 return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name);
2342 }
2343
2344 Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx,
2345 const Twine &Name = "") {
2346 return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
2347 }
2348
2349 Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
2350 const Twine &Name = "") {
2351 SmallVector<int, 16> IntMask;
2352 ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
2353 return CreateShuffleVector(V1, V2, IntMask, Name);
2354 }
2355
2356 /// See class ShuffleVectorInst for a description of the mask representation.
2357 Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
2358 const Twine &Name = "") {
2359 if (Value *V = Folder.FoldShuffleVector(V1, V2, Mask))
2360 return V;
2361 return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
2362 }
2363
2364 /// Create a unary shuffle. The second vector operand of the IR instruction
2365 /// is poison.
2366 Value *CreateShuffleVector(Value *V, ArrayRef<int> Mask,
2367 const Twine &Name = "") {
2368 return CreateShuffleVector(V, PoisonValue::get(V->getType()), Mask, Name);
2369 }
2370
2371 Value *CreateExtractValue(Value *Agg, ArrayRef<unsigned> Idxs,
2372 const Twine &Name = "") {
2373 if (auto *V = Folder.FoldExtractValue(Agg, Idxs))
2374 return V;
2375 return Insert(ExtractValueInst::Create(Agg, Idxs), Name);
2376 }
2377
2378 Value *CreateInsertValue(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2379 const Twine &Name = "") {
2380 if (auto *V = Folder.FoldInsertValue(Agg, Val, Idxs))
2381 return V;
2382 return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name);
2383 }
2384
2385 LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses,
2386 const Twine &Name = "") {
2387 return Insert(LandingPadInst::Create(Ty, NumClauses), Name);
2388 }
2389
2390 Value *CreateFreeze(Value *V, const Twine &Name = "") {
2391 return Insert(new FreezeInst(V), Name);
2392 }
2393
2394 //===--------------------------------------------------------------------===//
2395 // Utility creation methods
2396 //===--------------------------------------------------------------------===//
2397
2398 /// Return a boolean value testing if \p Arg == 0.
2399 Value *CreateIsNull(Value *Arg, const Twine &Name = "") {
2400 return CreateICmpEQ(Arg, ConstantInt::getNullValue(Arg->getType()), Name);
2401 }
2402
2403 /// Return a boolean value testing if \p Arg != 0.
2404 Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") {
2405 return CreateICmpNE(Arg, ConstantInt::getNullValue(Arg->getType()), Name);
2406 }
2407
2408 /// Return a boolean value testing if \p Arg < 0.
2409 Value *CreateIsNeg(Value *Arg, const Twine &Name = "") {
2410 return CreateICmpSLT(Arg, ConstantInt::getNullValue(Arg->getType()), Name);
2411 }
2412
2413 /// Return a boolean value testing if \p Arg > -1.
2414 Value *CreateIsNotNeg(Value *Arg, const Twine &Name = "") {
2415 return CreateICmpSGT(Arg, ConstantInt::getAllOnesValue(Arg->getType()),
2416 Name);
2417 }
2418
2419 /// Return the i64 difference between two pointer values, dividing out
2420 /// the size of the pointed-to objects.
2421 ///
2422 /// This is intended to implement C-style pointer subtraction. As such, the
2423 /// pointers must be appropriately aligned for their element types and
2424 /// pointing into the same object.
2425 Value *CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
2426 const Twine &Name = "");
2427
2428 /// Create a launder.invariant.group intrinsic call. If Ptr type is
2429 /// different from pointer to i8, it's casted to pointer to i8 in the same
2430 /// address space before call and casted back to Ptr type after call.
2431 Value *CreateLaunderInvariantGroup(Value *Ptr);
2432
2433 /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is
2434 /// different from pointer to i8, it's casted to pointer to i8 in the same
2435 /// address space before call and casted back to Ptr type after call.
2436 Value *CreateStripInvariantGroup(Value *Ptr);
2437
2438 /// Return a vector value that contains the vector V reversed
2439 Value *CreateVectorReverse(Value *V, const Twine &Name = "");
2440
2441 /// Return a vector splice intrinsic if using scalable vectors, otherwise
2442 /// return a shufflevector. If the immediate is positive, a vector is
2443 /// extracted from concat(V1, V2), starting at Imm. If the immediate
2444 /// is negative, we extract -Imm elements from V1 and the remaining
2445 /// elements from V2. Imm is a signed integer in the range
2446 /// -VL <= Imm < VL (where VL is the runtime vector length of the
2447 /// source/result vector)
2448 Value *CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
2449 const Twine &Name = "");
2450
2451 /// Return a vector value that contains \arg V broadcasted to \p
2452 /// NumElts elements.
2453 Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "");
2454
2455 /// Return a vector value that contains \arg V broadcasted to \p
2456 /// EC elements.
2457 Value *CreateVectorSplat(ElementCount EC, Value *V, const Twine &Name = "");
2458
2459 /// Return a value that has been extracted from a larger integer type.
2460 Value *CreateExtractInteger(const DataLayout &DL, Value *From,
2461 IntegerType *ExtractedTy, uint64_t Offset,
2462 const Twine &Name);
2463
2464 Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
2465 unsigned Dimension, unsigned LastIndex,
2466 MDNode *DbgInfo);
2467
2468 Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex,
2469 MDNode *DbgInfo);
2470
2471 Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base,
2472 unsigned Index, unsigned FieldIndex,
2473 MDNode *DbgInfo);
2474
2475private:
2476 /// Helper function that creates an assume intrinsic call that
2477 /// represents an alignment assumption on the provided pointer \p PtrValue
2478 /// with offset \p OffsetValue and alignment value \p AlignValue.
2479 CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
2480 Value *PtrValue, Value *AlignValue,
2481 Value *OffsetValue);
2482
2483public:
2484 /// Create an assume intrinsic call that represents an alignment
2485 /// assumption on the provided pointer.
2486 ///
2487 /// An optional offset can be provided, and if it is provided, the offset
2488 /// must be subtracted from the provided pointer to get the pointer with the
2489 /// specified alignment.
2490 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2491 unsigned Alignment,
2492 Value *OffsetValue = nullptr);
2493
2494 /// Create an assume intrinsic call that represents an alignment
2495 /// assumption on the provided pointer.
2496 ///
2497 /// An optional offset can be provided, and if it is provided, the offset
2498 /// must be subtracted from the provided pointer to get the pointer with the
2499 /// specified alignment.
2500 ///
2501 /// This overload handles the condition where the Alignment is dependent
2502 /// on an existing value rather than a static value.
2503 CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
2504 Value *Alignment,
2505 Value *OffsetValue = nullptr);
2506};
2507
2508/// This provides a uniform API for creating instructions and inserting
2509/// them into a basic block: either at the end of a BasicBlock, or at a specific
2510/// iterator location in a block.
2511///
2512/// Note that the builder does not expose the full generality of LLVM
2513/// instructions. For access to extra instruction properties, use the mutators
2514/// (e.g. setVolatile) on the instructions after they have been
2515/// created. Convenience state exists to specify fast-math flags and fp-math
2516/// tags.
2517///
2518/// The first template argument specifies a class to use for creating constants.
2519/// This defaults to creating minimally folded constants. The second template
2520/// argument allows clients to specify custom insertion hooks that are called on
2521/// every newly created insertion.
2522template <typename FolderTy = ConstantFolder,
2523 typename InserterTy = IRBuilderDefaultInserter>
2524class IRBuilder : public IRBuilderBase {
2525private:
2526 FolderTy Folder;
2527 InserterTy Inserter;
2528
2529public:
2530 IRBuilder(LLVMContext &C, FolderTy Folder, InserterTy Inserter = InserterTy(),
2531 MDNode *FPMathTag = nullptr,
2532 ArrayRef<OperandBundleDef> OpBundles = None)
2533 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles),
2534 Folder(Folder), Inserter(Inserter) {}
2535
2536 explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr,
2537 ArrayRef<OperandBundleDef> OpBundles = None)
2538 : IRBuilderBase(C, this->Folder, this->Inserter, FPMathTag, OpBundles) {}
2539
2540 explicit IRBuilder(BasicBlock *TheBB, FolderTy Folder,
2541 MDNode *FPMathTag = nullptr,
2542 ArrayRef<OperandBundleDef> OpBundles = None)
2543 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2544 FPMathTag, OpBundles), Folder(Folder) {
2545 SetInsertPoint(TheBB);
2546 }
2547
2548 explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr,
2549 ArrayRef<OperandBundleDef> OpBundles = None)
2550 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2551 FPMathTag, OpBundles) {
2552 SetInsertPoint(TheBB);
2553 }
2554
2555 explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr,
2556 ArrayRef<OperandBundleDef> OpBundles = None)
2557 : IRBuilderBase(IP->getContext(), this->Folder, this->Inserter,
2558 FPMathTag, OpBundles) {
2559 SetInsertPoint(IP);
2560 }
2561
2562 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, FolderTy Folder,
2563 MDNode *FPMathTag = nullptr,
2564 ArrayRef<OperandBundleDef> OpBundles = None)
2565 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2566 FPMathTag, OpBundles), Folder(Folder) {
2567 SetInsertPoint(TheBB, IP);
2568 }
2569
2570 IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP,
2571 MDNode *FPMathTag = nullptr,
2572 ArrayRef<OperandBundleDef> OpBundles = None)
2573 : IRBuilderBase(TheBB->getContext(), this->Folder, this->Inserter,
2574 FPMathTag, OpBundles) {
2575 SetInsertPoint(TheBB, IP);
2576 }
2577
2578 /// Avoid copying the full IRBuilder. Prefer using InsertPointGuard
2579 /// or FastMathFlagGuard instead.
2580 IRBuilder(const IRBuilder &) = delete;
2581
2582 InserterTy &getInserter() { return Inserter; }
2583};
2584
2585// Create wrappers for C Binding types (see CBindingWrapping.h).
2586DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast
<IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const
IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef
>(const_cast<IRBuilder<>*>(P)); }
2587
2588} // end namespace llvm
2589
2590#endif // LLVM_IR_IRBUILDER_H