Bug Summary

File:llvm/lib/Transforms/Scalar/LICM.cpp
Warning:line 1165, column 33
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name LICM.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Transforms/Scalar -I include -I /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-01-16-232930-107970-1 -x c++ /build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Transforms/Scalar/LICM.cpp

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/lib/Transforms/Scalar/LICM.cpp

1//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs loop invariant code motion, attempting to remove as much
10// code from the body of a loop as possible. It does this by either hoisting
11// code into the preheader block, or by sinking code to the exit blocks if it is
12// safe. This pass also promotes must-aliased memory locations in the loop to
13// live in registers, thus hoisting and sinking "invariant" loads and stores.
14//
15// Hoisting operations out of loops is a canonicalization transform. It
16// enables and simplifies subsequent optimizations in the middle-end.
17// Rematerialization of hoisted instructions to reduce register pressure is the
18// responsibility of the back-end, which has more accurate information about
19// register pressure and also handles other optimizations than LICM that
20// increase live-ranges.
21//
22// This pass uses alias analysis for two purposes:
23//
24// 1. Moving loop invariant loads and calls out of loops. If we can determine
25// that a load or call inside of a loop never aliases anything stored to,
26// we can hoist it or sink it like any other instruction.
27// 2. Scalar Promotion of Memory - If there is a store instruction inside of
28// the loop, we try to move the store to happen AFTER the loop instead of
29// inside of the loop. This can only happen if a few conditions are true:
30// A. The pointer stored through is loop invariant
31// B. There are no stores or loads in the loop which _may_ alias the
32// pointer. There are no calls in the loop which mod/ref the pointer.
33// If these conditions are true, we can promote the loads and stores in the
34// loop of the pointer to use a temporary alloca'd variable. We then use
35// the SSAUpdater to construct the appropriate SSA form for the value.
36//
37//===----------------------------------------------------------------------===//
38
39#include "llvm/Transforms/Scalar/LICM.h"
40#include "llvm/ADT/SetOperations.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/Analysis/AliasAnalysis.h"
43#include "llvm/Analysis/AliasSetTracker.h"
44#include "llvm/Analysis/BasicAliasAnalysis.h"
45#include "llvm/Analysis/BlockFrequencyInfo.h"
46#include "llvm/Analysis/CaptureTracking.h"
47#include "llvm/Analysis/ConstantFolding.h"
48#include "llvm/Analysis/GlobalsModRef.h"
49#include "llvm/Analysis/GuardUtils.h"
50#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
51#include "llvm/Analysis/Loads.h"
52#include "llvm/Analysis/LoopInfo.h"
53#include "llvm/Analysis/LoopIterator.h"
54#include "llvm/Analysis/LoopPass.h"
55#include "llvm/Analysis/MemoryBuiltins.h"
56#include "llvm/Analysis/MemorySSA.h"
57#include "llvm/Analysis/MemorySSAUpdater.h"
58#include "llvm/Analysis/MustExecute.h"
59#include "llvm/Analysis/OptimizationRemarkEmitter.h"
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
62#include "llvm/Analysis/TargetLibraryInfo.h"
63#include "llvm/Analysis/ValueTracking.h"
64#include "llvm/IR/CFG.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DerivedTypes.h"
69#include "llvm/IR/Dominators.h"
70#include "llvm/IR/Instructions.h"
71#include "llvm/IR/IntrinsicInst.h"
72#include "llvm/IR/LLVMContext.h"
73#include "llvm/IR/Metadata.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/PredIteratorCache.h"
76#include "llvm/InitializePasses.h"
77#include "llvm/Support/CommandLine.h"
78#include "llvm/Support/Debug.h"
79#include "llvm/Support/raw_ostream.h"
80#include "llvm/Transforms/Scalar.h"
81#include "llvm/Transforms/Scalar/LoopPassManager.h"
82#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
83#include "llvm/Transforms/Utils/BasicBlockUtils.h"
84#include "llvm/Transforms/Utils/Local.h"
85#include "llvm/Transforms/Utils/LoopUtils.h"
86#include "llvm/Transforms/Utils/SSAUpdater.h"
87#include <algorithm>
88#include <utility>
89using namespace llvm;
90
91#define DEBUG_TYPE"licm" "licm"
92
93STATISTIC(NumCreatedBlocks, "Number of blocks created")static llvm::Statistic NumCreatedBlocks = {"licm", "NumCreatedBlocks"
, "Number of blocks created"}
;
94STATISTIC(NumClonedBranches, "Number of branches cloned")static llvm::Statistic NumClonedBranches = {"licm", "NumClonedBranches"
, "Number of branches cloned"}
;
95STATISTIC(NumSunk, "Number of instructions sunk out of loop")static llvm::Statistic NumSunk = {"licm", "NumSunk", "Number of instructions sunk out of loop"
}
;
96STATISTIC(NumHoisted, "Number of instructions hoisted out of loop")static llvm::Statistic NumHoisted = {"licm", "NumHoisted", "Number of instructions hoisted out of loop"
}
;
97STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk")static llvm::Statistic NumMovedLoads = {"licm", "NumMovedLoads"
, "Number of load insts hoisted or sunk"}
;
98STATISTIC(NumMovedCalls, "Number of call insts hoisted or sunk")static llvm::Statistic NumMovedCalls = {"licm", "NumMovedCalls"
, "Number of call insts hoisted or sunk"}
;
99STATISTIC(NumPromoted, "Number of memory locations promoted to registers")static llvm::Statistic NumPromoted = {"licm", "NumPromoted", "Number of memory locations promoted to registers"
}
;
100
101/// Memory promotion is enabled by default.
102static cl::opt<bool>
103 DisablePromotion("disable-licm-promotion", cl::Hidden, cl::init(false),
104 cl::desc("Disable memory promotion in LICM pass"));
105
106static cl::opt<bool> ControlFlowHoisting(
107 "licm-control-flow-hoisting", cl::Hidden, cl::init(false),
108 cl::desc("Enable control flow (and PHI) hoisting in LICM"));
109
110static cl::opt<uint32_t> MaxNumUsesTraversed(
111 "licm-max-num-uses-traversed", cl::Hidden, cl::init(8),
112 cl::desc("Max num uses visited for identifying load "
113 "invariance in loop using invariant start (default = 8)"));
114
115// Experimental option to allow imprecision in LICM in pathological cases, in
116// exchange for faster compile. This is to be removed if MemorySSA starts to
117// address the same issue. This flag applies only when LICM uses MemorySSA
118// instead on AliasSetTracker. LICM calls MemorySSAWalker's
119// getClobberingMemoryAccess, up to the value of the Cap, getting perfect
120// accuracy. Afterwards, LICM will call into MemorySSA's getDefiningAccess,
121// which may not be precise, since optimizeUses is capped. The result is
122// correct, but we may not get as "far up" as possible to get which access is
123// clobbering the one queried.
124cl::opt<unsigned> llvm::SetLicmMssaOptCap(
125 "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
126 cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
127 "for faster compile. Caps the MemorySSA clobbering calls."));
128
129// Experimentally, memory promotion carries less importance than sinking and
130// hoisting. Limit when we do promotion when using MemorySSA, in order to save
131// compile time.
132cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
133 "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
134 cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
135 "effect. When MSSA in LICM is enabled, then this is the maximum "
136 "number of accesses allowed to be present in a loop in order to "
137 "enable memory promotion."));
138
139static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
140static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
141 const LoopSafetyInfo *SafetyInfo,
142 TargetTransformInfo *TTI, bool &FreeInLoop,
143 bool LoopNestMode);
144static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
145 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
146 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
147 OptimizationRemarkEmitter *ORE);
148static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
149 BlockFrequencyInfo *BFI, const Loop *CurLoop,
150 ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
151 OptimizationRemarkEmitter *ORE);
152static bool isSafeToExecuteUnconditionally(Instruction &Inst,
153 const DominatorTree *DT,
154 const TargetLibraryInfo *TLI,
155 const Loop *CurLoop,
156 const LoopSafetyInfo *SafetyInfo,
157 OptimizationRemarkEmitter *ORE,
158 const Instruction *CtxI = nullptr);
159static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
160 AliasSetTracker *CurAST, Loop *CurLoop,
161 AAResults *AA);
162static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
163 Loop *CurLoop, Instruction &I,
164 SinkAndHoistLICMFlags &Flags);
165static bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
166 MemoryUse &MU);
167static Instruction *cloneInstructionInExitBlock(
168 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
169 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
170
171static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
172 MemorySSAUpdater *MSSAU);
173
174static void moveInstructionBefore(Instruction &I, Instruction &Dest,
175 ICFLoopSafetyInfo &SafetyInfo,
176 MemorySSAUpdater *MSSAU, ScalarEvolution *SE);
177
178static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
179 function_ref<void(Instruction *)> Fn);
180static SmallVector<SmallSetVector<Value *, 8>, 0>
181collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L);
182
183namespace {
184struct LoopInvariantCodeMotion {
185 bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
186 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
187 TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
188 OptimizationRemarkEmitter *ORE, bool LoopNestMode = false);
189
190 LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
191 unsigned LicmMssaNoAccForPromotionCap)
192 : LicmMssaOptCap(LicmMssaOptCap),
193 LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}
194
195private:
196 unsigned LicmMssaOptCap;
197 unsigned LicmMssaNoAccForPromotionCap;
198};
199
200struct LegacyLICMPass : public LoopPass {
201 static char ID; // Pass identification, replacement for typeid
202 LegacyLICMPass(
203 unsigned LicmMssaOptCap = SetLicmMssaOptCap,
204 unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
205 : LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
206 initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
207 }
208
209 bool runOnLoop(Loop *L, LPPassManager &LPM) override {
210 if (skipLoop(L))
211 return false;
212
213 LLVM_DEBUG(dbgs() << "Perform LICM on Loop with header at block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
<< L->getHeader()->getNameOrAsOperand() <<
"\n"; } } while (false)
214 << L->getHeader()->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
<< L->getHeader()->getNameOrAsOperand() <<
"\n"; } } while (false)
;
215
216 auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
217 MemorySSA *MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
218 bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
219 BlockFrequencyInfo *BFI =
220 hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
221 : nullptr;
222 // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
223 // pass. Function analyses need to be preserved across loop transformations
224 // but ORE cannot be preserved (see comment before the pass definition).
225 OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
226 return LICM.runOnLoop(
227 L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
228 &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
229 &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
230 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
231 *L->getHeader()->getParent()),
232 &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
233 *L->getHeader()->getParent()),
234 SE ? &SE->getSE() : nullptr, MSSA, &ORE);
235 }
236
237 /// This transformation requires natural loop information & requires that
238 /// loop preheaders be inserted into the CFG...
239 ///
240 void getAnalysisUsage(AnalysisUsage &AU) const override {
241 AU.addPreserved<DominatorTreeWrapperPass>();
242 AU.addPreserved<LoopInfoWrapperPass>();
243 AU.addRequired<TargetLibraryInfoWrapperPass>();
244 AU.addRequired<MemorySSAWrapperPass>();
245 AU.addPreserved<MemorySSAWrapperPass>();
246 AU.addRequired<TargetTransformInfoWrapperPass>();
247 getLoopAnalysisUsage(AU);
248 LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
249 AU.addPreserved<LazyBlockFrequencyInfoPass>();
250 AU.addPreserved<LazyBranchProbabilityInfoPass>();
251 }
252
253private:
254 LoopInvariantCodeMotion LICM;
255};
256} // namespace
257
258PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
259 LoopStandardAnalysisResults &AR, LPMUpdater &) {
260 if (!AR.MSSA)
261 report_fatal_error("LICM requires MemorySSA (loop-mssa)");
262
263 // For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
264 // pass. Function analyses need to be preserved across loop transformations
265 // but ORE cannot be preserved (see comment before the pass definition).
266 OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
267
268 LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
269 if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
270 &AR.SE, AR.MSSA, &ORE))
271 return PreservedAnalyses::all();
272
273 auto PA = getLoopPassPreservedAnalyses();
274
275 PA.preserve<DominatorTreeAnalysis>();
276 PA.preserve<LoopAnalysis>();
277 PA.preserve<MemorySSAAnalysis>();
278
279 return PA;
280}
281
282PreservedAnalyses LNICMPass::run(LoopNest &LN, LoopAnalysisManager &AM,
283 LoopStandardAnalysisResults &AR,
284 LPMUpdater &) {
285 if (!AR.MSSA)
286 report_fatal_error("LNICM requires MemorySSA (loop-mssa)");
287
288 // For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
289 // pass. Function analyses need to be preserved across loop transformations
290 // but ORE cannot be preserved (see comment before the pass definition).
291 OptimizationRemarkEmitter ORE(LN.getParent());
292
293 LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
294
295 Loop &OutermostLoop = LN.getOutermostLoop();
296 bool Changed = LICM.runOnLoop(&OutermostLoop, &AR.AA, &AR.LI, &AR.DT, AR.BFI,
297 &AR.TLI, &AR.TTI, &AR.SE, AR.MSSA, &ORE, true);
298
299 if (!Changed)
300 return PreservedAnalyses::all();
301
302 auto PA = getLoopPassPreservedAnalyses();
303
304 PA.preserve<DominatorTreeAnalysis>();
305 PA.preserve<LoopAnalysis>();
306 PA.preserve<MemorySSAAnalysis>();
307
308 return PA;
309}
310
311char LegacyLICMPass::ID = 0;
312INITIALIZE_PASS_BEGIN(LegacyLICMPass, "licm", "Loop Invariant Code Motion",static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
313 false, false)static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
314INITIALIZE_PASS_DEPENDENCY(LoopPass)initializeLoopPassPass(Registry);
315INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
316INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
317INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry);
318INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)initializeLazyBFIPassPass(Registry);
319INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
320 false)PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
321
322Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
323Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
324 unsigned LicmMssaNoAccForPromotionCap) {
325 return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
326}
327
328llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(bool IsSink, Loop *L,
329 MemorySSA *MSSA)
330 : SinkAndHoistLICMFlags(SetLicmMssaOptCap, SetLicmMssaNoAccForPromotionCap,
331 IsSink, L, MSSA) {}
332
333llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(
334 unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
335 Loop *L, MemorySSA *MSSA)
336 : LicmMssaOptCap(LicmMssaOptCap),
337 LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap),
338 IsSink(IsSink) {
339 assert(((L != nullptr) == (MSSA != nullptr)) &&(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 340, __extension__ __PRETTY_FUNCTION__
))
340 "Unexpected values for SinkAndHoistLICMFlags")(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 340, __extension__ __PRETTY_FUNCTION__
))
;
341 if (!MSSA)
342 return;
343
344 unsigned AccessCapCount = 0;
345 for (auto *BB : L->getBlocks())
346 if (const auto *Accesses = MSSA->getBlockAccesses(BB))
347 for (const auto &MA : *Accesses) {
348 (void)MA;
349 ++AccessCapCount;
350 if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
351 NoOfMemAccTooLarge = true;
352 return;
353 }
354 }
355}
356
357/// Hoist expressions out of the specified loop. Note, alias info for inner
358/// loop is not preserved so it is not a good idea to run LICM multiple
359/// times on one loop.
360bool LoopInvariantCodeMotion::runOnLoop(
361 Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
362 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
363 ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE,
364 bool LoopNestMode) {
365 bool Changed = false;
366
367 assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop is not in LCSSA form."
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop is not in LCSSA form.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 367, __extension__ __PRETTY_FUNCTION__
))
;
368
369 // If this loop has metadata indicating that LICM is not to be performed then
370 // just exit.
371 if (hasDisableLICMTransformsHint(L)) {
372 return false;
373 }
374
375 // Don't sink stores from loops with coroutine suspend instructions.
376 // LICM would sink instructions into the default destination of
377 // the coroutine switch. The default destination of the switch is to
378 // handle the case where the coroutine is suspended, by which point the
379 // coroutine frame may have been destroyed. No instruction can be sunk there.
380 // FIXME: This would unfortunately hurt the performance of coroutines, however
381 // there is currently no general solution for this. Similar issues could also
382 // potentially happen in other passes where instructions are being moved
383 // across that edge.
384 bool HasCoroSuspendInst = llvm::any_of(L->getBlocks(), [](BasicBlock *BB) {
385 return llvm::any_of(*BB, [](Instruction &I) {
386 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
387 return II && II->getIntrinsicID() == Intrinsic::coro_suspend;
388 });
389 });
390
391 MemorySSAUpdater MSSAU(MSSA);
392 SinkAndHoistLICMFlags Flags(LicmMssaOptCap, LicmMssaNoAccForPromotionCap,
393 /*IsSink=*/true, L, MSSA);
394
395 // Get the preheader block to move instructions into...
396 BasicBlock *Preheader = L->getLoopPreheader();
397
398 // Compute loop safety information.
399 ICFLoopSafetyInfo SafetyInfo;
400 SafetyInfo.computeLoopSafetyInfo(L);
401
402 // We want to visit all of the instructions in this loop... that are not parts
403 // of our subloops (they have already had their invariants hoisted out of
404 // their loop, into this loop, so there is no need to process the BODIES of
405 // the subloops).
406 //
407 // Traverse the body of the loop in depth first order on the dominator tree so
408 // that we are guaranteed to see definitions before we see uses. This allows
409 // us to sink instructions in one pass, without iteration. After sinking
410 // instructions, we perform another pass to hoist them out of the loop.
411 if (L->hasDedicatedExits())
412 Changed |= LoopNestMode
413 ? sinkRegionForLoopNest(DT->getNode(L->getHeader()), AA, LI,
414 DT, BFI, TLI, TTI, L, &MSSAU,
415 &SafetyInfo, Flags, ORE)
416 : sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI,
417 TLI, TTI, L, &MSSAU, &SafetyInfo, Flags, ORE);
418 Flags.setIsSink(false);
419 if (Preheader)
420 Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, L,
421 &MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode);
422
423 // Now that all loop invariants have been removed from the loop, promote any
424 // memory references to scalars that we can.
425 // Don't sink stores from loops without dedicated block exits. Exits
426 // containing indirect branches are not transformed by loop simplify,
427 // make sure we catch that. An additional load may be generated in the
428 // preheader for SSA updater, so also avoid sinking when no preheader
429 // is available.
430 if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
431 !Flags.tooManyMemoryAccesses() && !HasCoroSuspendInst) {
432 // Figure out the loop exits and their insertion points
433 SmallVector<BasicBlock *, 8> ExitBlocks;
434 L->getUniqueExitBlocks(ExitBlocks);
435
436 // We can't insert into a catchswitch.
437 bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
438 return isa<CatchSwitchInst>(Exit->getTerminator());
439 });
440
441 if (!HasCatchSwitch) {
442 SmallVector<Instruction *, 8> InsertPts;
443 SmallVector<MemoryAccess *, 8> MSSAInsertPts;
444 InsertPts.reserve(ExitBlocks.size());
445 MSSAInsertPts.reserve(ExitBlocks.size());
446 for (BasicBlock *ExitBlock : ExitBlocks) {
447 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
448 MSSAInsertPts.push_back(nullptr);
449 }
450
451 PredIteratorCache PIC;
452
453 // Promoting one set of accesses may make the pointers for another set
454 // loop invariant, so run this in a loop (with the MaybePromotable set
455 // decreasing in size over time).
456 bool Promoted = false;
457 bool LocalPromoted;
458 do {
459 LocalPromoted = false;
460 for (const SmallSetVector<Value *, 8> &PointerMustAliases :
461 collectPromotionCandidates(MSSA, AA, L)) {
462 LocalPromoted |= promoteLoopAccessesToScalars(
463 PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC,
464 LI, DT, TLI, L, &MSSAU, &SafetyInfo, ORE);
465 }
466 Promoted |= LocalPromoted;
467 } while (LocalPromoted);
468
469 // Once we have promoted values across the loop body we have to
470 // recursively reform LCSSA as any nested loop may now have values defined
471 // within the loop used in the outer loop.
472 // FIXME: This is really heavy handed. It would be a bit better to use an
473 // SSAUpdater strategy during promotion that was LCSSA aware and reformed
474 // it as it went.
475 if (Promoted)
476 formLCSSARecursively(*L, *DT, LI, SE);
477
478 Changed |= Promoted;
479 }
480 }
481
482 // Check that neither this loop nor its parent have had LCSSA broken. LICM is
483 // specifically moving instructions across the loop boundary and so it is
484 // especially in need of basic functional correctness checking here.
485 assert(L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop not left in LCSSA form after LICM!\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 485, __extension__ __PRETTY_FUNCTION__
))
;
486 assert((L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) &&(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 487, __extension__ __PRETTY_FUNCTION__
))
487 "Parent loop not left in LCSSA form after LICM!")(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 487, __extension__ __PRETTY_FUNCTION__
))
;
488
489 if (VerifyMemorySSA)
490 MSSA->verifyMemorySSA();
491
492 if (Changed && SE)
493 SE->forgetLoopDispositions(L);
494 return Changed;
495}
496
497/// Walk the specified region of the CFG (defined by all blocks dominated by
498/// the specified block, and that are in the current loop) in reverse depth
499/// first order w.r.t the DominatorTree. This allows us to visit uses before
500/// definitions, allowing us to sink a loop body in one pass without iteration.
501///
502bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
503 DominatorTree *DT, BlockFrequencyInfo *BFI,
504 TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
505 Loop *CurLoop, MemorySSAUpdater *MSSAU,
506 ICFLoopSafetyInfo *SafetyInfo,
507 SinkAndHoistLICMFlags &Flags,
508 OptimizationRemarkEmitter *ORE, Loop *OutermostLoop) {
509
510 // Verify inputs.
511 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to sinkRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 513, __extension__ __PRETTY_FUNCTION__
))
512 CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to sinkRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 513, __extension__ __PRETTY_FUNCTION__
))
513 "Unexpected input to sinkRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to sinkRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 513, __extension__ __PRETTY_FUNCTION__
))
;
514
515 // We want to visit children before parents. We will enque all the parents
516 // before their children in the worklist and process the worklist in reverse
517 // order.
518 SmallVector<DomTreeNode *, 16> Worklist = collectChildrenInLoop(N, CurLoop);
519
520 bool Changed = false;
521 for (DomTreeNode *DTN : reverse(Worklist)) {
522 BasicBlock *BB = DTN->getBlock();
523 // Only need to process the contents of this block if it is not part of a
524 // subloop (which would already have been processed).
525 if (inSubLoop(BB, CurLoop, LI))
526 continue;
527
528 for (BasicBlock::iterator II = BB->end(); II != BB->begin();) {
529 Instruction &I = *--II;
530
531 // The instruction is not used in the loop if it is dead. In this case,
532 // we just delete it instead of sinking it.
533 if (isInstructionTriviallyDead(&I, TLI)) {
534 LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM deleting dead inst: " <<
I << '\n'; } } while (false)
;
535 salvageKnowledge(&I);
536 salvageDebugInfo(I);
537 ++II;
538 eraseInstruction(I, *SafetyInfo, MSSAU);
539 Changed = true;
540 continue;
541 }
542
543 // Check to see if we can sink this instruction to the exit blocks
544 // of the loop. We can do this if the all users of the instruction are
545 // outside of the loop. In this case, it doesn't even matter if the
546 // operands of the instruction are loop invariant.
547 //
548 bool FreeInLoop = false;
549 bool LoopNestMode = OutermostLoop != nullptr;
550 if (!I.mayHaveSideEffects() &&
551 isNotUsedOrFreeInLoop(I, LoopNestMode ? OutermostLoop : CurLoop,
552 SafetyInfo, TTI, FreeInLoop, LoopNestMode) &&
553 canSinkOrHoistInst(I, AA, DT, CurLoop, /*CurAST*/nullptr, MSSAU, true,
554 &Flags, ORE)) {
555 if (sink(I, LI, DT, BFI, CurLoop, SafetyInfo, MSSAU, ORE)) {
556 if (!FreeInLoop) {
557 ++II;
558 salvageDebugInfo(I);
559 eraseInstruction(I, *SafetyInfo, MSSAU);
560 }
561 Changed = true;
562 }
563 }
564 }
565 }
566 if (VerifyMemorySSA)
567 MSSAU->getMemorySSA()->verifyMemorySSA();
568 return Changed;
569}
570
571bool llvm::sinkRegionForLoopNest(
572 DomTreeNode *N, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
573 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
574 Loop *CurLoop, MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
575 SinkAndHoistLICMFlags &Flags, OptimizationRemarkEmitter *ORE) {
576
577 bool Changed = false;
578 SmallPriorityWorklist<Loop *, 4> Worklist;
579 Worklist.insert(CurLoop);
580 appendLoopsToWorklist(*CurLoop, Worklist);
581 while (!Worklist.empty()) {
582 Loop *L = Worklist.pop_back_val();
583 Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI,
584 TTI, L, MSSAU, SafetyInfo, Flags, ORE, CurLoop);
585 }
586 return Changed;
587}
588
589namespace {
590// This is a helper class for hoistRegion to make it able to hoist control flow
591// in order to be able to hoist phis. The way this works is that we initially
592// start hoisting to the loop preheader, and when we see a loop invariant branch
593// we make note of this. When we then come to hoist an instruction that's
594// conditional on such a branch we duplicate the branch and the relevant control
595// flow, then hoist the instruction into the block corresponding to its original
596// block in the duplicated control flow.
597class ControlFlowHoister {
598private:
599 // Information about the loop we are hoisting from
600 LoopInfo *LI;
601 DominatorTree *DT;
602 Loop *CurLoop;
603 MemorySSAUpdater *MSSAU;
604
605 // A map of blocks in the loop to the block their instructions will be hoisted
606 // to.
607 DenseMap<BasicBlock *, BasicBlock *> HoistDestinationMap;
608
609 // The branches that we can hoist, mapped to the block that marks a
610 // convergence point of their control flow.
611 DenseMap<BranchInst *, BasicBlock *> HoistableBranches;
612
613public:
614 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
615 MemorySSAUpdater *MSSAU)
616 : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}
617
618 void registerPossiblyHoistableBranch(BranchInst *BI) {
619 // We can only hoist conditional branches with loop invariant operands.
620 if (!ControlFlowHoisting || !BI->isConditional() ||
621 !CurLoop->hasLoopInvariantOperands(BI))
622 return;
623
624 // The branch destinations need to be in the loop, and we don't gain
625 // anything by duplicating conditional branches with duplicate successors,
626 // as it's essentially the same as an unconditional branch.
627 BasicBlock *TrueDest = BI->getSuccessor(0);
628 BasicBlock *FalseDest = BI->getSuccessor(1);
629 if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
630 TrueDest == FalseDest)
631 return;
632
633 // We can hoist BI if one branch destination is the successor of the other,
634 // or both have common successor which we check by seeing if the
635 // intersection of their successors is non-empty.
636 // TODO: This could be expanded to allowing branches where both ends
637 // eventually converge to a single block.
638 SmallPtrSet<BasicBlock *, 4> TrueDestSucc, FalseDestSucc;
639 TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
640 FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
641 BasicBlock *CommonSucc = nullptr;
642 if (TrueDestSucc.count(FalseDest)) {
643 CommonSucc = FalseDest;
644 } else if (FalseDestSucc.count(TrueDest)) {
645 CommonSucc = TrueDest;
646 } else {
647 set_intersect(TrueDestSucc, FalseDestSucc);
648 // If there's one common successor use that.
649 if (TrueDestSucc.size() == 1)
650 CommonSucc = *TrueDestSucc.begin();
651 // If there's more than one pick whichever appears first in the block list
652 // (we can't use the value returned by TrueDestSucc.begin() as it's
653 // unpredicatable which element gets returned).
654 else if (!TrueDestSucc.empty()) {
655 Function *F = TrueDest->getParent();
656 auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
657 auto It = llvm::find_if(*F, IsSucc);
658 assert(It != F->end() && "Could not find successor in function")(static_cast <bool> (It != F->end() && "Could not find successor in function"
) ? void (0) : __assert_fail ("It != F->end() && \"Could not find successor in function\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 658, __extension__ __PRETTY_FUNCTION__
))
;
659 CommonSucc = &*It;
660 }
661 }
662 // The common successor has to be dominated by the branch, as otherwise
663 // there will be some other path to the successor that will not be
664 // controlled by this branch so any phi we hoist would be controlled by the
665 // wrong condition. This also takes care of avoiding hoisting of loop back
666 // edges.
667 // TODO: In some cases this could be relaxed if the successor is dominated
668 // by another block that's been hoisted and we can guarantee that the
669 // control flow has been replicated exactly.
670 if (CommonSucc && DT->dominates(BI, CommonSucc))
671 HoistableBranches[BI] = CommonSucc;
672 }
673
674 bool canHoistPHI(PHINode *PN) {
675 // The phi must have loop invariant operands.
676 if (!ControlFlowHoisting || !CurLoop->hasLoopInvariantOperands(PN))
677 return false;
678 // We can hoist phis if the block they are in is the target of hoistable
679 // branches which cover all of the predecessors of the block.
680 SmallPtrSet<BasicBlock *, 8> PredecessorBlocks;
681 BasicBlock *BB = PN->getParent();
682 for (BasicBlock *PredBB : predecessors(BB))
683 PredecessorBlocks.insert(PredBB);
684 // If we have less predecessor blocks than predecessors then the phi will
685 // have more than one incoming value for the same block which we can't
686 // handle.
687 // TODO: This could be handled be erasing some of the duplicate incoming
688 // values.
689 if (PredecessorBlocks.size() != pred_size(BB))
690 return false;
691 for (auto &Pair : HoistableBranches) {
692 if (Pair.second == BB) {
693 // Which blocks are predecessors via this branch depends on if the
694 // branch is triangle-like or diamond-like.
695 if (Pair.first->getSuccessor(0) == BB) {
696 PredecessorBlocks.erase(Pair.first->getParent());
697 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
698 } else if (Pair.first->getSuccessor(1) == BB) {
699 PredecessorBlocks.erase(Pair.first->getParent());
700 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
701 } else {
702 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
703 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
704 }
705 }
706 }
707 // PredecessorBlocks will now be empty if for every predecessor of BB we
708 // found a hoistable branch source.
709 return PredecessorBlocks.empty();
710 }
711
712 BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
713 if (!ControlFlowHoisting)
714 return CurLoop->getLoopPreheader();
715 // If BB has already been hoisted, return that
716 if (HoistDestinationMap.count(BB))
717 return HoistDestinationMap[BB];
718
719 // Check if this block is conditional based on a pending branch
720 auto HasBBAsSuccessor =
721 [&](DenseMap<BranchInst *, BasicBlock *>::value_type &Pair) {
722 return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
723 Pair.first->getSuccessor(1) == BB);
724 };
725 auto It = llvm::find_if(HoistableBranches, HasBBAsSuccessor);
726
727 // If not involved in a pending branch, hoist to preheader
728 BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
729 if (It == HoistableBranches.end()) {
730 LLVM_DEBUG(dbgs() << "LICM using "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
731 << InitialPreheader->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
732 << " as hoist destination for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
733 << BB->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
;
734 HoistDestinationMap[BB] = InitialPreheader;
735 return InitialPreheader;
736 }
737 BranchInst *BI = It->first;
738 assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 740, __extension__ __PRETTY_FUNCTION__
))
739 HoistableBranches.end() &&(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 740, __extension__ __PRETTY_FUNCTION__
))
740 "BB is expected to be the target of at most one branch")(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 740, __extension__ __PRETTY_FUNCTION__
))
;
741
742 LLVMContext &C = BB->getContext();
743 BasicBlock *TrueDest = BI->getSuccessor(0);
744 BasicBlock *FalseDest = BI->getSuccessor(1);
745 BasicBlock *CommonSucc = HoistableBranches[BI];
746 BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());
747
748 // Create hoisted versions of blocks that currently don't have them
749 auto CreateHoistedBlock = [&](BasicBlock *Orig) {
750 if (HoistDestinationMap.count(Orig))
751 return HoistDestinationMap[Orig];
752 BasicBlock *New =
753 BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
754 HoistDestinationMap[Orig] = New;
755 DT->addNewBlock(New, HoistTarget);
756 if (CurLoop->getParentLoop())
757 CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
758 ++NumCreatedBlocks;
759 LLVM_DEBUG(dbgs() << "LICM created " << New->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
760 << " as hoist destination for " << Orig->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
761 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
;
762 return New;
763 };
764 BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
765 BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
766 BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);
767
768 // Link up these blocks with branches.
769 if (!HoistCommonSucc->getTerminator()) {
770 // The new common successor we've generated will branch to whatever that
771 // hoist target branched to.
772 BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
773 assert(TargetSucc && "Expected hoist target to have a single successor")(static_cast <bool> (TargetSucc && "Expected hoist target to have a single successor"
) ? void (0) : __assert_fail ("TargetSucc && \"Expected hoist target to have a single successor\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 773, __extension__ __PRETTY_FUNCTION__
))
;
774 HoistCommonSucc->moveBefore(TargetSucc);
775 BranchInst::Create(TargetSucc, HoistCommonSucc);
776 }
777 if (!HoistTrueDest->getTerminator()) {
778 HoistTrueDest->moveBefore(HoistCommonSucc);
779 BranchInst::Create(HoistCommonSucc, HoistTrueDest);
780 }
781 if (!HoistFalseDest->getTerminator()) {
782 HoistFalseDest->moveBefore(HoistCommonSucc);
783 BranchInst::Create(HoistCommonSucc, HoistFalseDest);
784 }
785
786 // If BI is being cloned to what was originally the preheader then
787 // HoistCommonSucc will now be the new preheader.
788 if (HoistTarget == InitialPreheader) {
789 // Phis in the loop header now need to use the new preheader.
790 InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
791 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
792 HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
793 // The new preheader dominates the loop header.
794 DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
795 DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
796 DT->changeImmediateDominator(HeaderNode, PreheaderNode);
797 // The preheader hoist destination is now the new preheader, with the
798 // exception of the hoist destination of this branch.
799 for (auto &Pair : HoistDestinationMap)
800 if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
801 Pair.second = HoistCommonSucc;
802 }
803
804 // Now finally clone BI.
805 ReplaceInstWithInst(
806 HoistTarget->getTerminator(),
807 BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
808 ++NumClonedBranches;
809
810 assert(CurLoop->getLoopPreheader() &&(static_cast <bool> (CurLoop->getLoopPreheader() &&
"Hoisting blocks should not have destroyed preheader") ? void
(0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 811, __extension__ __PRETTY_FUNCTION__
))
811 "Hoisting blocks should not have destroyed preheader")(static_cast <bool> (CurLoop->getLoopPreheader() &&
"Hoisting blocks should not have destroyed preheader") ? void
(0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 811, __extension__ __PRETTY_FUNCTION__
))
;
812 return HoistDestinationMap[BB];
813 }
814};
815} // namespace
816
817/// Walk the specified region of the CFG (defined by all blocks dominated by
818/// the specified block, and that are in the current loop) in depth first
819/// order w.r.t the DominatorTree. This allows us to visit definitions before
820/// uses, allowing us to hoist a loop body in one pass without iteration.
821///
822bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
823 DominatorTree *DT, BlockFrequencyInfo *BFI,
824 TargetLibraryInfo *TLI, Loop *CurLoop,
825 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
826 ICFLoopSafetyInfo *SafetyInfo,
827 SinkAndHoistLICMFlags &Flags,
828 OptimizationRemarkEmitter *ORE, bool LoopNestMode) {
829 // Verify inputs.
830 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to hoistRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 832, __extension__ __PRETTY_FUNCTION__
))
831 CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to hoistRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 832, __extension__ __PRETTY_FUNCTION__
))
832 "Unexpected input to hoistRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to hoistRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 832, __extension__ __PRETTY_FUNCTION__
))
;
833
834 ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);
835
836 // Keep track of instructions that have been hoisted, as they may need to be
837 // re-hoisted if they end up not dominating all of their uses.
838 SmallVector<Instruction *, 16> HoistedInstructions;
839
840 // For PHI hoisting to work we need to hoist blocks before their successors.
841 // We can do this by iterating through the blocks in the loop in reverse
842 // post-order.
843 LoopBlocksRPO Worklist(CurLoop);
844 Worklist.perform(LI);
845 bool Changed = false;
846 for (BasicBlock *BB : Worklist) {
847 // Only need to process the contents of this block if it is not part of a
848 // subloop (which would already have been processed).
849 if (!LoopNestMode && inSubLoop(BB, CurLoop, LI))
850 continue;
851
852 for (Instruction &I : llvm::make_early_inc_range(*BB)) {
853 // Try constant folding this instruction. If all the operands are
854 // constants, it is technically hoistable, but it would be better to
855 // just fold it.
856 if (Constant *C = ConstantFoldInstruction(
857 &I, I.getModule()->getDataLayout(), TLI)) {
858 LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << " --> " << *Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
" --> " << *C << '\n'; } } while (false)
859 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
" --> " << *C << '\n'; } } while (false)
;
860 // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
861 I.replaceAllUsesWith(C);
862 if (isInstructionTriviallyDead(&I, TLI))
863 eraseInstruction(I, *SafetyInfo, MSSAU);
864 Changed = true;
865 continue;
866 }
867
868 // Try hoisting the instruction out to the preheader. We can only do
869 // this if all of the operands of the instruction are loop invariant and
870 // if it is safe to hoist the instruction. We also check block frequency
871 // to make sure instruction only gets hoisted into colder blocks.
872 // TODO: It may be safe to hoist if we are hoisting to a conditional block
873 // and we have accurately duplicated the control flow from the loop header
874 // to that block.
875 if (CurLoop->hasLoopInvariantOperands(&I) &&
876 canSinkOrHoistInst(I, AA, DT, CurLoop, /*CurAST*/ nullptr, MSSAU,
877 true, &Flags, ORE) &&
878 isSafeToExecuteUnconditionally(
879 I, DT, TLI, CurLoop, SafetyInfo, ORE,
880 CurLoop->getLoopPreheader()->getTerminator())) {
881 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
882 MSSAU, SE, ORE);
883 HoistedInstructions.push_back(&I);
884 Changed = true;
885 continue;
886 }
887
888 // Attempt to remove floating point division out of the loop by
889 // converting it to a reciprocal multiplication.
890 if (I.getOpcode() == Instruction::FDiv && I.hasAllowReciprocal() &&
891 CurLoop->isLoopInvariant(I.getOperand(1))) {
892 auto Divisor = I.getOperand(1);
893 auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
894 auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
895 ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
896 SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
897 ReciprocalDivisor->insertBefore(&I);
898
899 auto Product =
900 BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
901 Product->setFastMathFlags(I.getFastMathFlags());
902 SafetyInfo->insertInstructionTo(Product, I.getParent());
903 Product->insertAfter(&I);
904 I.replaceAllUsesWith(Product);
905 eraseInstruction(I, *SafetyInfo, MSSAU);
906
907 hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
908 SafetyInfo, MSSAU, SE, ORE);
909 HoistedInstructions.push_back(ReciprocalDivisor);
910 Changed = true;
911 continue;
912 }
913
914 auto IsInvariantStart = [&](Instruction &I) {
915 using namespace PatternMatch;
916 return I.use_empty() &&
917 match(&I, m_Intrinsic<Intrinsic::invariant_start>());
918 };
919 auto MustExecuteWithoutWritesBefore = [&](Instruction &I) {
920 return SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
921 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop);
922 };
923 if ((IsInvariantStart(I) || isGuard(&I)) &&
924 CurLoop->hasLoopInvariantOperands(&I) &&
925 MustExecuteWithoutWritesBefore(I)) {
926 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
927 MSSAU, SE, ORE);
928 HoistedInstructions.push_back(&I);
929 Changed = true;
930 continue;
931 }
932
933 if (PHINode *PN = dyn_cast<PHINode>(&I)) {
934 if (CFH.canHoistPHI(PN)) {
935 // Redirect incoming blocks first to ensure that we create hoisted
936 // versions of those blocks before we hoist the phi.
937 for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
938 PN->setIncomingBlock(
939 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
940 hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
941 MSSAU, SE, ORE);
942 assert(DT->dominates(PN, BB) && "Conditional PHIs not expected")(static_cast <bool> (DT->dominates(PN, BB) &&
"Conditional PHIs not expected") ? void (0) : __assert_fail (
"DT->dominates(PN, BB) && \"Conditional PHIs not expected\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 942, __extension__ __PRETTY_FUNCTION__
))
;
943 Changed = true;
944 continue;
945 }
946 }
947
948 // Remember possibly hoistable branches so we can actually hoist them
949 // later if needed.
950 if (BranchInst *BI = dyn_cast<BranchInst>(&I))
951 CFH.registerPossiblyHoistableBranch(BI);
952 }
953 }
954
955 // If we hoisted instructions to a conditional block they may not dominate
956 // their uses that weren't hoisted (such as phis where some operands are not
957 // loop invariant). If so make them unconditional by moving them to their
958 // immediate dominator. We iterate through the instructions in reverse order
959 // which ensures that when we rehoist an instruction we rehoist its operands,
960 // and also keep track of where in the block we are rehoisting to to make sure
961 // that we rehoist instructions before the instructions that use them.
962 Instruction *HoistPoint = nullptr;
963 if (ControlFlowHoisting) {
964 for (Instruction *I : reverse(HoistedInstructions)) {
965 if (!llvm::all_of(I->uses(),
966 [&](Use &U) { return DT->dominates(I, U); })) {
967 BasicBlock *Dominator =
968 DT->getNode(I->getParent())->getIDom()->getBlock();
969 if (!HoistPoint || !DT->dominates(HoistPoint->getParent(), Dominator)) {
970 if (HoistPoint)
971 assert(DT->dominates(Dominator, HoistPoint->getParent()) &&(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 972, __extension__ __PRETTY_FUNCTION__
))
972 "New hoist point expected to dominate old hoist point")(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 972, __extension__ __PRETTY_FUNCTION__
))
;
973 HoistPoint = Dominator->getTerminator();
974 }
975 LLVM_DEBUG(dbgs() << "LICM rehoisting to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
976 << HoistPoint->getParent()->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
977 << ": " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
;
978 moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
979 HoistPoint = I;
980 Changed = true;
981 }
982 }
983 }
984 if (VerifyMemorySSA)
985 MSSAU->getMemorySSA()->verifyMemorySSA();
986
987 // Now that we've finished hoisting make sure that LI and DT are still
988 // valid.
989#ifdef EXPENSIVE_CHECKS
990 if (Changed) {
991 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
(0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 992, __extension__ __PRETTY_FUNCTION__
))
992 "Dominator tree verification failed")(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
(0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 992, __extension__ __PRETTY_FUNCTION__
))
;
993 LI->verify(*DT);
994 }
995#endif
996
997 return Changed;
998}
999
1000// Return true if LI is invariant within scope of the loop. LI is invariant if
1001// CurLoop is dominated by an invariant.start representing the same memory
1002// location and size as the memory location LI loads from, and also the
1003// invariant.start has no uses.
1004static bool isLoadInvariantInLoop(LoadInst *LI, DominatorTree *DT,
1005 Loop *CurLoop) {
1006 Value *Addr = LI->getOperand(0);
1007 const DataLayout &DL = LI->getModule()->getDataLayout();
1008 const TypeSize LocSizeInBits = DL.getTypeSizeInBits(LI->getType());
1009
1010 // It is not currently possible for clang to generate an invariant.start
1011 // intrinsic with scalable vector types because we don't support thread local
1012 // sizeless types and we don't permit sizeless types in structs or classes.
1013 // Furthermore, even if support is added for this in future the intrinsic
1014 // itself is defined to have a size of -1 for variable sized objects. This
1015 // makes it impossible to verify if the intrinsic envelops our region of
1016 // interest. For example, both <vscale x 32 x i8> and <vscale x 16 x i8>
1017 // types would have a -1 parameter, but the former is clearly double the size
1018 // of the latter.
1019 if (LocSizeInBits.isScalable())
29
Calling 'LinearPolySize::isScalable'
32
Returning from 'LinearPolySize::isScalable'
33
Taking true branch
1020 return false;
34
Returning zero, which participates in a condition later
1021
1022 // if the type is i8 addrspace(x)*, we know this is the type of
1023 // llvm.invariant.start operand
1024 auto *PtrInt8Ty = PointerType::get(Type::getInt8Ty(LI->getContext()),
1025 LI->getPointerAddressSpace());
1026 unsigned BitcastsVisited = 0;
1027 // Look through bitcasts until we reach the i8* type (this is invariant.start
1028 // operand type).
1029 while (Addr->getType() != PtrInt8Ty) {
1030 auto *BC = dyn_cast<BitCastInst>(Addr);
1031 // Avoid traversing high number of bitcast uses.
1032 if (++BitcastsVisited > MaxNumUsesTraversed || !BC)
1033 return false;
1034 Addr = BC->getOperand(0);
1035 }
1036 // If we've ended up at a global/constant, bail. We shouldn't be looking at
1037 // uselists for non-local Values in a loop pass.
1038 if (isa<Constant>(Addr))
1039 return false;
1040
1041 unsigned UsesVisited = 0;
1042 // Traverse all uses of the load operand value, to see if invariant.start is
1043 // one of the uses, and whether it dominates the load instruction.
1044 for (auto *U : Addr->users()) {
1045 // Avoid traversing for Load operand with high number of users.
1046 if (++UsesVisited > MaxNumUsesTraversed)
1047 return false;
1048 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
1049 // If there are escaping uses of invariant.start instruction, the load maybe
1050 // non-invariant.
1051 if (!II || II->getIntrinsicID() != Intrinsic::invariant_start ||
1052 !II->use_empty())
1053 continue;
1054 ConstantInt *InvariantSize = cast<ConstantInt>(II->getArgOperand(0));
1055 // The intrinsic supports having a -1 argument for variable sized objects
1056 // so we should check for that here.
1057 if (InvariantSize->isNegative())
1058 continue;
1059 uint64_t InvariantSizeInBits = InvariantSize->getSExtValue() * 8;
1060 // Confirm the invariant.start location size contains the load operand size
1061 // in bits. Also, the invariant.start should dominate the load, and we
1062 // should not hoist the load out of a loop that contains this dominating
1063 // invariant.start.
1064 if (LocSizeInBits.getFixedSize() <= InvariantSizeInBits &&
1065 DT->properlyDominates(II->getParent(), CurLoop->getHeader()))
1066 return true;
1067 }
1068
1069 return false;
1070}
1071
1072namespace {
1073/// Return true if-and-only-if we know how to (mechanically) both hoist and
1074/// sink a given instruction out of a loop. Does not address legality
1075/// concerns such as aliasing or speculation safety.
1076bool isHoistableAndSinkableInst(Instruction &I) {
1077 // Only these instructions are hoistable/sinkable.
1078 return (isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) ||
5
Assuming 'I' is a 'LoadInst'
6
Returning the value 1, which participates in a condition later
1079 isa<FenceInst>(I) || isa<CastInst>(I) || isa<UnaryOperator>(I) ||
1080 isa<BinaryOperator>(I) || isa<SelectInst>(I) ||
1081 isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
1082 isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
1083 isa<ShuffleVectorInst>(I) || isa<ExtractValueInst>(I) ||
1084 isa<InsertValueInst>(I) || isa<FreezeInst>(I));
1085}
1086/// Return true if all of the alias sets within this AST are known not to
1087/// contain a Mod, or if MSSA knows there are no MemoryDefs in the loop.
1088bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
1089 const Loop *L) {
1090 if (CurAST) {
1091 for (AliasSet &AS : *CurAST) {
1092 if (!AS.isForwardingAliasSet() && AS.isMod()) {
1093 return false;
1094 }
1095 }
1096 return true;
1097 } else { /*MSSAU*/
1098 for (auto *BB : L->getBlocks())
1099 if (MSSAU->getMemorySSA()->getBlockDefs(BB))
1100 return false;
1101 return true;
1102 }
1103}
1104
1105/// Return true if I is the only Instruction with a MemoryAccess in L.
1106bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
1107 const MemorySSAUpdater *MSSAU) {
1108 for (auto *BB : L->getBlocks())
1109 if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
1110 int NotAPhi = 0;
1111 for (const auto &Acc : *Accs) {
1112 if (isa<MemoryPhi>(&Acc))
1113 continue;
1114 const auto *MUD = cast<MemoryUseOrDef>(&Acc);
1115 if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
1116 return false;
1117 }
1118 }
1119 return true;
1120}
1121}
1122
1123bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
1124 Loop *CurLoop, AliasSetTracker *CurAST,
1125 MemorySSAUpdater *MSSAU,
1126 bool TargetExecutesOncePerLoop,
1127 SinkAndHoistLICMFlags *Flags,
1128 OptimizationRemarkEmitter *ORE) {
1129 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1130, __extension__ __PRETTY_FUNCTION__
))
1
Assuming pointer value is null
2
Assuming the condition is true
3
'?' condition is true
1130 "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1130, __extension__ __PRETTY_FUNCTION__
))
;
1131
1132 // If we don't understand the instruction, bail early.
1133 if (!isHoistableAndSinkableInst(I))
4
Calling 'isHoistableAndSinkableInst'
7
Returning from 'isHoistableAndSinkableInst'
1134 return false;
1135
1136 MemorySSA *MSSA = MSSAU
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
? MSSAU->getMemorySSA() : nullptr;
8
Taking false branch
9
'?' condition is true
10
'MSSA' initialized here
1137 if (MSSA)
11
Assuming 'MSSA' is null
12
Taking false branch
1138 assert(Flags != nullptr && "Flags cannot be null.")(static_cast <bool> (Flags != nullptr && "Flags cannot be null."
) ? void (0) : __assert_fail ("Flags != nullptr && \"Flags cannot be null.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1138, __extension__ __PRETTY_FUNCTION__
))
;
1139
1140 // Loads have extra constraints we have to verify before we can hoist them.
1141 if (LoadInst *LI
13.1
'LI' is non-null
13.1
'LI' is non-null
13.1
'LI' is non-null
13.1
'LI' is non-null
= dyn_cast<LoadInst>(&I)) {
13
Assuming the object is a 'LoadInst'
14
Taking true branch
1142 if (!LI->isUnordered())
15
Calling 'LoadInst::isUnordered'
19
Returning from 'LoadInst::isUnordered'
20
Taking false branch
1143 return false; // Don't sink/hoist volatile or ordered atomic loads!
1144
1145 // Loads from constant memory are always safe to move, even if they end up
1146 // in the same alias set as something that ends up being modified.
1147 if (AA->pointsToConstantMemory(LI->getOperand(0)))
21
Assuming the condition is false
22
Taking false branch
1148 return true;
1149 if (LI->hasMetadata(LLVMContext::MD_invariant_load))
23
Calling 'Instruction::hasMetadata'
26
Returning from 'Instruction::hasMetadata'
1150 return true;
1151
1152 if (LI->isAtomic() && !TargetExecutesOncePerLoop)
27
Assuming the condition is false
1153 return false; // Don't risk duplicating unordered loads
1154
1155 // This checks for an invariant.start dominating the load.
1156 if (isLoadInvariantInLoop(LI, DT, CurLoop))
28
Calling 'isLoadInvariantInLoop'
35
Returning from 'isLoadInvariantInLoop'
36
Taking false branch
1157 return true;
1158
1159 bool Invalidated;
1160 if (CurAST
36.1
'CurAST' is null
36.1
'CurAST' is null
36.1
'CurAST' is null
36.1
'CurAST' is null
)
37
Taking false branch
1161 Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
1162 CurLoop, AA);
1163 else
1164 Invalidated = pointerInvalidatedByLoopWithMSSA(
1165 MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, I, *Flags);
38
Called C++ object pointer is null
1166 // Check loop-invariant address because this may also be a sinkable load
1167 // whose address is not necessarily loop-invariant.
1168 if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
1169 ORE->emit([&]() {
1170 return OptimizationRemarkMissed(
1171 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressInvalidated", LI)
1172 << "failed to move load with loop-invariant address "
1173 "because the loop may invalidate its value";
1174 });
1175
1176 return !Invalidated;
1177 } else if (CallInst *CI = dyn_cast<CallInst>(&I)) {
1178 // Don't sink or hoist dbg info; it's legal, but not useful.
1179 if (isa<DbgInfoIntrinsic>(I))
1180 return false;
1181
1182 // Don't sink calls which can throw.
1183 if (CI->mayThrow())
1184 return false;
1185
1186 // Convergent attribute has been used on operations that involve
1187 // inter-thread communication which results are implicitly affected by the
1188 // enclosing control flows. It is not safe to hoist or sink such operations
1189 // across control flow.
1190 if (CI->isConvergent())
1191 return false;
1192
1193 using namespace PatternMatch;
1194 if (match(CI, m_Intrinsic<Intrinsic::assume>()))
1195 // Assumes don't actually alias anything or throw
1196 return true;
1197
1198 if (match(CI, m_Intrinsic<Intrinsic::experimental_widenable_condition>()))
1199 // Widenable conditions don't actually alias anything or throw
1200 return true;
1201
1202 // Handle simple cases by querying alias analysis.
1203 FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
1204 if (Behavior == FMRB_DoesNotAccessMemory)
1205 return true;
1206 if (AAResults::onlyReadsMemory(Behavior)) {
1207 // A readonly argmemonly function only reads from memory pointed to by
1208 // it's arguments with arbitrary offsets. If we can prove there are no
1209 // writes to this memory in the loop, we can hoist or sink.
1210 if (AAResults::onlyAccessesArgPointees(Behavior)) {
1211 // TODO: expand to writeable arguments
1212 for (Value *Op : CI->args())
1213 if (Op->getType()->isPointerTy()) {
1214 bool Invalidated;
1215 if (CurAST)
1216 Invalidated = pointerInvalidatedByLoop(
1217 MemoryLocation::getBeforeOrAfter(Op), CurAST, CurLoop, AA);
1218 else
1219 Invalidated = pointerInvalidatedByLoopWithMSSA(
1220 MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I,
1221 *Flags);
1222 if (Invalidated)
1223 return false;
1224 }
1225 return true;
1226 }
1227
1228 // If this call only reads from memory and there are no writes to memory
1229 // in the loop, we can hoist or sink the call as appropriate.
1230 if (isReadOnly(CurAST, MSSAU, CurLoop))
1231 return true;
1232 }
1233
1234 // FIXME: This should use mod/ref information to see if we can hoist or
1235 // sink the call.
1236
1237 return false;
1238 } else if (auto *FI = dyn_cast<FenceInst>(&I)) {
1239 // Fences alias (most) everything to provide ordering. For the moment,
1240 // just give up if there are any other memory operations in the loop.
1241 if (CurAST) {
1242 auto Begin = CurAST->begin();
1243 assert(Begin != CurAST->end() && "must contain FI")(static_cast <bool> (Begin != CurAST->end() &&
"must contain FI") ? void (0) : __assert_fail ("Begin != CurAST->end() && \"must contain FI\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1243, __extension__ __PRETTY_FUNCTION__
))
;
1244 if (std::next(Begin) != CurAST->end())
1245 // constant memory for instance, TODO: handle better
1246 return false;
1247 auto *UniqueI = Begin->getUniqueInstruction();
1248 if (!UniqueI)
1249 // other memory op, give up
1250 return false;
1251 (void)FI; // suppress unused variable warning
1252 assert(UniqueI == FI && "AS must contain FI")(static_cast <bool> (UniqueI == FI && "AS must contain FI"
) ? void (0) : __assert_fail ("UniqueI == FI && \"AS must contain FI\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1252, __extension__ __PRETTY_FUNCTION__
))
;
1253 return true;
1254 } else // MSSAU
1255 return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
1256 } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
1257 if (!SI->isUnordered())
1258 return false; // Don't sink/hoist volatile or ordered atomic store!
1259
1260 // We can only hoist a store that we can prove writes a value which is not
1261 // read or overwritten within the loop. For those cases, we fallback to
1262 // load store promotion instead. TODO: We can extend this to cases where
1263 // there is exactly one write to the location and that write dominates an
1264 // arbitrary number of reads in the loop.
1265 if (CurAST) {
1266 auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
1267
1268 if (AS.isRef() || !AS.isMustAlias())
1269 // Quick exit test, handled by the full path below as well.
1270 return false;
1271 auto *UniqueI = AS.getUniqueInstruction();
1272 if (!UniqueI)
1273 // other memory op, give up
1274 return false;
1275 assert(UniqueI == SI && "AS must contain SI")(static_cast <bool> (UniqueI == SI && "AS must contain SI"
) ? void (0) : __assert_fail ("UniqueI == SI && \"AS must contain SI\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1275, __extension__ __PRETTY_FUNCTION__
))
;
1276 return true;
1277 } else { // MSSAU
1278 if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
1279 return true;
1280 // If there are more accesses than the Promotion cap or no "quota" to
1281 // check clobber, then give up as we're not walking a list that long.
1282 if (Flags->tooManyMemoryAccesses() || Flags->tooManyClobberingCalls())
1283 return false;
1284 // If there are interfering Uses (i.e. their defining access is in the
1285 // loop), or ordered loads (stored as Defs!), don't move this store.
1286 // Could do better here, but this is conservatively correct.
1287 // TODO: Cache set of Uses on the first walk in runOnLoop, update when
1288 // moving accesses. Can also extend to dominating uses.
1289 auto *SIMD = MSSA->getMemoryAccess(SI);
1290 for (auto *BB : CurLoop->getBlocks())
1291 if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
1292 for (const auto &MA : *Accesses)
1293 if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
1294 auto *MD = MU->getDefiningAccess();
1295 if (!MSSA->isLiveOnEntryDef(MD) &&
1296 CurLoop->contains(MD->getBlock()))
1297 return false;
1298 // Disable hoisting past potentially interfering loads. Optimized
1299 // Uses may point to an access outside the loop, as getClobbering
1300 // checks the previous iteration when walking the backedge.
1301 // FIXME: More precise: no Uses that alias SI.
1302 if (!Flags->getIsSink() && !MSSA->dominates(SIMD, MU))
1303 return false;
1304 } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
1305 if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
1306 (void)LI; // Silence warning.
1307 assert(!LI->isUnordered() && "Expected unordered load")(static_cast <bool> (!LI->isUnordered() && "Expected unordered load"
) ? void (0) : __assert_fail ("!LI->isUnordered() && \"Expected unordered load\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1307, __extension__ __PRETTY_FUNCTION__
))
;
1308 return false;
1309 }
1310 // Any call, while it may not be clobbering SI, it may be a use.
1311 if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
1312 // Check if the call may read from the memory location written
1313 // to by SI. Check CI's attributes and arguments; the number of
1314 // such checks performed is limited above by NoOfMemAccTooLarge.
1315 ModRefInfo MRI = AA->getModRefInfo(CI, MemoryLocation::get(SI));
1316 if (isModOrRefSet(MRI))
1317 return false;
1318 }
1319 }
1320 }
1321 auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
1322 Flags->incrementClobberingCalls();
1323 // If there are no clobbering Defs in the loop, store is safe to hoist.
1324 return MSSA->isLiveOnEntryDef(Source) ||
1325 !CurLoop->contains(Source->getBlock());
1326 }
1327 }
1328
1329 assert(!I.mayReadOrWriteMemory() && "unhandled aliasing")(static_cast <bool> (!I.mayReadOrWriteMemory() &&
"unhandled aliasing") ? void (0) : __assert_fail ("!I.mayReadOrWriteMemory() && \"unhandled aliasing\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1329, __extension__ __PRETTY_FUNCTION__
))
;
1330
1331 // We've established mechanical ability and aliasing, it's up to the caller
1332 // to check fault safety
1333 return true;
1334}
1335
1336/// Returns true if a PHINode is a trivially replaceable with an
1337/// Instruction.
1338/// This is true when all incoming values are that instruction.
1339/// This pattern occurs most often with LCSSA PHI nodes.
1340///
1341static bool isTriviallyReplaceablePHI(const PHINode &PN, const Instruction &I) {
1342 for (const Value *IncValue : PN.incoming_values())
1343 if (IncValue != &I)
1344 return false;
1345
1346 return true;
1347}
1348
1349/// Return true if the instruction is free in the loop.
1350static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
1351 const TargetTransformInfo *TTI) {
1352
1353 if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I)) {
1354 if (TTI->getUserCost(GEP, TargetTransformInfo::TCK_SizeAndLatency) !=
1355 TargetTransformInfo::TCC_Free)
1356 return false;
1357 // For a GEP, we cannot simply use getUserCost because currently it
1358 // optimistically assume that a GEP will fold into addressing mode
1359 // regardless of its users.
1360 const BasicBlock *BB = GEP->getParent();
1361 for (const User *U : GEP->users()) {
1362 const Instruction *UI = cast<Instruction>(U);
1363 if (CurLoop->contains(UI) &&
1364 (BB != UI->getParent() ||
1365 (!isa<StoreInst>(UI) && !isa<LoadInst>(UI))))
1366 return false;
1367 }
1368 return true;
1369 } else
1370 return TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) ==
1371 TargetTransformInfo::TCC_Free;
1372}
1373
1374/// Return true if the only users of this instruction are outside of
1375/// the loop. If this is true, we can sink the instruction to the exit
1376/// blocks of the loop.
1377///
1378/// We also return true if the instruction could be folded away in lowering.
1379/// (e.g., a GEP can be folded into a load as an addressing mode in the loop).
1380static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
1381 const LoopSafetyInfo *SafetyInfo,
1382 TargetTransformInfo *TTI, bool &FreeInLoop,
1383 bool LoopNestMode) {
1384 const auto &BlockColors = SafetyInfo->getBlockColors();
1385 bool IsFree = isFreeInLoop(I, CurLoop, TTI);
1386 for (const User *U : I.users()) {
1387 const Instruction *UI = cast<Instruction>(U);
1388 if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
1389 const BasicBlock *BB = PN->getParent();
1390 // We cannot sink uses in catchswitches.
1391 if (isa<CatchSwitchInst>(BB->getTerminator()))
1392 return false;
1393
1394 // We need to sink a callsite to a unique funclet. Avoid sinking if the
1395 // phi use is too muddled.
1396 if (isa<CallInst>(I))
1397 if (!BlockColors.empty() &&
1398 BlockColors.find(const_cast<BasicBlock *>(BB))->second.size() != 1)
1399 return false;
1400
1401 if (LoopNestMode) {
1402 while (isa<PHINode>(UI) && UI->hasOneUser() &&
1403 UI->getNumOperands() == 1) {
1404 if (!CurLoop->contains(UI))
1405 break;
1406 UI = cast<Instruction>(UI->user_back());
1407 }
1408 }
1409 }
1410
1411 if (CurLoop->contains(UI)) {
1412 if (IsFree) {
1413 FreeInLoop = true;
1414 continue;
1415 }
1416 return false;
1417 }
1418 }
1419 return true;
1420}
1421
1422static Instruction *cloneInstructionInExitBlock(
1423 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
1424 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
1425 Instruction *New;
1426 if (auto *CI = dyn_cast<CallInst>(&I)) {
1427 const auto &BlockColors = SafetyInfo->getBlockColors();
1428
1429 // Sinking call-sites need to be handled differently from other
1430 // instructions. The cloned call-site needs a funclet bundle operand
1431 // appropriate for its location in the CFG.
1432 SmallVector<OperandBundleDef, 1> OpBundles;
1433 for (unsigned BundleIdx = 0, BundleEnd = CI->getNumOperandBundles();
1434 BundleIdx != BundleEnd; ++BundleIdx) {
1435 OperandBundleUse Bundle = CI->getOperandBundleAt(BundleIdx);
1436 if (Bundle.getTagID() == LLVMContext::OB_funclet)
1437 continue;
1438
1439 OpBundles.emplace_back(Bundle);
1440 }
1441
1442 if (!BlockColors.empty()) {
1443 const ColorVector &CV = BlockColors.find(&ExitBlock)->second;
1444 assert(CV.size() == 1 && "non-unique color for exit block!")(static_cast <bool> (CV.size() == 1 && "non-unique color for exit block!"
) ? void (0) : __assert_fail ("CV.size() == 1 && \"non-unique color for exit block!\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1444, __extension__ __PRETTY_FUNCTION__
))
;
1445 BasicBlock *BBColor = CV.front();
1446 Instruction *EHPad = BBColor->getFirstNonPHI();
1447 if (EHPad->isEHPad())
1448 OpBundles.emplace_back("funclet", EHPad);
1449 }
1450
1451 New = CallInst::Create(CI, OpBundles);
1452 } else {
1453 New = I.clone();
1454 }
1455
1456 ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
1457 if (!I.getName().empty())
1458 New->setName(I.getName() + ".le");
1459
1460 if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
1461 // Create a new MemoryAccess and let MemorySSA set its defining access.
1462 MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
1463 New, nullptr, New->getParent(), MemorySSA::Beginning);
1464 if (NewMemAcc) {
1465 if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
1466 MSSAU->insertDef(MemDef, /*RenameUses=*/true);
1467 else {
1468 auto *MemUse = cast<MemoryUse>(NewMemAcc);
1469 MSSAU->insertUse(MemUse, /*RenameUses=*/true);
1470 }
1471 }
1472 }
1473
1474 // Build LCSSA PHI nodes for any in-loop operands (if legal). Note that
1475 // this is particularly cheap because we can rip off the PHI node that we're
1476 // replacing for the number and blocks of the predecessors.
1477 // OPT: If this shows up in a profile, we can instead finish sinking all
1478 // invariant instructions, and then walk their operands to re-establish
1479 // LCSSA. That will eliminate creating PHI nodes just to nuke them when
1480 // sinking bottom-up.
1481 for (Use &Op : New->operands())
1482 if (LI->wouldBeOutOfLoopUseRequiringLCSSA(Op.get(), PN.getParent())) {
1483 auto *OInst = cast<Instruction>(Op.get());
1484 PHINode *OpPN =
1485 PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
1486 OInst->getName() + ".lcssa", &ExitBlock.front());
1487 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1488 OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
1489 Op = OpPN;
1490 }
1491 return New;
1492}
1493
1494static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
1495 MemorySSAUpdater *MSSAU) {
1496 if (MSSAU)
1497 MSSAU->removeMemoryAccess(&I);
1498 SafetyInfo.removeInstruction(&I);
1499 I.eraseFromParent();
1500}
1501
1502static void moveInstructionBefore(Instruction &I, Instruction &Dest,
1503 ICFLoopSafetyInfo &SafetyInfo,
1504 MemorySSAUpdater *MSSAU,
1505 ScalarEvolution *SE) {
1506 SafetyInfo.removeInstruction(&I);
1507 SafetyInfo.insertInstructionTo(&I, Dest.getParent());
1508 I.moveBefore(&Dest);
1509 if (MSSAU)
1510 if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
1511 MSSAU->getMemorySSA()->getMemoryAccess(&I)))
1512 MSSAU->moveToPlace(OldMemAcc, Dest.getParent(),
1513 MemorySSA::BeforeTerminator);
1514 if (SE)
1515 SE->forgetValue(&I);
1516}
1517
1518static Instruction *sinkThroughTriviallyReplaceablePHI(
1519 PHINode *TPN, Instruction *I, LoopInfo *LI,
1520 SmallDenseMap<BasicBlock *, Instruction *, 32> &SunkCopies,
1521 const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
1522 MemorySSAUpdater *MSSAU) {
1523 assert(isTriviallyReplaceablePHI(*TPN, *I) &&(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1524, __extension__ __PRETTY_FUNCTION__
))
1524 "Expect only trivially replaceable PHI")(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1524, __extension__ __PRETTY_FUNCTION__
))
;
1525 BasicBlock *ExitBlock = TPN->getParent();
1526 Instruction *New;
1527 auto It = SunkCopies.find(ExitBlock);
1528 if (It != SunkCopies.end())
1529 New = It->second;
1530 else
1531 New = SunkCopies[ExitBlock] = cloneInstructionInExitBlock(
1532 *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
1533 return New;
1534}
1535
1536static bool canSplitPredecessors(PHINode *PN, LoopSafetyInfo *SafetyInfo) {
1537 BasicBlock *BB = PN->getParent();
1538 if (!BB->canSplitPredecessors())
1539 return false;
1540 // It's not impossible to split EHPad blocks, but if BlockColors already exist
1541 // it require updating BlockColors for all offspring blocks accordingly. By
1542 // skipping such corner case, we can make updating BlockColors after splitting
1543 // predecessor fairly simple.
1544 if (!SafetyInfo->getBlockColors().empty() && BB->getFirstNonPHI()->isEHPad())
1545 return false;
1546 for (BasicBlock *BBPred : predecessors(BB)) {
1547 if (isa<IndirectBrInst>(BBPred->getTerminator()) ||
1548 isa<CallBrInst>(BBPred->getTerminator()))
1549 return false;
1550 }
1551 return true;
1552}
1553
1554static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
1555 LoopInfo *LI, const Loop *CurLoop,
1556 LoopSafetyInfo *SafetyInfo,
1557 MemorySSAUpdater *MSSAU) {
1558#ifndef NDEBUG
1559 SmallVector<BasicBlock *, 32> ExitBlocks;
1560 CurLoop->getUniqueExitBlocks(ExitBlocks);
1561 SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
1562 ExitBlocks.end());
1563#endif
1564 BasicBlock *ExitBB = PN->getParent();
1565 assert(ExitBlockSet.count(ExitBB) && "Expect the PHI is in an exit block.")(static_cast <bool> (ExitBlockSet.count(ExitBB) &&
"Expect the PHI is in an exit block.") ? void (0) : __assert_fail
("ExitBlockSet.count(ExitBB) && \"Expect the PHI is in an exit block.\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1565, __extension__ __PRETTY_FUNCTION__
))
;
1566
1567 // Split predecessors of the loop exit to make instructions in the loop are
1568 // exposed to exit blocks through trivially replaceable PHIs while keeping the
1569 // loop in the canonical form where each predecessor of each exit block should
1570 // be contained within the loop. For example, this will convert the loop below
1571 // from
1572 //
1573 // LB1:
1574 // %v1 =
1575 // br %LE, %LB2
1576 // LB2:
1577 // %v2 =
1578 // br %LE, %LB1
1579 // LE:
1580 // %p = phi [%v1, %LB1], [%v2, %LB2] <-- non-trivially replaceable
1581 //
1582 // to
1583 //
1584 // LB1:
1585 // %v1 =
1586 // br %LE.split, %LB2
1587 // LB2:
1588 // %v2 =
1589 // br %LE.split2, %LB1
1590 // LE.split:
1591 // %p1 = phi [%v1, %LB1] <-- trivially replaceable
1592 // br %LE
1593 // LE.split2:
1594 // %p2 = phi [%v2, %LB2] <-- trivially replaceable
1595 // br %LE
1596 // LE:
1597 // %p = phi [%p1, %LE.split], [%p2, %LE.split2]
1598 //
1599 const auto &BlockColors = SafetyInfo->getBlockColors();
1600 SmallSetVector<BasicBlock *, 8> PredBBs(pred_begin(ExitBB), pred_end(ExitBB));
1601 while (!PredBBs.empty()) {
1602 BasicBlock *PredBB = *PredBBs.begin();
1603 assert(CurLoop->contains(PredBB) &&(static_cast <bool> (CurLoop->contains(PredBB) &&
"Expect all predecessors are in the loop") ? void (0) : __assert_fail
("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1604, __extension__ __PRETTY_FUNCTION__
))
1604 "Expect all predecessors are in the loop")(static_cast <bool> (CurLoop->contains(PredBB) &&
"Expect all predecessors are in the loop") ? void (0) : __assert_fail
("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1604, __extension__ __PRETTY_FUNCTION__
))
;
1605 if (PN->getBasicBlockIndex(PredBB) >= 0) {
1606 BasicBlock *NewPred = SplitBlockPredecessors(
1607 ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
1608 // Since we do not allow splitting EH-block with BlockColors in
1609 // canSplitPredecessors(), we can simply assign predecessor's color to
1610 // the new block.
1611 if (!BlockColors.empty())
1612 // Grab a reference to the ColorVector to be inserted before getting the
1613 // reference to the vector we are copying because inserting the new
1614 // element in BlockColors might cause the map to be reallocated.
1615 SafetyInfo->copyColors(NewPred, PredBB);
1616 }
1617 PredBBs.remove(PredBB);
1618 }
1619}
1620
1621/// When an instruction is found to only be used outside of the loop, this
1622/// function moves it to the exit blocks and patches up SSA form as needed.
1623/// This method is guaranteed to remove the original instruction from its
1624/// position, and may either delete it or move it to outside of the loop.
1625///
1626static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
1627 BlockFrequencyInfo *BFI, const Loop *CurLoop,
1628 ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
1629 OptimizationRemarkEmitter *ORE) {
1630 bool Changed = false;
1631 LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM sinking instruction: " <<
I << "\n"; } } while (false)
;
1632
1633 // Iterate over users to be ready for actual sinking. Replace users via
1634 // unreachable blocks with undef and make all user PHIs trivially replaceable.
1635 SmallPtrSet<Instruction *, 8> VisitedUsers;
1636 for (Value::user_iterator UI = I.user_begin(), UE = I.user_end(); UI != UE;) {
1637 auto *User = cast<Instruction>(*UI);
1638 Use &U = UI.getUse();
1639 ++UI;
1640
1641 if (VisitedUsers.count(User) || CurLoop->contains(User))
1642 continue;
1643
1644 if (!DT->isReachableFromEntry(User->getParent())) {
1645 U = UndefValue::get(I.getType());
1646 Changed = true;
1647 continue;
1648 }
1649
1650 // The user must be a PHI node.
1651 PHINode *PN = cast<PHINode>(User);
1652
1653 // Surprisingly, instructions can be used outside of loops without any
1654 // exits. This can only happen in PHI nodes if the incoming block is
1655 // unreachable.
1656 BasicBlock *BB = PN->getIncomingBlock(U);
1657 if (!DT->isReachableFromEntry(BB)) {
1658 U = UndefValue::get(I.getType());
1659 Changed = true;
1660 continue;
1661 }
1662
1663 VisitedUsers.insert(PN);
1664 if (isTriviallyReplaceablePHI(*PN, I))
1665 continue;
1666
1667 if (!canSplitPredecessors(PN, SafetyInfo))
1668 return Changed;
1669
1670 // Split predecessors of the PHI so that we can make users trivially
1671 // replaceable.
1672 splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);
1673
1674 // Should rebuild the iterators, as they may be invalidated by
1675 // splitPredecessorsOfLoopExit().
1676 UI = I.user_begin();
1677 UE = I.user_end();
1678 }
1679
1680 if (VisitedUsers.empty())
1681 return Changed;
1682
1683 ORE->emit([&]() {
1684 return OptimizationRemark(DEBUG_TYPE"licm", "InstSunk", &I)
1685 << "sinking " << ore::NV("Inst", &I);
1686 });
1687 if (isa<LoadInst>(I))
1688 ++NumMovedLoads;
1689 else if (isa<CallInst>(I))
1690 ++NumMovedCalls;
1691 ++NumSunk;
1692
1693#ifndef NDEBUG
1694 SmallVector<BasicBlock *, 32> ExitBlocks;
1695 CurLoop->getUniqueExitBlocks(ExitBlocks);
1696 SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
1697 ExitBlocks.end());
1698#endif
1699
1700 // Clones of this instruction. Don't create more than one per exit block!
1701 SmallDenseMap<BasicBlock *, Instruction *, 32> SunkCopies;
1702
1703 // If this instruction is only used outside of the loop, then all users are
1704 // PHI nodes in exit blocks due to LCSSA form. Just RAUW them with clones of
1705 // the instruction.
1706 // First check if I is worth sinking for all uses. Sink only when it is worth
1707 // across all uses.
1708 SmallSetVector<User*, 8> Users(I.user_begin(), I.user_end());
1709 for (auto *UI : Users) {
1710 auto *User = cast<Instruction>(UI);
1711
1712 if (CurLoop->contains(User))
1713 continue;
1714
1715 PHINode *PN = cast<PHINode>(User);
1716 assert(ExitBlockSet.count(PN->getParent()) &&(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
(0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1717, __extension__ __PRETTY_FUNCTION__
))
1717 "The LCSSA PHI is not in an exit block!")(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
(0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1717, __extension__ __PRETTY_FUNCTION__
))
;
1718
1719 // The PHI must be trivially replaceable.
1720 Instruction *New = sinkThroughTriviallyReplaceablePHI(
1721 PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
1722 PN->replaceAllUsesWith(New);
1723 eraseInstruction(*PN, *SafetyInfo, nullptr);
1724 Changed = true;
1725 }
1726 return Changed;
1727}
1728
1729/// When an instruction is found to only use loop invariant operands that
1730/// is safe to hoist, this instruction is called to do the dirty work.
1731///
1732static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
1733 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
1734 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
1735 OptimizationRemarkEmitter *ORE) {
1736 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getNameOrAsOperand() << ": "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
1737 << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
;
1738 ORE->emit([&]() {
1739 return OptimizationRemark(DEBUG_TYPE"licm", "Hoisted", &I) << "hoisting "
1740 << ore::NV("Inst", &I);
1741 });
1742
1743 // Metadata can be dependent on conditions we are hoisting above.
1744 // Conservatively strip all metadata on the instruction unless we were
1745 // guaranteed to execute I if we entered the loop, in which case the metadata
1746 // is valid in the loop preheader.
1747 // Similarly, If I is a call and it is not guaranteed to execute in the loop,
1748 // then moving to the preheader means we should strip attributes on the call
1749 // that can cause UB since we may be hoisting above conditions that allowed
1750 // inferring those attributes. They may not be valid at the preheader.
1751 if ((I.hasMetadataOtherThanDebugLoc() || isa<CallInst>(I)) &&
1752 // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
1753 // time in isGuaranteedToExecute if we don't actually have anything to
1754 // drop. It is a compile time optimization, not required for correctness.
1755 !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
1756 I.dropUndefImplyingAttrsAndUnknownMetadata();
1757
1758 if (isa<PHINode>(I))
1759 // Move the new node to the end of the phi list in the destination block.
1760 moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
1761 else
1762 // Move the new node to the destination block, before its terminator.
1763 moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);
1764
1765 I.updateLocationAfterHoist();
1766
1767 if (isa<LoadInst>(I))
1768 ++NumMovedLoads;
1769 else if (isa<CallInst>(I))
1770 ++NumMovedCalls;
1771 ++NumHoisted;
1772}
1773
1774/// Only sink or hoist an instruction if it is not a trapping instruction,
1775/// or if the instruction is known not to trap when moved to the preheader.
1776/// or if it is a trapping instruction and is guaranteed to execute.
1777static bool isSafeToExecuteUnconditionally(Instruction &Inst,
1778 const DominatorTree *DT,
1779 const TargetLibraryInfo *TLI,
1780 const Loop *CurLoop,
1781 const LoopSafetyInfo *SafetyInfo,
1782 OptimizationRemarkEmitter *ORE,
1783 const Instruction *CtxI) {
1784 if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
1785 return true;
1786
1787 bool GuaranteedToExecute =
1788 SafetyInfo->isGuaranteedToExecute(Inst, DT, CurLoop);
1789
1790 if (!GuaranteedToExecute) {
1791 auto *LI = dyn_cast<LoadInst>(&Inst);
1792 if (LI && CurLoop->isLoopInvariant(LI->getPointerOperand()))
1793 ORE->emit([&]() {
1794 return OptimizationRemarkMissed(
1795 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressCondExecuted", LI)
1796 << "failed to hoist load with loop-invariant address "
1797 "because load is conditionally executed";
1798 });
1799 }
1800
1801 return GuaranteedToExecute;
1802}
1803
1804namespace {
1805class LoopPromoter : public LoadAndStorePromoter {
1806 Value *SomePtr; // Designated pointer to store to.
1807 const SmallSetVector<Value *, 8> &PointerMustAliases;
1808 SmallVectorImpl<BasicBlock *> &LoopExitBlocks;
1809 SmallVectorImpl<Instruction *> &LoopInsertPts;
1810 SmallVectorImpl<MemoryAccess *> &MSSAInsertPts;
1811 PredIteratorCache &PredCache;
1812 MemorySSAUpdater *MSSAU;
1813 LoopInfo &LI;
1814 DebugLoc DL;
1815 Align Alignment;
1816 bool UnorderedAtomic;
1817 AAMDNodes AATags;
1818 ICFLoopSafetyInfo &SafetyInfo;
1819 bool CanInsertStoresInExitBlocks;
1820
1821 // We're about to add a use of V in a loop exit block. Insert an LCSSA phi
1822 // (if legal) if doing so would add an out-of-loop use to an instruction
1823 // defined in-loop.
1824 Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
1825 if (!LI.wouldBeOutOfLoopUseRequiringLCSSA(V, BB))
1826 return V;
1827
1828 Instruction *I = cast<Instruction>(V);
1829 // We need to create an LCSSA PHI node for the incoming value and
1830 // store that.
1831 PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
1832 I->getName() + ".lcssa", &BB->front());
1833 for (BasicBlock *Pred : PredCache.get(BB))
1834 PN->addIncoming(I, Pred);
1835 return PN;
1836 }
1837
1838public:
1839 LoopPromoter(Value *SP, ArrayRef<const Instruction *> Insts, SSAUpdater &S,
1840 const SmallSetVector<Value *, 8> &PMA,
1841 SmallVectorImpl<BasicBlock *> &LEB,
1842 SmallVectorImpl<Instruction *> &LIP,
1843 SmallVectorImpl<MemoryAccess *> &MSSAIP, PredIteratorCache &PIC,
1844 MemorySSAUpdater *MSSAU, LoopInfo &li, DebugLoc dl,
1845 Align Alignment, bool UnorderedAtomic, const AAMDNodes &AATags,
1846 ICFLoopSafetyInfo &SafetyInfo, bool CanInsertStoresInExitBlocks)
1847 : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
1848 LoopExitBlocks(LEB), LoopInsertPts(LIP), MSSAInsertPts(MSSAIP),
1849 PredCache(PIC), MSSAU(MSSAU), LI(li), DL(std::move(dl)),
1850 Alignment(Alignment), UnorderedAtomic(UnorderedAtomic), AATags(AATags),
1851 SafetyInfo(SafetyInfo),
1852 CanInsertStoresInExitBlocks(CanInsertStoresInExitBlocks) {}
1853
1854 bool isInstInList(Instruction *I,
1855 const SmallVectorImpl<Instruction *> &) const override {
1856 Value *Ptr;
1857 if (LoadInst *LI = dyn_cast<LoadInst>(I))
1858 Ptr = LI->getOperand(0);
1859 else
1860 Ptr = cast<StoreInst>(I)->getPointerOperand();
1861 return PointerMustAliases.count(Ptr);
1862 }
1863
1864 void insertStoresInLoopExitBlocks() {
1865 // Insert stores after in the loop exit blocks. Each exit block gets a
1866 // store of the live-out values that feed them. Since we've already told
1867 // the SSA updater about the defs in the loop and the preheader
1868 // definition, it is all set and we can start using it.
1869 for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) {
1870 BasicBlock *ExitBlock = LoopExitBlocks[i];
1871 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
1872 LiveInValue = maybeInsertLCSSAPHI(LiveInValue, ExitBlock);
1873 Value *Ptr = maybeInsertLCSSAPHI(SomePtr, ExitBlock);
1874 Instruction *InsertPos = LoopInsertPts[i];
1875 StoreInst *NewSI = new StoreInst(LiveInValue, Ptr, InsertPos);
1876 if (UnorderedAtomic)
1877 NewSI->setOrdering(AtomicOrdering::Unordered);
1878 NewSI->setAlignment(Alignment);
1879 NewSI->setDebugLoc(DL);
1880 if (AATags)
1881 NewSI->setAAMetadata(AATags);
1882
1883 MemoryAccess *MSSAInsertPoint = MSSAInsertPts[i];
1884 MemoryAccess *NewMemAcc;
1885 if (!MSSAInsertPoint) {
1886 NewMemAcc = MSSAU->createMemoryAccessInBB(
1887 NewSI, nullptr, NewSI->getParent(), MemorySSA::Beginning);
1888 } else {
1889 NewMemAcc =
1890 MSSAU->createMemoryAccessAfter(NewSI, nullptr, MSSAInsertPoint);
1891 }
1892 MSSAInsertPts[i] = NewMemAcc;
1893 MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
1894 // FIXME: true for safety, false may still be correct.
1895 }
1896 }
1897
1898 void doExtraRewritesBeforeFinalDeletion() override {
1899 if (CanInsertStoresInExitBlocks)
1900 insertStoresInLoopExitBlocks();
1901 }
1902
1903 void instructionDeleted(Instruction *I) const override {
1904 SafetyInfo.removeInstruction(I);
1905 MSSAU->removeMemoryAccess(I);
1906 }
1907
1908 bool shouldDelete(Instruction *I) const override {
1909 if (isa<StoreInst>(I))
1910 return CanInsertStoresInExitBlocks;
1911 return true;
1912 }
1913};
1914
1915bool isNotCapturedBeforeOrInLoop(const Value *V, const Loop *L,
1916 DominatorTree *DT) {
1917 // We can perform the captured-before check against any instruction in the
1918 // loop header, as the loop header is reachable from any instruction inside
1919 // the loop.
1920 // TODO: ReturnCaptures=true shouldn't be necessary here.
1921 return !PointerMayBeCapturedBefore(V, /* ReturnCaptures */ true,
1922 /* StoreCaptures */ true,
1923 L->getHeader()->getTerminator(), DT);
1924}
1925
1926/// Return true if we can prove that a caller cannot inspect the object if an
1927/// unwind occurs inside the loop.
1928bool isNotVisibleOnUnwind(Value *Object, const Loop *L, DominatorTree *DT) {
1929 if (isa<AllocaInst>(Object))
1930 // Since the alloca goes out of scope, we know the caller can't retain a
1931 // reference to it and be well defined. Thus, we don't need to check for
1932 // capture.
1933 return true;
1934
1935 // For all other objects we need to know that the caller can't possibly
1936 // have gotten a reference to the object prior to an unwind in the loop.
1937 // There are two components of that:
1938 // 1) Object can't have been captured prior to the definition site.
1939 // For this, we need to know the return value is noalias.
1940 // 1) Object can't be captured before or inside the loop. This is what
1941 // isNotCapturedBeforeOrInLoop() checks.
1942 return isNoAliasCall(Object) && isNotCapturedBeforeOrInLoop(Object, L, DT);
1943}
1944
1945} // namespace
1946
1947/// Try to promote memory values to scalars by sinking stores out of the
1948/// loop and moving loads to before the loop. We do this by looping over
1949/// the stores in the loop, looking for stores to Must pointers which are
1950/// loop invariant.
1951///
1952bool llvm::promoteLoopAccessesToScalars(
1953 const SmallSetVector<Value *, 8> &PointerMustAliases,
1954 SmallVectorImpl<BasicBlock *> &ExitBlocks,
1955 SmallVectorImpl<Instruction *> &InsertPts,
1956 SmallVectorImpl<MemoryAccess *> &MSSAInsertPts, PredIteratorCache &PIC,
1957 LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
1958 Loop *CurLoop, MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
1959 OptimizationRemarkEmitter *ORE) {
1960 // Verify inputs.
1961 assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1963, __extension__ __PRETTY_FUNCTION__
))
1962 SafetyInfo != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1963, __extension__ __PRETTY_FUNCTION__
))
1963 "Unexpected Input to promoteLoopAccessesToScalars")(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 1963, __extension__ __PRETTY_FUNCTION__
))
;
1964
1965 Value *SomePtr = *PointerMustAliases.begin();
1966 BasicBlock *Preheader = CurLoop->getLoopPreheader();
1967
1968 // It is not safe to promote a load/store from the loop if the load/store is
1969 // conditional. For example, turning:
1970 //
1971 // for () { if (c) *P += 1; }
1972 //
1973 // into:
1974 //
1975 // tmp = *P; for () { if (c) tmp +=1; } *P = tmp;
1976 //
1977 // is not safe, because *P may only be valid to access if 'c' is true.
1978 //
1979 // The safety property divides into two parts:
1980 // p1) The memory may not be dereferenceable on entry to the loop. In this
1981 // case, we can't insert the required load in the preheader.
1982 // p2) The memory model does not allow us to insert a store along any dynamic
1983 // path which did not originally have one.
1984 //
1985 // If at least one store is guaranteed to execute, both properties are
1986 // satisfied, and promotion is legal.
1987 //
1988 // This, however, is not a necessary condition. Even if no store/load is
1989 // guaranteed to execute, we can still establish these properties.
1990 // We can establish (p1) by proving that hoisting the load into the preheader
1991 // is safe (i.e. proving dereferenceability on all paths through the loop). We
1992 // can use any access within the alias set to prove dereferenceability,
1993 // since they're all must alias.
1994 //
1995 // There are two ways establish (p2):
1996 // a) Prove the location is thread-local. In this case the memory model
1997 // requirement does not apply, and stores are safe to insert.
1998 // b) Prove a store dominates every exit block. In this case, if an exit
1999 // blocks is reached, the original dynamic path would have taken us through
2000 // the store, so inserting a store into the exit block is safe. Note that this
2001 // is different from the store being guaranteed to execute. For instance,
2002 // if an exception is thrown on the first iteration of the loop, the original
2003 // store is never executed, but the exit blocks are not executed either.
2004
2005 bool DereferenceableInPH = false;
2006 bool SafeToInsertStore = false;
2007 bool FoundLoadToPromote = false;
2008
2009 SmallVector<Instruction *, 64> LoopUses;
2010
2011 // We start with an alignment of one and try to find instructions that allow
2012 // us to prove better alignment.
2013 Align Alignment;
2014 // Keep track of which types of access we see
2015 bool SawUnorderedAtomic = false;
2016 bool SawNotAtomic = false;
2017 AAMDNodes AATags;
2018
2019 const DataLayout &MDL = Preheader->getModule()->getDataLayout();
2020
2021 bool IsKnownThreadLocalObject = false;
2022 if (SafetyInfo->anyBlockMayThrow()) {
2023 // If a loop can throw, we have to insert a store along each unwind edge.
2024 // That said, we can't actually make the unwind edge explicit. Therefore,
2025 // we have to prove that the store is dead along the unwind edge. We do
2026 // this by proving that the caller can't have a reference to the object
2027 // after return and thus can't possibly load from the object.
2028 Value *Object = getUnderlyingObject(SomePtr);
2029 if (!isNotVisibleOnUnwind(Object, CurLoop, DT))
2030 return false;
2031 // Subtlety: Alloca's aren't visible to callers, but *are* potentially
2032 // visible to other threads if captured and used during their lifetimes.
2033 IsKnownThreadLocalObject = !isa<AllocaInst>(Object);
2034 }
2035
2036 // Check that all accesses to pointers in the aliass set use the same type.
2037 // We cannot (yet) promote a memory location that is loaded and stored in
2038 // different sizes. While we are at it, collect alignment and AA info.
2039 Type *AccessTy = nullptr;
2040 for (Value *ASIV : PointerMustAliases) {
2041 for (User *U : ASIV->users()) {
2042 // Ignore instructions that are outside the loop.
2043 Instruction *UI = dyn_cast<Instruction>(U);
2044 if (!UI || !CurLoop->contains(UI))
2045 continue;
2046
2047 // If there is an non-load/store instruction in the loop, we can't promote
2048 // it.
2049 if (LoadInst *Load = dyn_cast<LoadInst>(UI)) {
2050 if (!Load->isUnordered())
2051 return false;
2052
2053 SawUnorderedAtomic |= Load->isAtomic();
2054 SawNotAtomic |= !Load->isAtomic();
2055 FoundLoadToPromote = true;
2056
2057 Align InstAlignment = Load->getAlign();
2058
2059 // Note that proving a load safe to speculate requires proving
2060 // sufficient alignment at the target location. Proving it guaranteed
2061 // to execute does as well. Thus we can increase our guaranteed
2062 // alignment as well.
2063 if (!DereferenceableInPH || (InstAlignment > Alignment))
2064 if (isSafeToExecuteUnconditionally(*Load, DT, TLI, CurLoop,
2065 SafetyInfo, ORE,
2066 Preheader->getTerminator())) {
2067 DereferenceableInPH = true;
2068 Alignment = std::max(Alignment, InstAlignment);
2069 }
2070 } else if (const StoreInst *Store = dyn_cast<StoreInst>(UI)) {
2071 // Stores *of* the pointer are not interesting, only stores *to* the
2072 // pointer.
2073 if (UI->getOperand(1) != ASIV)
2074 continue;
2075 if (!Store->isUnordered())
2076 return false;
2077
2078 SawUnorderedAtomic |= Store->isAtomic();
2079 SawNotAtomic |= !Store->isAtomic();
2080
2081 // If the store is guaranteed to execute, both properties are satisfied.
2082 // We may want to check if a store is guaranteed to execute even if we
2083 // already know that promotion is safe, since it may have higher
2084 // alignment than any other guaranteed stores, in which case we can
2085 // raise the alignment on the promoted store.
2086 Align InstAlignment = Store->getAlign();
2087
2088 if (!DereferenceableInPH || !SafeToInsertStore ||
2089 (InstAlignment > Alignment)) {
2090 if (SafetyInfo->isGuaranteedToExecute(*UI, DT, CurLoop)) {
2091 DereferenceableInPH = true;
2092 SafeToInsertStore = true;
2093 Alignment = std::max(Alignment, InstAlignment);
2094 }
2095 }
2096
2097 // If a store dominates all exit blocks, it is safe to sink.
2098 // As explained above, if an exit block was executed, a dominating
2099 // store must have been executed at least once, so we are not
2100 // introducing stores on paths that did not have them.
2101 // Note that this only looks at explicit exit blocks. If we ever
2102 // start sinking stores into unwind edges (see above), this will break.
2103 if (!SafeToInsertStore)
2104 SafeToInsertStore = llvm::all_of(ExitBlocks, [&](BasicBlock *Exit) {
2105 return DT->dominates(Store->getParent(), Exit);
2106 });
2107
2108 // If the store is not guaranteed to execute, we may still get
2109 // deref info through it.
2110 if (!DereferenceableInPH) {
2111 DereferenceableInPH = isDereferenceableAndAlignedPointer(
2112 Store->getPointerOperand(), Store->getValueOperand()->getType(),
2113 Store->getAlign(), MDL, Preheader->getTerminator(), DT, TLI);
2114 }
2115 } else
2116 return false; // Not a load or store.
2117
2118 if (!AccessTy)
2119 AccessTy = getLoadStoreType(UI);
2120 else if (AccessTy != getLoadStoreType(UI))
2121 return false;
2122
2123 // Merge the AA tags.
2124 if (LoopUses.empty()) {
2125 // On the first load/store, just take its AA tags.
2126 AATags = UI->getAAMetadata();
2127 } else if (AATags) {
2128 AATags = AATags.merge(UI->getAAMetadata());
2129 }
2130
2131 LoopUses.push_back(UI);
2132 }
2133 }
2134
2135 // If we found both an unordered atomic instruction and a non-atomic memory
2136 // access, bail. We can't blindly promote non-atomic to atomic since we
2137 // might not be able to lower the result. We can't downgrade since that
2138 // would violate memory model. Also, align 0 is an error for atomics.
2139 if (SawUnorderedAtomic && SawNotAtomic)
2140 return false;
2141
2142 // If we're inserting an atomic load in the preheader, we must be able to
2143 // lower it. We're only guaranteed to be able to lower naturally aligned
2144 // atomics.
2145 if (SawUnorderedAtomic && Alignment < MDL.getTypeStoreSize(AccessTy))
2146 return false;
2147
2148 // If we couldn't prove we can hoist the load, bail.
2149 if (!DereferenceableInPH)
2150 return false;
2151
2152 // We know we can hoist the load, but don't have a guaranteed store.
2153 // Check whether the location is thread-local. If it is, then we can insert
2154 // stores along paths which originally didn't have them without violating the
2155 // memory model.
2156 if (!SafeToInsertStore) {
2157 if (IsKnownThreadLocalObject)
2158 SafeToInsertStore = true;
2159 else {
2160 Value *Object = getUnderlyingObject(SomePtr);
2161 SafeToInsertStore =
2162 (isNoAliasCall(Object) || isa<AllocaInst>(Object)) &&
2163 isNotCapturedBeforeOrInLoop(Object, CurLoop, DT);
2164 }
2165 }
2166
2167 // If we've still failed to prove we can sink the store, hoist the load
2168 // only, if possible.
2169 if (!SafeToInsertStore && !FoundLoadToPromote)
2170 // If we cannot hoist the load either, give up.
2171 return false;
2172
2173 // Lets do the promotion!
2174 if (SafeToInsertStore)
2175 LLVM_DEBUG(dbgs() << "LICM: Promoting load/store of the value: " << *SomePtrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting load/store of the value: "
<< *SomePtr << '\n'; } } while (false)
2176 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting load/store of the value: "
<< *SomePtr << '\n'; } } while (false)
;
2177 else
2178 LLVM_DEBUG(dbgs() << "LICM: Promoting load of the value: " << *SomePtrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting load of the value: "
<< *SomePtr << '\n'; } } while (false)
2179 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting load of the value: "
<< *SomePtr << '\n'; } } while (false)
;
2180
2181 ORE->emit([&]() {
2182 return OptimizationRemark(DEBUG_TYPE"licm", "PromoteLoopAccessesToScalar",
2183 LoopUses[0])
2184 << "Moving accesses to memory location out of the loop";
2185 });
2186 ++NumPromoted;
2187
2188 // Look at all the loop uses, and try to merge their locations.
2189 std::vector<const DILocation *> LoopUsesLocs;
2190 for (auto U : LoopUses)
2191 LoopUsesLocs.push_back(U->getDebugLoc().get());
2192 auto DL = DebugLoc(DILocation::getMergedLocations(LoopUsesLocs));
2193
2194 // We use the SSAUpdater interface to insert phi nodes as required.
2195 SmallVector<PHINode *, 16> NewPHIs;
2196 SSAUpdater SSA(&NewPHIs);
2197 LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
2198 InsertPts, MSSAInsertPts, PIC, MSSAU, *LI, DL,
2199 Alignment, SawUnorderedAtomic, AATags, *SafetyInfo,
2200 SafeToInsertStore);
2201
2202 // Set up the preheader to have a definition of the value. It is the live-out
2203 // value from the preheader that uses in the loop will use.
2204 LoadInst *PreheaderLoad = new LoadInst(
2205 AccessTy, SomePtr, SomePtr->getName() + ".promoted",
2206 Preheader->getTerminator());
2207 if (SawUnorderedAtomic)
2208 PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
2209 PreheaderLoad->setAlignment(Alignment);
2210 PreheaderLoad->setDebugLoc(DebugLoc());
2211 if (AATags)
2212 PreheaderLoad->setAAMetadata(AATags);
2213 SSA.AddAvailableValue(Preheader, PreheaderLoad);
2214
2215 MemoryAccess *PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
2216 PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
2217 MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
2218 MSSAU->insertUse(NewMemUse, /*RenameUses=*/true);
2219
2220 if (VerifyMemorySSA)
2221 MSSAU->getMemorySSA()->verifyMemorySSA();
2222 // Rewrite all the loads in the loop and remember all the definitions from
2223 // stores in the loop.
2224 Promoter.run(LoopUses);
2225
2226 if (VerifyMemorySSA)
2227 MSSAU->getMemorySSA()->verifyMemorySSA();
2228 // If the SSAUpdater didn't use the load in the preheader, just zap it now.
2229 if (PreheaderLoad->use_empty())
2230 eraseInstruction(*PreheaderLoad, *SafetyInfo, MSSAU);
2231
2232 return true;
2233}
2234
2235static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
2236 function_ref<void(Instruction *)> Fn) {
2237 for (const BasicBlock *BB : L->blocks())
2238 if (const auto *Accesses = MSSA->getBlockAccesses(BB))
2239 for (const auto &Access : *Accesses)
2240 if (const auto *MUD = dyn_cast<MemoryUseOrDef>(&Access))
2241 Fn(MUD->getMemoryInst());
2242}
2243
2244static SmallVector<SmallSetVector<Value *, 8>, 0>
2245collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
2246 AliasSetTracker AST(*AA);
2247
2248 auto IsPotentiallyPromotable = [L](const Instruction *I) {
2249 if (const auto *SI = dyn_cast<StoreInst>(I))
2250 return L->isLoopInvariant(SI->getPointerOperand());
2251 if (const auto *LI = dyn_cast<LoadInst>(I))
2252 return L->isLoopInvariant(LI->getPointerOperand());
2253 return false;
2254 };
2255
2256 // Populate AST with potentially promotable accesses and remove them from
2257 // MaybePromotable, so they will not be checked again on the next iteration.
2258 SmallPtrSet<Value *, 16> AttemptingPromotion;
2259 foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
2260 if (IsPotentiallyPromotable(I)) {
2261 AttemptingPromotion.insert(I);
2262 AST.add(I);
2263 }
2264 });
2265
2266 // We're only interested in must-alias sets that contain a mod.
2267 SmallVector<const AliasSet *, 8> Sets;
2268 for (AliasSet &AS : AST)
2269 if (!AS.isForwardingAliasSet() && AS.isMod() && AS.isMustAlias())
2270 Sets.push_back(&AS);
2271
2272 if (Sets.empty())
2273 return {}; // Nothing to promote...
2274
2275 // Discard any sets for which there is an aliasing non-promotable access.
2276 foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
2277 if (AttemptingPromotion.contains(I))
2278 return;
2279
2280 llvm::erase_if(Sets, [&](const AliasSet *AS) {
2281 return AS->aliasesUnknownInst(I, *AA);
2282 });
2283 });
2284
2285 SmallVector<SmallSetVector<Value *, 8>, 0> Result;
2286 for (const AliasSet *Set : Sets) {
2287 SmallSetVector<Value *, 8> PointerMustAliases;
2288 for (const auto &ASI : *Set)
2289 PointerMustAliases.insert(ASI.getValue());
2290 Result.push_back(std::move(PointerMustAliases));
2291 }
2292
2293 return Result;
2294}
2295
2296static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
2297 AliasSetTracker *CurAST, Loop *CurLoop,
2298 AAResults *AA) {
2299 return CurAST->getAliasSetFor(MemLoc).isMod();
2300}
2301
2302bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
2303 Loop *CurLoop, Instruction &I,
2304 SinkAndHoistLICMFlags &Flags) {
2305 // For hoisting, use the walker to determine safety
2306 if (!Flags.getIsSink()) {
2307 MemoryAccess *Source;
2308 // See declaration of SetLicmMssaOptCap for usage details.
2309 if (Flags.tooManyClobberingCalls())
2310 Source = MU->getDefiningAccess();
2311 else {
2312 Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
2313 Flags.incrementClobberingCalls();
2314 }
2315 return !MSSA->isLiveOnEntryDef(Source) &&
2316 CurLoop->contains(Source->getBlock());
2317 }
2318
2319 // For sinking, we'd need to check all Defs below this use. The getClobbering
2320 // call will look on the backedge of the loop, but will check aliasing with
2321 // the instructions on the previous iteration.
2322 // For example:
2323 // for (i ... )
2324 // load a[i] ( Use (LoE)
2325 // store a[i] ( 1 = Def (2), with 2 = Phi for the loop.
2326 // i++;
2327 // The load sees no clobbering inside the loop, as the backedge alias check
2328 // does phi translation, and will check aliasing against store a[i-1].
2329 // However sinking the load outside the loop, below the store is incorrect.
2330
2331 // For now, only sink if there are no Defs in the loop, and the existing ones
2332 // precede the use and are in the same block.
2333 // FIXME: Increase precision: Safe to sink if Use post dominates the Def;
2334 // needs PostDominatorTreeAnalysis.
2335 // FIXME: More precise: no Defs that alias this Use.
2336 if (Flags.tooManyMemoryAccesses())
2337 return true;
2338 for (auto *BB : CurLoop->getBlocks())
2339 if (pointerInvalidatedByBlockWithMSSA(*BB, *MSSA, *MU))
2340 return true;
2341 // When sinking, the source block may not be part of the loop so check it.
2342 if (!CurLoop->contains(&I))
2343 return pointerInvalidatedByBlockWithMSSA(*I.getParent(), *MSSA, *MU);
2344
2345 return false;
2346}
2347
2348bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
2349 MemoryUse &MU) {
2350 if (const auto *Accesses = MSSA.getBlockDefs(&BB))
2351 for (const auto &MA : *Accesses)
2352 if (const auto *MD = dyn_cast<MemoryDef>(&MA))
2353 if (MU.getBlock() != MD->getBlock() || !MSSA.locallyDominates(MD, &MU))
2354 return true;
2355 return false;
2356}
2357
2358/// Little predicate that returns true if the specified basic block is in
2359/// a subloop of the current one, not the current one itself.
2360///
2361static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI) {
2362 assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop")(static_cast <bool> (CurLoop->contains(BB) &&
"Only valid if BB is IN the loop") ? void (0) : __assert_fail
("CurLoop->contains(BB) && \"Only valid if BB is IN the loop\""
, "llvm/lib/Transforms/Scalar/LICM.cpp", 2362, __extension__ __PRETTY_FUNCTION__
))
;
2363 return LI->getLoopFor(BB) != CurLoop;
2364}

/build/llvm-toolchain-snapshot-14~++20220116100644+5f782d25a742/llvm/include/llvm/IR/Instructions.h

</
1//===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file exposes the class definitions of all of the subclasses of the
10// Instruction class. This is meant to be an easy way to get access to all
11// instruction subclasses.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRUCTIONS_H
16#define LLVM_IR_INSTRUCTIONS_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/Bitfields.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/ADT/iterator.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/CFG.h"
32#include "llvm/IR/Constant.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/OperandTraits.h"
38#include "llvm/IR/Type.h"
39#include "llvm/IR/Use.h"
40#include "llvm/IR/User.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstddef>
47#include <cstdint>
48#include <iterator>
49
50namespace llvm {
51
52class APInt;
53class ConstantInt;
54class DataLayout;
55class LLVMContext;
56
57//===----------------------------------------------------------------------===//
58// AllocaInst Class
59//===----------------------------------------------------------------------===//
60
61/// an instruction to allocate memory on the stack
62class AllocaInst : public UnaryInstruction {
63 Type *AllocatedType;
64
65 using AlignmentField = AlignmentBitfieldElementT<0>;
66 using UsedWithInAllocaField = BoolBitfieldElementT<AlignmentField::NextBit>;
67 using SwiftErrorField = BoolBitfieldElementT<UsedWithInAllocaField::NextBit>;
68 static_assert(Bitfield::areContiguous<AlignmentField, UsedWithInAllocaField,
69 SwiftErrorField>(),
70 "Bitfields must be contiguous");
71
72protected:
73 // Note: Instruction needs to be a friend here to call cloneImpl.
74 friend class Instruction;
75
76 AllocaInst *cloneImpl() const;
77
78public:
79 explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
80 const Twine &Name, Instruction *InsertBefore);
81 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
82 const Twine &Name, BasicBlock *InsertAtEnd);
83
84 AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
85 Instruction *InsertBefore);
86 AllocaInst(Type *Ty, unsigned AddrSpace,
87 const Twine &Name, BasicBlock *InsertAtEnd);
88
89 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
90 const Twine &Name = "", Instruction *InsertBefore = nullptr);
91 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
92 const Twine &Name, BasicBlock *InsertAtEnd);
93
94 /// Return true if there is an allocation size parameter to the allocation
95 /// instruction that is not 1.
96 bool isArrayAllocation() const;
97
98 /// Get the number of elements allocated. For a simple allocation of a single
99 /// element, this will return a constant 1 value.
100 const Value *getArraySize() const { return getOperand(0); }
101 Value *getArraySize() { return getOperand(0); }
102
103 /// Overload to return most specific pointer type.
104 PointerType *getType() const {
105 return cast<PointerType>(Instruction::getType());
106 }
107
108 /// Return the address space for the allocation.
109 unsigned getAddressSpace() const {
110 return getType()->getAddressSpace();
111 }
112
113 /// Get allocation size in bits. Returns None if size can't be determined,
114 /// e.g. in case of a VLA.
115 Optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
116
117 /// Return the type that is being allocated by the instruction.
118 Type *getAllocatedType() const { return AllocatedType; }
119 /// for use only in special circumstances that need to generically
120 /// transform a whole instruction (eg: IR linking and vectorization).
121 void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
122
123 /// Return the alignment of the memory that is being allocated by the
124 /// instruction.
125 Align getAlign() const {
126 return Align(1ULL << getSubclassData<AlignmentField>());
127 }
128
129 void setAlignment(Align Align) {
130 setSubclassData<AlignmentField>(Log2(Align));
131 }
132
133 // FIXME: Remove this one transition to Align is over.
134 uint64_t getAlignment() const { return getAlign().value(); }
135
136 /// Return true if this alloca is in the entry block of the function and is a
137 /// constant size. If so, the code generator will fold it into the
138 /// prolog/epilog code, so it is basically free.
139 bool isStaticAlloca() const;
140
141 /// Return true if this alloca is used as an inalloca argument to a call. Such
142 /// allocas are never considered static even if they are in the entry block.
143 bool isUsedWithInAlloca() const {
144 return getSubclassData<UsedWithInAllocaField>();
145 }
146
147 /// Specify whether this alloca is used to represent the arguments to a call.
148 void setUsedWithInAlloca(bool V) {
149 setSubclassData<UsedWithInAllocaField>(V);
150 }
151
152 /// Return true if this alloca is used as a swifterror argument to a call.
153 bool isSwiftError() const { return getSubclassData<SwiftErrorField>(); }
154 /// Specify whether this alloca is used to represent a swifterror.
155 void setSwiftError(bool V) { setSubclassData<SwiftErrorField>(V); }
156
157 // Methods for support type inquiry through isa, cast, and dyn_cast:
158 static bool classof(const Instruction *I) {
159 return (I->getOpcode() == Instruction::Alloca);
160 }
161 static bool classof(const Value *V) {
162 return isa<Instruction>(V) && classof(cast<Instruction>(V));
163 }
164
165private:
166 // Shadow Instruction::setInstructionSubclassData with a private forwarding
167 // method so that subclasses cannot accidentally use it.
168 template <typename Bitfield>
169 void setSubclassData(typename Bitfield::Type Value) {
170 Instruction::setSubclassData<Bitfield>(Value);
171 }
172};
173
174//===----------------------------------------------------------------------===//
175// LoadInst Class
176//===----------------------------------------------------------------------===//
177
178/// An instruction for reading from memory. This uses the SubclassData field in
179/// Value to store whether or not the load is volatile.
180class LoadInst : public UnaryInstruction {
181 using VolatileField = BoolBitfieldElementT<0>;
182 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
183 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
184 static_assert(
185 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
186 "Bitfields must be contiguous");
187
188 void AssertOK();
189
190protected:
191 // Note: Instruction needs to be a friend here to call cloneImpl.
192 friend class Instruction;
193
194 LoadInst *cloneImpl() const;
195
196public:
197 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
198 Instruction *InsertBefore);
199 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
200 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
201 Instruction *InsertBefore);
202 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
203 BasicBlock *InsertAtEnd);
204 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
205 Align Align, Instruction *InsertBefore = nullptr);
206 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
207 Align Align, BasicBlock *InsertAtEnd);
208 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
209 Align Align, AtomicOrdering Order,
210 SyncScope::ID SSID = SyncScope::System,
211 Instruction *InsertBefore = nullptr);
212 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
213 Align Align, AtomicOrdering Order, SyncScope::ID SSID,
214 BasicBlock *InsertAtEnd);
215
216 /// Return true if this is a load from a volatile memory location.
217 bool isVolatile() const { return getSubclassData<VolatileField>(); }
218
219 /// Specify whether this is a volatile load or not.
220 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
221
222 /// Return the alignment of the access that is being performed.
223 /// FIXME: Remove this function once transition to Align is over.
224 /// Use getAlign() instead.
225 uint64_t getAlignment() const { return getAlign().value(); }
226
227 /// Return the alignment of the access that is being performed.
228 Align getAlign() const {
229 return Align(1ULL << (getSubclassData<AlignmentField>()));
230 }
231
232 void setAlignment(Align Align) {
233 setSubclassData<AlignmentField>(Log2(Align));
234 }
235
236 /// Returns the ordering constraint of this load instruction.
237 AtomicOrdering getOrdering() const {
238 return getSubclassData<OrderingField>();
239 }
240 /// Sets the ordering constraint of this load instruction. May not be Release
241 /// or AcquireRelease.
242 void setOrdering(AtomicOrdering Ordering) {
243 setSubclassData<OrderingField>(Ordering);
244 }
245
246 /// Returns the synchronization scope ID of this load instruction.
247 SyncScope::ID getSyncScopeID() const {
248 return SSID;
249 }
250
251 /// Sets the synchronization scope ID of this load instruction.
252 void setSyncScopeID(SyncScope::ID SSID) {
253 this->SSID = SSID;
254 }
255
256 /// Sets the ordering constraint and the synchronization scope ID of this load
257 /// instruction.
258 void setAtomic(AtomicOrdering Ordering,
259 SyncScope::ID SSID = SyncScope::System) {
260 setOrdering(Ordering);
261 setSyncScopeID(SSID);
262 }
263
264 bool isSimple() const { return !isAtomic() && !isVolatile(); }
265
266 bool isUnordered() const {
267 return (getOrdering() == AtomicOrdering::NotAtomic ||
16
Assuming the condition is true
18
Returning the value 1, which participates in a condition later
268 getOrdering() == AtomicOrdering::Unordered) &&
269 !isVolatile();
17
Assuming the condition is true
270 }
271
272 Value *getPointerOperand() { return getOperand(0); }
273 const Value *getPointerOperand() const { return getOperand(0); }
274 static unsigned getPointerOperandIndex() { return 0U; }
275 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
276
277 /// Returns the address space of the pointer operand.
278 unsigned getPointerAddressSpace() const {
279 return getPointerOperandType()->getPointerAddressSpace();
280 }
281
282 // Methods for support type inquiry through isa, cast, and dyn_cast:
283 static bool classof(const Instruction *I) {
284 return I->getOpcode() == Instruction::Load;
285 }
286 static bool classof(const Value *V) {
287 return isa<Instruction>(V) && classof(cast<Instruction>(V));
288 }
289
290private:
291 // Shadow Instruction::setInstructionSubclassData with a private forwarding
292 // method so that subclasses cannot accidentally use it.
293 template <typename Bitfield>
294 void setSubclassData(typename Bitfield::Type Value) {
295 Instruction::setSubclassData<Bitfield>(Value);
296 }
297
298 /// The synchronization scope ID of this load instruction. Not quite enough
299 /// room in SubClassData for everything, so synchronization scope ID gets its
300 /// own field.
301 SyncScope::ID SSID;
302};
303
304//===----------------------------------------------------------------------===//
305// StoreInst Class
306//===----------------------------------------------------------------------===//
307
308/// An instruction for storing to memory.
309class StoreInst : public Instruction {
310 using VolatileField = BoolBitfieldElementT<0>;
311 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
312 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
313 static_assert(
314 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
315 "Bitfields must be contiguous");
316
317 void AssertOK();
318
319protected:
320 // Note: Instruction needs to be a friend here to call cloneImpl.
321 friend class Instruction;
322
323 StoreInst *cloneImpl() const;
324
325public:
326 StoreInst(Value *Val, Value *Ptr, Instruction *InsertBefore);
327 StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
328 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Instruction *InsertBefore);
329 StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
330 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
331 Instruction *InsertBefore = nullptr);
332 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
333 BasicBlock *InsertAtEnd);
334 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
335 AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
336 Instruction *InsertBefore = nullptr);
337 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
338 AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd);
339
340 // allocate space for exactly two operands
341 void *operator new(size_t S) { return User::operator new(S, 2); }
342 void operator delete(void *Ptr) { User::operator delete(Ptr); }
343
344 /// Return true if this is a store to a volatile memory location.
345 bool isVolatile() const { return getSubclassData<VolatileField>(); }
346
347 /// Specify whether this is a volatile store or not.
348 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
349
350 /// Transparently provide more efficient getOperand methods.
351 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
352
353 /// Return the alignment of the access that is being performed
354 /// FIXME: Remove this function once transition to Align is over.
355 /// Use getAlign() instead.
356 uint64_t getAlignment() const { return getAlign().value(); }
357
358 Align getAlign() const {
359 return Align(1ULL << (getSubclassData<AlignmentField>()));
360 }
361
362 void setAlignment(Align Align) {
363 setSubclassData<AlignmentField>(Log2(Align));
364 }
365
366 /// Returns the ordering constraint of this store instruction.
367 AtomicOrdering getOrdering() const {
368 return getSubclassData<OrderingField>();
369 }
370
371 /// Sets the ordering constraint of this store instruction. May not be
372 /// Acquire or AcquireRelease.
373 void setOrdering(AtomicOrdering Ordering) {
374 setSubclassData<OrderingField>(Ordering);
375 }
376
377 /// Returns the synchronization scope ID of this store instruction.
378 SyncScope::ID getSyncScopeID() const {
379 return SSID;
380 }
381
382 /// Sets the synchronization scope ID of this store instruction.
383 void setSyncScopeID(SyncScope::ID SSID) {
384 this->SSID = SSID;
385 }
386
387 /// Sets the ordering constraint and the synchronization scope ID of this
388 /// store instruction.
389 void setAtomic(AtomicOrdering Ordering,
390 SyncScope::ID SSID = SyncScope::System) {
391 setOrdering(Ordering);
392 setSyncScopeID(SSID);
393 }
394
395 bool isSimple() const { return !isAtomic() && !isVolatile(); }
396
397 bool isUnordered() const {
398 return (getOrdering() == AtomicOrdering::NotAtomic ||
399 getOrdering() == AtomicOrdering::Unordered) &&
400 !isVolatile();
401 }
402
403 Value *getValueOperand() { return getOperand(0); }
404 const Value *getValueOperand() const { return getOperand(0); }
405
406 Value *getPointerOperand() { return getOperand(1); }
407 const Value *getPointerOperand() const { return getOperand(1); }
408 static unsigned getPointerOperandIndex() { return 1U; }
409 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
410
411 /// Returns the address space of the pointer operand.
412 unsigned getPointerAddressSpace() const {
413 return getPointerOperandType()->getPointerAddressSpace();
414 }
415
416 // Methods for support type inquiry through isa, cast, and dyn_cast:
417 static bool classof(const Instruction *I) {
418 return I->getOpcode() == Instruction::Store;
419 }
420 static bool classof(const Value *V) {
421 return isa<Instruction>(V) && classof(cast<Instruction>(V));
422 }
423
424private:
425 // Shadow Instruction::setInstructionSubclassData with a private forwarding
426 // method so that subclasses cannot accidentally use it.
427 template <typename Bitfield>
428 void setSubclassData(typename Bitfield::Type Value) {
429 Instruction::setSubclassData<Bitfield>(Value);
430 }
431
432 /// The synchronization scope ID of this store instruction. Not quite enough
433 /// room in SubClassData for everything, so synchronization scope ID gets its
434 /// own field.
435 SyncScope::ID SSID;
436};
437
438template <>
439struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
440};
441
442DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)StoreInst::op_iterator StoreInst::op_begin() { return OperandTraits
<StoreInst>::op_begin(this); } StoreInst::const_op_iterator
StoreInst::op_begin() const { return OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this)); } StoreInst
::op_iterator StoreInst::op_end() { return OperandTraits<StoreInst
>::op_end(this); } StoreInst::const_op_iterator StoreInst::
op_end() const { return OperandTraits<StoreInst>::op_end
(const_cast<StoreInst*>(this)); } Value *StoreInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<StoreInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 442, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this))[i_nocapture
].get()); } void StoreInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<StoreInst>::operands(this) && "setOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 442, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<StoreInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned StoreInst::getNumOperands() const
{ return OperandTraits<StoreInst>::operands(this); } template
<int Idx_nocapture> Use &StoreInst::Op() { return this
->OpFrom<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &StoreInst::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
443
444//===----------------------------------------------------------------------===//
445// FenceInst Class
446//===----------------------------------------------------------------------===//
447
448/// An instruction for ordering other memory operations.
449class FenceInst : public Instruction {
450 using OrderingField = AtomicOrderingBitfieldElementT<0>;
451
452 void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
453
454protected:
455 // Note: Instruction needs to be a friend here to call cloneImpl.
456 friend class Instruction;
457
458 FenceInst *cloneImpl() const;
459
460public:
461 // Ordering may only be Acquire, Release, AcquireRelease, or
462 // SequentiallyConsistent.
463 FenceInst(LLVMContext &C, AtomicOrdering Ordering,
464 SyncScope::ID SSID = SyncScope::System,
465 Instruction *InsertBefore = nullptr);
466 FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
467 BasicBlock *InsertAtEnd);
468
469 // allocate space for exactly zero operands
470 void *operator new(size_t S) { return User::operator new(S, 0); }
471 void operator delete(void *Ptr) { User::operator delete(Ptr); }
472
473 /// Returns the ordering constraint of this fence instruction.
474 AtomicOrdering getOrdering() const {
475 return getSubclassData<OrderingField>();
476 }
477
478 /// Sets the ordering constraint of this fence instruction. May only be
479 /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
480 void setOrdering(AtomicOrdering Ordering) {
481 setSubclassData<OrderingField>(Ordering);
482 }
483
484 /// Returns the synchronization scope ID of this fence instruction.
485 SyncScope::ID getSyncScopeID() const {
486 return SSID;
487 }
488
489 /// Sets the synchronization scope ID of this fence instruction.
490 void setSyncScopeID(SyncScope::ID SSID) {
491 this->SSID = SSID;
492 }
493
494 // Methods for support type inquiry through isa, cast, and dyn_cast:
495 static bool classof(const Instruction *I) {
496 return I->getOpcode() == Instruction::Fence;
497 }
498 static bool classof(const Value *V) {
499 return isa<Instruction>(V) && classof(cast<Instruction>(V));
500 }
501
502private:
503 // Shadow Instruction::setInstructionSubclassData with a private forwarding
504 // method so that subclasses cannot accidentally use it.
505 template <typename Bitfield>
506 void setSubclassData(typename Bitfield::Type Value) {
507 Instruction::setSubclassData<Bitfield>(Value);
508 }
509
510 /// The synchronization scope ID of this fence instruction. Not quite enough
511 /// room in SubClassData for everything, so synchronization scope ID gets its
512 /// own field.
513 SyncScope::ID SSID;
514};
515
516//===----------------------------------------------------------------------===//
517// AtomicCmpXchgInst Class
518//===----------------------------------------------------------------------===//
519
520/// An instruction that atomically checks whether a
521/// specified value is in a memory location, and, if it is, stores a new value
522/// there. The value returned by this instruction is a pair containing the
523/// original value as first element, and an i1 indicating success (true) or
524/// failure (false) as second element.
525///
526class AtomicCmpXchgInst : public Instruction {
527 void Init(Value *Ptr, Value *Cmp, Value *NewVal, Align Align,
528 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
529 SyncScope::ID SSID);
530
531 template <unsigned Offset>
532 using AtomicOrderingBitfieldElement =
533 typename Bitfield::Element<AtomicOrdering, Offset, 3,
534 AtomicOrdering::LAST>;
535
536protected:
537 // Note: Instruction needs to be a friend here to call cloneImpl.
538 friend class Instruction;
539
540 AtomicCmpXchgInst *cloneImpl() const;
541
542public:
543 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
544 AtomicOrdering SuccessOrdering,
545 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
546 Instruction *InsertBefore = nullptr);
547 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
548 AtomicOrdering SuccessOrdering,
549 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
550 BasicBlock *InsertAtEnd);
551
552 // allocate space for exactly three operands
553 void *operator new(size_t S) { return User::operator new(S, 3); }
554 void operator delete(void *Ptr) { User::operator delete(Ptr); }
555
556 using VolatileField = BoolBitfieldElementT<0>;
557 using WeakField = BoolBitfieldElementT<VolatileField::NextBit>;
558 using SuccessOrderingField =
559 AtomicOrderingBitfieldElementT<WeakField::NextBit>;
560 using FailureOrderingField =
561 AtomicOrderingBitfieldElementT<SuccessOrderingField::NextBit>;
562 using AlignmentField =
563 AlignmentBitfieldElementT<FailureOrderingField::NextBit>;
564 static_assert(
565 Bitfield::areContiguous<VolatileField, WeakField, SuccessOrderingField,
566 FailureOrderingField, AlignmentField>(),
567 "Bitfields must be contiguous");
568
569 /// Return the alignment of the memory that is being allocated by the
570 /// instruction.
571 Align getAlign() const {
572 return Align(1ULL << getSubclassData<AlignmentField>());
573 }
574
575 void setAlignment(Align Align) {
576 setSubclassData<AlignmentField>(Log2(Align));
577 }
578
579 /// Return true if this is a cmpxchg from a volatile memory
580 /// location.
581 ///
582 bool isVolatile() const { return getSubclassData<VolatileField>(); }
583
584 /// Specify whether this is a volatile cmpxchg.
585 ///
586 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
587
588 /// Return true if this cmpxchg may spuriously fail.
589 bool isWeak() const { return getSubclassData<WeakField>(); }
590
591 void setWeak(bool IsWeak) { setSubclassData<WeakField>(IsWeak); }
592
593 /// Transparently provide more efficient getOperand methods.
594 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
595
596 static bool isValidSuccessOrdering(AtomicOrdering Ordering) {
597 return Ordering != AtomicOrdering::NotAtomic &&
598 Ordering != AtomicOrdering::Unordered;
599 }
600
601 static bool isValidFailureOrdering(AtomicOrdering Ordering) {
602 return Ordering != AtomicOrdering::NotAtomic &&
603 Ordering != AtomicOrdering::Unordered &&
604 Ordering != AtomicOrdering::AcquireRelease &&
605 Ordering != AtomicOrdering::Release;
606 }
607
608 /// Returns the success ordering constraint of this cmpxchg instruction.
609 AtomicOrdering getSuccessOrdering() const {
610 return getSubclassData<SuccessOrderingField>();
611 }
612
613 /// Sets the success ordering constraint of this cmpxchg instruction.
614 void setSuccessOrdering(AtomicOrdering Ordering) {
615 assert(isValidSuccessOrdering(Ordering) &&(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "llvm/include/llvm/IR/Instructions.h", 616, __extension__ __PRETTY_FUNCTION__
))
616 "invalid CmpXchg success ordering")(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "llvm/include/llvm/IR/Instructions.h", 616, __extension__ __PRETTY_FUNCTION__
))
;
617 setSubclassData<SuccessOrderingField>(Ordering);
618 }
619
620 /// Returns the failure ordering constraint of this cmpxchg instruction.
621 AtomicOrdering getFailureOrdering() const {
622 return getSubclassData<FailureOrderingField>();
623 }
624
625 /// Sets the failure ordering constraint of this cmpxchg instruction.
626 void setFailureOrdering(AtomicOrdering Ordering) {
627 assert(isValidFailureOrdering(Ordering) &&(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "llvm/include/llvm/IR/Instructions.h", 628, __extension__ __PRETTY_FUNCTION__
))
628 "invalid CmpXchg failure ordering")(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "llvm/include/llvm/IR/Instructions.h", 628, __extension__ __PRETTY_FUNCTION__
))
;
629 setSubclassData<FailureOrderingField>(Ordering);
630 }
631
632 /// Returns a single ordering which is at least as strong as both the
633 /// success and failure orderings for this cmpxchg.
634 AtomicOrdering getMergedOrdering() const {
635 if (getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
636 return AtomicOrdering::SequentiallyConsistent;
637 if (getFailureOrdering() == AtomicOrdering::Acquire) {
638 if (getSuccessOrdering() == AtomicOrdering::Monotonic)
639 return AtomicOrdering::Acquire;
640 if (getSuccessOrdering() == AtomicOrdering::Release)
641 return AtomicOrdering::AcquireRelease;
642 }
643 return getSuccessOrdering();
644 }
645
646 /// Returns the synchronization scope ID of this cmpxchg instruction.
647 SyncScope::ID getSyncScopeID() const {
648 return SSID;
649 }
650
651 /// Sets the synchronization scope ID of this cmpxchg instruction.
652 void setSyncScopeID(SyncScope::ID SSID) {
653 this->SSID = SSID;
654 }
655
656 Value *getPointerOperand() { return getOperand(0); }
657 const Value *getPointerOperand() const { return getOperand(0); }
658 static unsigned getPointerOperandIndex() { return 0U; }
659
660 Value *getCompareOperand() { return getOperand(1); }
661 const Value *getCompareOperand() const { return getOperand(1); }
662
663 Value *getNewValOperand() { return getOperand(2); }
664 const Value *getNewValOperand() const { return getOperand(2); }
665
666 /// Returns the address space of the pointer operand.
667 unsigned getPointerAddressSpace() const {
668 return getPointerOperand()->getType()->getPointerAddressSpace();
669 }
670
671 /// Returns the strongest permitted ordering on failure, given the
672 /// desired ordering on success.
673 ///
674 /// If the comparison in a cmpxchg operation fails, there is no atomic store
675 /// so release semantics cannot be provided. So this function drops explicit
676 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
677 /// operation would remain SequentiallyConsistent.
678 static AtomicOrdering
679 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
680 switch (SuccessOrdering) {
681 default:
682 llvm_unreachable("invalid cmpxchg success ordering")::llvm::llvm_unreachable_internal("invalid cmpxchg success ordering"
, "llvm/include/llvm/IR/Instructions.h", 682)
;
683 case AtomicOrdering::Release:
684 case AtomicOrdering::Monotonic:
685 return AtomicOrdering::Monotonic;
686 case AtomicOrdering::AcquireRelease:
687 case AtomicOrdering::Acquire:
688 return AtomicOrdering::Acquire;
689 case AtomicOrdering::SequentiallyConsistent:
690 return AtomicOrdering::SequentiallyConsistent;
691 }
692 }
693
694 // Methods for support type inquiry through isa, cast, and dyn_cast:
695 static bool classof(const Instruction *I) {
696 return I->getOpcode() == Instruction::AtomicCmpXchg;
697 }
698 static bool classof(const Value *V) {
699 return isa<Instruction>(V) && classof(cast<Instruction>(V));
700 }
701
702private:
703 // Shadow Instruction::setInstructionSubclassData with a private forwarding
704 // method so that subclasses cannot accidentally use it.
705 template <typename Bitfield>
706 void setSubclassData(typename Bitfield::Type Value) {
707 Instruction::setSubclassData<Bitfield>(Value);
708 }
709
710 /// The synchronization scope ID of this cmpxchg instruction. Not quite
711 /// enough room in SubClassData for everything, so synchronization scope ID
712 /// gets its own field.
713 SyncScope::ID SSID;
714};
715
716template <>
717struct OperandTraits<AtomicCmpXchgInst> :
718 public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
719};
720
721DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)AtomicCmpXchgInst::op_iterator AtomicCmpXchgInst::op_begin() {
return OperandTraits<AtomicCmpXchgInst>::op_begin(this
); } AtomicCmpXchgInst::const_op_iterator AtomicCmpXchgInst::
op_begin() const { return OperandTraits<AtomicCmpXchgInst>
::op_begin(const_cast<AtomicCmpXchgInst*>(this)); } AtomicCmpXchgInst
::op_iterator AtomicCmpXchgInst::op_end() { return OperandTraits
<AtomicCmpXchgInst>::op_end(this); } AtomicCmpXchgInst::
const_op_iterator AtomicCmpXchgInst::op_end() const { return OperandTraits
<AtomicCmpXchgInst>::op_end(const_cast<AtomicCmpXchgInst
*>(this)); } Value *AtomicCmpXchgInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 721, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<AtomicCmpXchgInst
>::op_begin(const_cast<AtomicCmpXchgInst*>(this))[i_nocapture
].get()); } void AtomicCmpXchgInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 721, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<AtomicCmpXchgInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned AtomicCmpXchgInst::getNumOperands
() const { return OperandTraits<AtomicCmpXchgInst>::operands
(this); } template <int Idx_nocapture> Use &AtomicCmpXchgInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &AtomicCmpXchgInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
722
723//===----------------------------------------------------------------------===//
724// AtomicRMWInst Class
725//===----------------------------------------------------------------------===//
726
727/// an instruction that atomically reads a memory location,
728/// combines it with another value, and then stores the result back. Returns
729/// the old value.
730///
731class AtomicRMWInst : public Instruction {
732protected:
733 // Note: Instruction needs to be a friend here to call cloneImpl.
734 friend class Instruction;
735
736 AtomicRMWInst *cloneImpl() const;
737
738public:
739 /// This enumeration lists the possible modifications atomicrmw can make. In
740 /// the descriptions, 'p' is the pointer to the instruction's memory location,
741 /// 'old' is the initial value of *p, and 'v' is the other value passed to the
742 /// instruction. These instructions always return 'old'.
743 enum BinOp : unsigned {
744 /// *p = v
745 Xchg,
746 /// *p = old + v
747 Add,
748 /// *p = old - v
749 Sub,
750 /// *p = old & v
751 And,
752 /// *p = ~(old & v)
753 Nand,
754 /// *p = old | v
755 Or,
756 /// *p = old ^ v
757 Xor,
758 /// *p = old >signed v ? old : v
759 Max,
760 /// *p = old <signed v ? old : v
761 Min,
762 /// *p = old >unsigned v ? old : v
763 UMax,
764 /// *p = old <unsigned v ? old : v
765 UMin,
766
767 /// *p = old + v
768 FAdd,
769
770 /// *p = old - v
771 FSub,
772
773 FIRST_BINOP = Xchg,
774 LAST_BINOP = FSub,
775 BAD_BINOP
776 };
777
778private:
779 template <unsigned Offset>
780 using AtomicOrderingBitfieldElement =
781 typename Bitfield::Element<AtomicOrdering, Offset, 3,
782 AtomicOrdering::LAST>;
783
784 template <unsigned Offset>
785 using BinOpBitfieldElement =
786 typename Bitfield::Element<BinOp, Offset, 4, BinOp::LAST_BINOP>;
787
788public:
789 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
790 AtomicOrdering Ordering, SyncScope::ID SSID,
791 Instruction *InsertBefore = nullptr);
792 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
793 AtomicOrdering Ordering, SyncScope::ID SSID,
794 BasicBlock *InsertAtEnd);
795
796 // allocate space for exactly two operands
797 void *operator new(size_t S) { return User::operator new(S, 2); }
798 void operator delete(void *Ptr) { User::operator delete(Ptr); }
799
800 using VolatileField = BoolBitfieldElementT<0>;
801 using AtomicOrderingField =
802 AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
803 using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
804 using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
805 static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
806 OperationField, AlignmentField>(),
807 "Bitfields must be contiguous");
808
809 BinOp getOperation() const { return getSubclassData<OperationField>(); }
810
811 static StringRef getOperationName(BinOp Op);
812
813 static bool isFPOperation(BinOp Op) {
814 switch (Op) {
815 case AtomicRMWInst::FAdd:
816 case AtomicRMWInst::FSub:
817 return true;
818 default:
819 return false;
820 }
821 }
822
823 void setOperation(BinOp Operation) {
824 setSubclassData<OperationField>(Operation);
825 }
826
827 /// Return the alignment of the memory that is being allocated by the
828 /// instruction.
829 Align getAlign() const {
830 return Align(1ULL << getSubclassData<AlignmentField>());
831 }
832
833 void setAlignment(Align Align) {
834 setSubclassData<AlignmentField>(Log2(Align));
835 }
836
837 /// Return true if this is a RMW on a volatile memory location.
838 ///
839 bool isVolatile() const { return getSubclassData<VolatileField>(); }
840
841 /// Specify whether this is a volatile RMW or not.
842 ///
843 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
844
845 /// Transparently provide more efficient getOperand methods.
846 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
847
848 /// Returns the ordering constraint of this rmw instruction.
849 AtomicOrdering getOrdering() const {
850 return getSubclassData<AtomicOrderingField>();
851 }
852
853 /// Sets the ordering constraint of this rmw instruction.
854 void setOrdering(AtomicOrdering Ordering) {
855 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "llvm/include/llvm/IR/Instructions.h", 856, __extension__ __PRETTY_FUNCTION__
))
856 "atomicrmw instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "llvm/include/llvm/IR/Instructions.h", 856, __extension__ __PRETTY_FUNCTION__
))
;
857 setSubclassData<AtomicOrderingField>(Ordering);
858 }
859
860 /// Returns the synchronization scope ID of this rmw instruction.
861 SyncScope::ID getSyncScopeID() const {
862 return SSID;
863 }
864
865 /// Sets the synchronization scope ID of this rmw instruction.
866 void setSyncScopeID(SyncScope::ID SSID) {
867 this->SSID = SSID;
868 }
869
870 Value *getPointerOperand() { return getOperand(0); }
871 const Value *getPointerOperand() const { return getOperand(0); }
872 static unsigned getPointerOperandIndex() { return 0U; }
873
874 Value *getValOperand() { return getOperand(1); }
875 const Value *getValOperand() const { return getOperand(1); }
876
877 /// Returns the address space of the pointer operand.
878 unsigned getPointerAddressSpace() const {
879 return getPointerOperand()->getType()->getPointerAddressSpace();
880 }
881
882 bool isFloatingPointOperation() const {
883 return isFPOperation(getOperation());
884 }
885
886 // Methods for support type inquiry through isa, cast, and dyn_cast:
887 static bool classof(const Instruction *I) {
888 return I->getOpcode() == Instruction::AtomicRMW;
889 }
890 static bool classof(const Value *V) {
891 return isa<Instruction>(V) && classof(cast<Instruction>(V));
892 }
893
894private:
895 void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
896 AtomicOrdering Ordering, SyncScope::ID SSID);
897
898 // Shadow Instruction::setInstructionSubclassData with a private forwarding
899 // method so that subclasses cannot accidentally use it.
900 template <typename Bitfield>
901 void setSubclassData(typename Bitfield::Type Value) {
902 Instruction::setSubclassData<Bitfield>(Value);
903 }
904
905 /// The synchronization scope ID of this rmw instruction. Not quite enough
906 /// room in SubClassData for everything, so synchronization scope ID gets its
907 /// own field.
908 SyncScope::ID SSID;
909};
910
911template <>
912struct OperandTraits<AtomicRMWInst>
913 : public FixedNumOperandTraits<AtomicRMWInst,2> {
914};
915
916DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)AtomicRMWInst::op_iterator AtomicRMWInst::op_begin() { return
OperandTraits<AtomicRMWInst>::op_begin(this); } AtomicRMWInst
::const_op_iterator AtomicRMWInst::op_begin() const { return OperandTraits
<AtomicRMWInst>::op_begin(const_cast<AtomicRMWInst*>
(this)); } AtomicRMWInst::op_iterator AtomicRMWInst::op_end()
{ return OperandTraits<AtomicRMWInst>::op_end(this); }
AtomicRMWInst::const_op_iterator AtomicRMWInst::op_end() const
{ return OperandTraits<AtomicRMWInst>::op_end(const_cast
<AtomicRMWInst*>(this)); } Value *AtomicRMWInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 916, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<AtomicRMWInst
>::op_begin(const_cast<AtomicRMWInst*>(this))[i_nocapture
].get()); } void AtomicRMWInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 916, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<AtomicRMWInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned AtomicRMWInst::getNumOperands()
const { return OperandTraits<AtomicRMWInst>::operands(
this); } template <int Idx_nocapture> Use &AtomicRMWInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &AtomicRMWInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
917
918//===----------------------------------------------------------------------===//
919// GetElementPtrInst Class
920//===----------------------------------------------------------------------===//
921
922// checkGEPType - Simple wrapper function to give a better assertion failure
923// message on bad indexes for a gep instruction.
924//
925inline Type *checkGEPType(Type *Ty) {
926 assert(Ty && "Invalid GetElementPtrInst indices for type!")(static_cast <bool> (Ty && "Invalid GetElementPtrInst indices for type!"
) ? void (0) : __assert_fail ("Ty && \"Invalid GetElementPtrInst indices for type!\""
, "llvm/include/llvm/IR/Instructions.h", 926, __extension__ __PRETTY_FUNCTION__
))
;
927 return Ty;
928}
929
930/// an instruction for type-safe pointer arithmetic to
931/// access elements of arrays and structs
932///
933class GetElementPtrInst : public Instruction {
934 Type *SourceElementType;
935 Type *ResultElementType;
936
937 GetElementPtrInst(const GetElementPtrInst &GEPI);
938
939 /// Constructors - Create a getelementptr instruction with a base pointer an
940 /// list of indices. The first ctor can optionally insert before an existing
941 /// instruction, the second appends the new instruction to the specified
942 /// BasicBlock.
943 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
944 ArrayRef<Value *> IdxList, unsigned Values,
945 const Twine &NameStr, Instruction *InsertBefore);
946 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
947 ArrayRef<Value *> IdxList, unsigned Values,
948 const Twine &NameStr, BasicBlock *InsertAtEnd);
949
950 void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
951
952protected:
953 // Note: Instruction needs to be a friend here to call cloneImpl.
954 friend class Instruction;
955
956 GetElementPtrInst *cloneImpl() const;
957
958public:
959 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
960 ArrayRef<Value *> IdxList,
961 const Twine &NameStr = "",
962 Instruction *InsertBefore = nullptr) {
963 unsigned Values = 1 + unsigned(IdxList.size());
964 assert(PointeeType && "Must specify element type")(static_cast <bool> (PointeeType && "Must specify element type"
) ? void (0) : __assert_fail ("PointeeType && \"Must specify element type\""
, "llvm/include/llvm/IR/Instructions.h", 964, __extension__ __PRETTY_FUNCTION__
))
;
965 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 966, __extension__ __PRETTY_FUNCTION__
))
966 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 966, __extension__ __PRETTY_FUNCTION__
))
;
967 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
968 NameStr, InsertBefore);
969 }
970
971 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
972 ArrayRef<Value *> IdxList,
973 const Twine &NameStr,
974 BasicBlock *InsertAtEnd) {
975 unsigned Values = 1 + unsigned(IdxList.size());
976 assert(PointeeType && "Must specify element type")(static_cast <bool> (PointeeType && "Must specify element type"
) ? void (0) : __assert_fail ("PointeeType && \"Must specify element type\""
, "llvm/include/llvm/IR/Instructions.h", 976, __extension__ __PRETTY_FUNCTION__
))
;
977 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 978, __extension__ __PRETTY_FUNCTION__
))
978 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "llvm/include/llvm/IR/Instructions.h", 978, __extension__ __PRETTY_FUNCTION__
))
;
979 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
980 NameStr, InsertAtEnd);
981 }
982
983 /// Create an "inbounds" getelementptr. See the documentation for the
984 /// "inbounds" flag in LangRef.html for details.
985 static GetElementPtrInst *
986 CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef<Value *> IdxList,
987 const Twine &NameStr = "",
988 Instruction *InsertBefore = nullptr) {
989 GetElementPtrInst *GEP =
990 Create(PointeeType, Ptr, IdxList, NameStr, InsertBefore);
991 GEP->setIsInBounds(true);
992 return GEP;
993 }
994
995 static GetElementPtrInst *CreateInBounds(Type *PointeeType, Value *Ptr,
996 ArrayRef<Value *> IdxList,
997 const Twine &NameStr,
998 BasicBlock *InsertAtEnd) {
999 GetElementPtrInst *GEP =
1000 Create(PointeeType, Ptr, IdxList, NameStr, InsertAtEnd);
1001 GEP->setIsInBounds(true);
1002 return GEP;
1003 }
1004
1005 /// Transparently provide more efficient getOperand methods.
1006 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1007
1008 Type *getSourceElementType() const { return SourceElementType; }
1009
1010 void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
1011 void setResultElementType(Type *Ty) { ResultElementType = Ty; }
1012
1013 Type *getResultElementType() const {
1014 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1015, __extension__ __PRETTY_FUNCTION__
))
1015 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1015, __extension__ __PRETTY_FUNCTION__
))
;
1016 return ResultElementType;
1017 }
1018
1019 /// Returns the address space of this instruction's pointer type.
1020 unsigned getAddressSpace() const {
1021 // Note that this is always the same as the pointer operand's address space
1022 // and that is cheaper to compute, so cheat here.
1023 return getPointerAddressSpace();
1024 }
1025
1026 /// Returns the result type of a getelementptr with the given source
1027 /// element type and indexes.
1028 ///
1029 /// Null is returned if the indices are invalid for the specified
1030 /// source element type.
1031 static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
1032 static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
1033 static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
1034
1035 /// Return the type of the element at the given index of an indexable
1036 /// type. This is equivalent to "getIndexedType(Agg, {Zero, Idx})".
1037 ///
1038 /// Returns null if the type can't be indexed, or the given index is not
1039 /// legal for the given type.
1040 static Type *getTypeAtIndex(Type *Ty, Value *Idx);
1041 static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
1042
1043 inline op_iterator idx_begin() { return op_begin()+1; }
1044 inline const_op_iterator idx_begin() const { return op_begin()+1; }
1045 inline op_iterator idx_end() { return op_end(); }
1046 inline const_op_iterator idx_end() const { return op_end(); }
1047
1048 inline iterator_range<op_iterator> indices() {
1049 return make_range(idx_begin(), idx_end());
1050 }
1051
1052 inline iterator_range<const_op_iterator> indices() const {
1053 return make_range(idx_begin(), idx_end());
1054 }
1055
1056 Value *getPointerOperand() {
1057 return getOperand(0);
1058 }
1059 const Value *getPointerOperand() const {
1060 return getOperand(0);
1061 }
1062 static unsigned getPointerOperandIndex() {
1063 return 0U; // get index for modifying correct operand.
1064 }
1065
1066 /// Method to return the pointer operand as a
1067 /// PointerType.
1068 Type *getPointerOperandType() const {
1069 return getPointerOperand()->getType();
1070 }
1071
1072 /// Returns the address space of the pointer operand.
1073 unsigned getPointerAddressSpace() const {
1074 return getPointerOperandType()->getPointerAddressSpace();
1075 }
1076
1077 /// Returns the pointer type returned by the GEP
1078 /// instruction, which may be a vector of pointers.
1079 static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
1080 ArrayRef<Value *> IdxList) {
1081 PointerType *OrigPtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
1082 unsigned AddrSpace = OrigPtrTy->getAddressSpace();
1083 Type *ResultElemTy = checkGEPType(getIndexedType(ElTy, IdxList));
1084 Type *PtrTy = OrigPtrTy->isOpaque()
1085 ? PointerType::get(OrigPtrTy->getContext(), AddrSpace)
1086 : PointerType::get(ResultElemTy, AddrSpace);
1087 // Vector GEP
1088 if (auto *PtrVTy = dyn_cast<VectorType>(Ptr->getType())) {
1089 ElementCount EltCount = PtrVTy->getElementCount();
1090 return VectorType::get(PtrTy, EltCount);
1091 }
1092 for (Value *Index : IdxList)
1093 if (auto *IndexVTy = dyn_cast<VectorType>(Index->getType())) {
1094 ElementCount EltCount = IndexVTy->getElementCount();
1095 return VectorType::get(PtrTy, EltCount);
1096 }
1097 // Scalar GEP
1098 return PtrTy;
1099 }
1100
1101 unsigned getNumIndices() const { // Note: always non-negative
1102 return getNumOperands() - 1;
1103 }
1104
1105 bool hasIndices() const {
1106 return getNumOperands() > 1;
1107 }
1108
1109 /// Return true if all of the indices of this GEP are
1110 /// zeros. If so, the result pointer and the first operand have the same
1111 /// value, just potentially different types.
1112 bool hasAllZeroIndices() const;
1113
1114 /// Return true if all of the indices of this GEP are
1115 /// constant integers. If so, the result pointer and the first operand have
1116 /// a constant offset between them.
1117 bool hasAllConstantIndices() const;
1118
1119 /// Set or clear the inbounds flag on this GEP instruction.
1120 /// See LangRef.html for the meaning of inbounds on a getelementptr.
1121 void setIsInBounds(bool b = true);
1122
1123 /// Determine whether the GEP has the inbounds flag.
1124 bool isInBounds() const;
1125
1126 /// Accumulate the constant address offset of this GEP if possible.
1127 ///
1128 /// This routine accepts an APInt into which it will accumulate the constant
1129 /// offset of this GEP if the GEP is in fact constant. If the GEP is not
1130 /// all-constant, it returns false and the value of the offset APInt is
1131 /// undefined (it is *not* preserved!). The APInt passed into this routine
1132 /// must be at least as wide as the IntPtr type for the address space of
1133 /// the base GEP pointer.
1134 bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
1135 bool collectOffset(const DataLayout &DL, unsigned BitWidth,
1136 MapVector<Value *, APInt> &VariableOffsets,
1137 APInt &ConstantOffset) const;
1138 // Methods for support type inquiry through isa, cast, and dyn_cast:
1139 static bool classof(const Instruction *I) {
1140 return (I->getOpcode() == Instruction::GetElementPtr);
1141 }
1142 static bool classof(const Value *V) {
1143 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1144 }
1145};
1146
1147template <>
1148struct OperandTraits<GetElementPtrInst> :
1149 public VariadicOperandTraits<GetElementPtrInst, 1> {
1150};
1151
1152GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1153 ArrayRef<Value *> IdxList, unsigned Values,
1154 const Twine &NameStr,
1155 Instruction *InsertBefore)
1156 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1157 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1158 Values, InsertBefore),
1159 SourceElementType(PointeeType),
1160 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1161 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1162, __extension__ __PRETTY_FUNCTION__
))
1162 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1162, __extension__ __PRETTY_FUNCTION__
))
;
1163 init(Ptr, IdxList, NameStr);
1164}
1165
1166GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1167 ArrayRef<Value *> IdxList, unsigned Values,
1168 const Twine &NameStr,
1169 BasicBlock *InsertAtEnd)
1170 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1171 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1172 Values, InsertAtEnd),
1173 SourceElementType(PointeeType),
1174 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1175 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1176, __extension__ __PRETTY_FUNCTION__
))
1176 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "llvm/include/llvm/IR/Instructions.h", 1176, __extension__ __PRETTY_FUNCTION__
))
;
1177 init(Ptr, IdxList, NameStr);
1178}
1179
1180DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)GetElementPtrInst::op_iterator GetElementPtrInst::op_begin() {
return OperandTraits<GetElementPtrInst>::op_begin(this
); } GetElementPtrInst::const_op_iterator GetElementPtrInst::
op_begin() const { return OperandTraits<GetElementPtrInst>
::op_begin(const_cast<GetElementPtrInst*>(this)); } GetElementPtrInst
::op_iterator GetElementPtrInst::op_end() { return OperandTraits
<GetElementPtrInst>::op_end(this); } GetElementPtrInst::
const_op_iterator GetElementPtrInst::op_end() const { return OperandTraits
<GetElementPtrInst>::op_end(const_cast<GetElementPtrInst
*>(this)); } Value *GetElementPtrInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<GetElementPtrInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1180, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<GetElementPtrInst
>::op_begin(const_cast<GetElementPtrInst*>(this))[i_nocapture
].get()); } void GetElementPtrInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<GetElementPtrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1180, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<GetElementPtrInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned GetElementPtrInst::getNumOperands
() const { return OperandTraits<GetElementPtrInst>::operands
(this); } template <int Idx_nocapture> Use &GetElementPtrInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &GetElementPtrInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1181
1182//===----------------------------------------------------------------------===//
1183// ICmpInst Class
1184//===----------------------------------------------------------------------===//
1185
1186/// This instruction compares its operands according to the predicate given
1187/// to the constructor. It only operates on integers or pointers. The operands
1188/// must be identical types.
1189/// Represent an integer comparison operator.
1190class ICmpInst: public CmpInst {
1191 void AssertOK() {
1192 assert(isIntPredicate() &&(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "llvm/include/llvm/IR/Instructions.h", 1193, __extension__ __PRETTY_FUNCTION__
))
1193 "Invalid ICmp predicate value")(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "llvm/include/llvm/IR/Instructions.h", 1193, __extension__ __PRETTY_FUNCTION__
))
;
1194 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1195, __extension__ __PRETTY_FUNCTION__
))
1195 "Both operands to ICmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1195, __extension__ __PRETTY_FUNCTION__
))
;
1196 // Check that the operands are the right type
1197 assert((getOperand(0)->getType()->isIntOrIntVectorTy() ||(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1199, __extension__ __PRETTY_FUNCTION__
))
1198 getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1199, __extension__ __PRETTY_FUNCTION__
))
1199 "Invalid operand types for ICmp instruction")(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1199, __extension__ __PRETTY_FUNCTION__
))
;
1200 }
1201
1202protected:
1203 // Note: Instruction needs to be a friend here to call cloneImpl.
1204 friend class Instruction;
1205
1206 /// Clone an identical ICmpInst
1207 ICmpInst *cloneImpl() const;
1208
1209public:
1210 /// Constructor with insert-before-instruction semantics.
1211 ICmpInst(
1212 Instruction *InsertBefore, ///< Where to insert
1213 Predicate pred, ///< The predicate to use for the comparison
1214 Value *LHS, ///< The left-hand-side of the expression
1215 Value *RHS, ///< The right-hand-side of the expression
1216 const Twine &NameStr = "" ///< Name of the instruction
1217 ) : CmpInst(makeCmpResultType(LHS->getType()),
1218 Instruction::ICmp, pred, LHS, RHS, NameStr,
1219 InsertBefore) {
1220#ifndef NDEBUG
1221 AssertOK();
1222#endif
1223 }
1224
1225 /// Constructor with insert-at-end semantics.
1226 ICmpInst(
1227 BasicBlock &InsertAtEnd, ///< Block to insert into.
1228 Predicate pred, ///< The predicate to use for the comparison
1229 Value *LHS, ///< The left-hand-side of the expression
1230 Value *RHS, ///< The right-hand-side of the expression
1231 const Twine &NameStr = "" ///< Name of the instruction
1232 ) : CmpInst(makeCmpResultType(LHS->getType()),
1233 Instruction::ICmp, pred, LHS, RHS, NameStr,
1234 &InsertAtEnd) {
1235#ifndef NDEBUG
1236 AssertOK();
1237#endif
1238 }
1239
1240 /// Constructor with no-insertion semantics
1241 ICmpInst(
1242 Predicate pred, ///< The predicate to use for the comparison
1243 Value *LHS, ///< The left-hand-side of the expression
1244 Value *RHS, ///< The right-hand-side of the expression
1245 const Twine &NameStr = "" ///< Name of the instruction
1246 ) : CmpInst(makeCmpResultType(LHS->getType()),
1247 Instruction::ICmp, pred, LHS, RHS, NameStr) {
1248#ifndef NDEBUG
1249 AssertOK();
1250#endif
1251 }
1252
1253 /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
1254 /// @returns the predicate that would be the result if the operand were
1255 /// regarded as signed.
1256 /// Return the signed version of the predicate
1257 Predicate getSignedPredicate() const {
1258 return getSignedPredicate(getPredicate());
1259 }
1260
1261 /// This is a static version that you can use without an instruction.
1262 /// Return the signed version of the predicate.
1263 static Predicate getSignedPredicate(Predicate pred);
1264
1265 /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
1266 /// @returns the predicate that would be the result if the operand were
1267 /// regarded as unsigned.
1268 /// Return the unsigned version of the predicate
1269 Predicate getUnsignedPredicate() const {
1270 return getUnsignedPredicate(getPredicate());
1271 }
1272
1273 /// This is a static version that you can use without an instruction.
1274 /// Return the unsigned version of the predicate.
1275 static Predicate getUnsignedPredicate(Predicate pred);
1276
1277 /// Return true if this predicate is either EQ or NE. This also
1278 /// tests for commutativity.
1279 static bool isEquality(Predicate P) {
1280 return P == ICMP_EQ || P == ICMP_NE;
1281 }
1282
1283 /// Return true if this predicate is either EQ or NE. This also
1284 /// tests for commutativity.
1285 bool isEquality() const {
1286 return isEquality(getPredicate());
1287 }
1288
1289 /// @returns true if the predicate of this ICmpInst is commutative
1290 /// Determine if this relation is commutative.
1291 bool isCommutative() const { return isEquality(); }
1292
1293 /// Return true if the predicate is relational (not EQ or NE).
1294 ///
1295 bool isRelational() const {
1296 return !isEquality();
1297 }
1298
1299 /// Return true if the predicate is relational (not EQ or NE).
1300 ///
1301 static bool isRelational(Predicate P) {
1302 return !isEquality(P);
1303 }
1304
1305 /// Return true if the predicate is SGT or UGT.
1306 ///
1307 static bool isGT(Predicate P) {
1308 return P == ICMP_SGT || P == ICMP_UGT;
1309 }
1310
1311 /// Return true if the predicate is SLT or ULT.
1312 ///
1313 static bool isLT(Predicate P) {
1314 return P == ICMP_SLT || P == ICMP_ULT;
1315 }
1316
1317 /// Return true if the predicate is SGE or UGE.
1318 ///
1319 static bool isGE(Predicate P) {
1320 return P == ICMP_SGE || P == ICMP_UGE;
1321 }
1322
1323 /// Return true if the predicate is SLE or ULE.
1324 ///
1325 static bool isLE(Predicate P) {
1326 return P == ICMP_SLE || P == ICMP_ULE;
1327 }
1328
1329 /// Returns the sequence of all ICmp predicates.
1330 ///
1331 static auto predicates() { return ICmpPredicates(); }
1332
1333 /// Exchange the two operands to this instruction in such a way that it does
1334 /// not modify the semantics of the instruction. The predicate value may be
1335 /// changed to retain the same result if the predicate is order dependent
1336 /// (e.g. ult).
1337 /// Swap operands and adjust predicate.
1338 void swapOperands() {
1339 setPredicate(getSwappedPredicate());
1340 Op<0>().swap(Op<1>());
1341 }
1342
1343 /// Return result of `LHS Pred RHS` comparison.
1344 static bool compare(const APInt &LHS, const APInt &RHS,
1345 ICmpInst::Predicate Pred);
1346
1347 // Methods for support type inquiry through isa, cast, and dyn_cast:
1348 static bool classof(const Instruction *I) {
1349 return I->getOpcode() == Instruction::ICmp;
1350 }
1351 static bool classof(const Value *V) {
1352 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1353 }
1354};
1355
1356//===----------------------------------------------------------------------===//
1357// FCmpInst Class
1358//===----------------------------------------------------------------------===//
1359
1360/// This instruction compares its operands according to the predicate given
1361/// to the constructor. It only operates on floating point values or packed
1362/// vectors of floating point values. The operands must be identical types.
1363/// Represents a floating point comparison operator.
1364class FCmpInst: public CmpInst {
1365 void AssertOK() {
1366 assert(isFPPredicate() && "Invalid FCmp predicate value")(static_cast <bool> (isFPPredicate() && "Invalid FCmp predicate value"
) ? void (0) : __assert_fail ("isFPPredicate() && \"Invalid FCmp predicate value\""
, "llvm/include/llvm/IR/Instructions.h", 1366, __extension__ __PRETTY_FUNCTION__
))
;
1367 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1368, __extension__ __PRETTY_FUNCTION__
))
1368 "Both operands to FCmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "llvm/include/llvm/IR/Instructions.h", 1368, __extension__ __PRETTY_FUNCTION__
))
;
1369 // Check that the operands are the right type
1370 assert(getOperand(0)->getType()->isFPOrFPVectorTy() &&(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1371, __extension__ __PRETTY_FUNCTION__
))
1371 "Invalid operand types for FCmp instruction")(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "llvm/include/llvm/IR/Instructions.h", 1371, __extension__ __PRETTY_FUNCTION__
))
;
1372 }
1373
1374protected:
1375 // Note: Instruction needs to be a friend here to call cloneImpl.
1376 friend class Instruction;
1377
1378 /// Clone an identical FCmpInst
1379 FCmpInst *cloneImpl() const;
1380
1381public:
1382 /// Constructor with insert-before-instruction semantics.
1383 FCmpInst(
1384 Instruction *InsertBefore, ///< Where to insert
1385 Predicate pred, ///< The predicate to use for the comparison
1386 Value *LHS, ///< The left-hand-side of the expression
1387 Value *RHS, ///< The right-hand-side of the expression
1388 const Twine &NameStr = "" ///< Name of the instruction
1389 ) : CmpInst(makeCmpResultType(LHS->getType()),
1390 Instruction::FCmp, pred, LHS, RHS, NameStr,
1391 InsertBefore) {
1392 AssertOK();
1393 }
1394
1395 /// Constructor with insert-at-end semantics.
1396 FCmpInst(
1397 BasicBlock &InsertAtEnd, ///< Block to insert into.
1398 Predicate pred, ///< The predicate to use for the comparison
1399 Value *LHS, ///< The left-hand-side of the expression
1400 Value *RHS, ///< The right-hand-side of the expression
1401 const Twine &NameStr = "" ///< Name of the instruction
1402 ) : CmpInst(makeCmpResultType(LHS->getType()),
1403 Instruction::FCmp, pred, LHS, RHS, NameStr,
1404 &InsertAtEnd) {
1405 AssertOK();
1406 }
1407
1408 /// Constructor with no-insertion semantics
1409 FCmpInst(
1410 Predicate Pred, ///< The predicate to use for the comparison
1411 Value *LHS, ///< The left-hand-side of the expression
1412 Value *RHS, ///< The right-hand-side of the expression
1413 const Twine &NameStr = "", ///< Name of the instruction
1414 Instruction *FlagsSource = nullptr
1415 ) : CmpInst(makeCmpResultType(LHS->getType()), Instruction::FCmp, Pred, LHS,
1416 RHS, NameStr, nullptr, FlagsSource) {
1417 AssertOK();
1418 }
1419
1420 /// @returns true if the predicate of this instruction is EQ or NE.
1421 /// Determine if this is an equality predicate.
1422 static bool isEquality(Predicate Pred) {
1423 return Pred == FCMP_OEQ || Pred == FCMP_ONE || Pred == FCMP_UEQ ||
1424 Pred == FCMP_UNE;
1425 }
1426
1427 /// @returns true if the predicate of this instruction is EQ or NE.
1428 /// Determine if this is an equality predicate.
1429 bool isEquality() const { return isEquality(getPredicate()); }
1430
1431 /// @returns true if the predicate of this instruction is commutative.
1432 /// Determine if this is a commutative predicate.
1433 bool isCommutative() const {
1434 return isEquality() ||
1435 getPredicate() == FCMP_FALSE ||
1436 getPredicate() == FCMP_TRUE ||
1437 getPredicate() == FCMP_ORD ||
1438 getPredicate() == FCMP_UNO;
1439 }
1440
1441 /// @returns true if the predicate is relational (not EQ or NE).
1442 /// Determine if this a relational predicate.
1443 bool isRelational() const { return !isEquality(); }
1444
1445 /// Exchange the two operands to this instruction in such a way that it does
1446 /// not modify the semantics of the instruction. The predicate value may be
1447 /// changed to retain the same result if the predicate is order dependent
1448 /// (e.g. ult).
1449 /// Swap operands and adjust predicate.
1450 void swapOperands() {
1451 setPredicate(getSwappedPredicate());
1452 Op<0>().swap(Op<1>());
1453 }
1454
1455 /// Returns the sequence of all FCmp predicates.
1456 ///
1457 static auto predicates() { return FCmpPredicates(); }
1458
1459 /// Return result of `LHS Pred RHS` comparison.
1460 static bool compare(const APFloat &LHS, const APFloat &RHS,
1461 FCmpInst::Predicate Pred);
1462
1463 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1464 static bool classof(const Instruction *I) {
1465 return I->getOpcode() == Instruction::FCmp;
1466 }
1467 static bool classof(const Value *V) {
1468 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1469 }
1470};
1471
1472//===----------------------------------------------------------------------===//
1473/// This class represents a function call, abstracting a target
1474/// machine's calling convention. This class uses low bit of the SubClassData
1475/// field to indicate whether or not this is a tail call. The rest of the bits
1476/// hold the calling convention of the call.
1477///
1478class CallInst : public CallBase {
1479 CallInst(const CallInst &CI);
1480
1481 /// Construct a CallInst given a range of arguments.
1482 /// Construct a CallInst from a range of arguments
1483 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1484 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1485 Instruction *InsertBefore);
1486
1487 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1488 const Twine &NameStr, Instruction *InsertBefore)
1489 : CallInst(Ty, Func, Args, None, NameStr, InsertBefore) {}
1490
1491 /// Construct a CallInst given a range of arguments.
1492 /// Construct a CallInst from a range of arguments
1493 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1494 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1495 BasicBlock *InsertAtEnd);
1496
1497 explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
1498 Instruction *InsertBefore);
1499
1500 CallInst(FunctionType *ty, Value *F, const Twine &NameStr,
1501 BasicBlock *InsertAtEnd);
1502
1503 void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
1504 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
1505 void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
1506
1507 /// Compute the number of operands to allocate.
1508 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
1509 // We need one operand for the called function, plus the input operand
1510 // counts provided.
1511 return 1 + NumArgs + NumBundleInputs;
1512 }
1513
1514protected:
1515 // Note: Instruction needs to be a friend here to call cloneImpl.
1516 friend class Instruction;
1517
1518 CallInst *cloneImpl() const;
1519
1520public:
1521 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
1522 Instruction *InsertBefore = nullptr) {
1523 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertBefore);
1524 }
1525
1526 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1527 const Twine &NameStr,
1528 Instruction *InsertBefore = nullptr) {
1529 return new (ComputeNumOperands(Args.size()))
1530 CallInst(Ty, Func, Args, None, NameStr, InsertBefore);
1531 }
1532
1533 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1534 ArrayRef<OperandBundleDef> Bundles = None,
1535 const Twine &NameStr = "",
1536 Instruction *InsertBefore = nullptr) {
1537 const int NumOperands =
1538 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1539 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1540
1541 return new (NumOperands, DescriptorBytes)
1542 CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
1543 }
1544
1545 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr,
1546 BasicBlock *InsertAtEnd) {
1547 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertAtEnd);
1548 }
1549
1550 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1551 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1552 return new (ComputeNumOperands(Args.size()))
1553 CallInst(Ty, Func, Args, None, NameStr, InsertAtEnd);
1554 }
1555
1556 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1557 ArrayRef<OperandBundleDef> Bundles,
1558 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1559 const int NumOperands =
1560 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1561 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1562
1563 return new (NumOperands, DescriptorBytes)
1564 CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd);
1565 }
1566
1567 static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "",
1568 Instruction *InsertBefore = nullptr) {
1569 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1570 InsertBefore);
1571 }
1572
1573 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1574 ArrayRef<OperandBundleDef> Bundles = None,
1575 const Twine &NameStr = "",
1576 Instruction *InsertBefore = nullptr) {
1577 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1578 NameStr, InsertBefore);
1579 }
1580
1581 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1582 const Twine &NameStr,
1583 Instruction *InsertBefore = nullptr) {
1584 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1585 InsertBefore);
1586 }
1587
1588 static CallInst *Create(FunctionCallee Func, const Twine &NameStr,
1589 BasicBlock *InsertAtEnd) {
1590 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1591 InsertAtEnd);
1592 }
1593
1594 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1595 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1596 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1597 InsertAtEnd);
1598 }
1599
1600 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1601 ArrayRef<OperandBundleDef> Bundles,
1602 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1603 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1604 NameStr, InsertAtEnd);
1605 }
1606
1607 /// Create a clone of \p CI with a different set of operand bundles and
1608 /// insert it before \p InsertPt.
1609 ///
1610 /// The returned call instruction is identical \p CI in every way except that
1611 /// the operand bundles for the new instruction are set to the operand bundles
1612 /// in \p Bundles.
1613 static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
1614 Instruction *InsertPt = nullptr);
1615
1616 /// Generate the IR for a call to malloc:
1617 /// 1. Compute the malloc call's argument as the specified type's size,
1618 /// possibly multiplied by the array size if the array size is not
1619 /// constant 1.
1620 /// 2. Call malloc with that argument.
1621 /// 3. Bitcast the result of the malloc call to the specified type.
1622 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1623 Type *AllocTy, Value *AllocSize,
1624 Value *ArraySize = nullptr,
1625 Function *MallocF = nullptr,
1626 const Twine &Name = "");
1627 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1628 Type *AllocTy, Value *AllocSize,
1629 Value *ArraySize = nullptr,
1630 Function *MallocF = nullptr,
1631 const Twine &Name = "");
1632 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1633 Type *AllocTy, Value *AllocSize,
1634 Value *ArraySize = nullptr,
1635 ArrayRef<OperandBundleDef> Bundles = None,
1636 Function *MallocF = nullptr,
1637 const Twine &Name = "");
1638 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1639 Type *AllocTy, Value *AllocSize,
1640 Value *ArraySize = nullptr,
1641 ArrayRef<OperandBundleDef> Bundles = None,
1642 Function *MallocF = nullptr,
1643 const Twine &Name = "");
1644 /// Generate the IR for a call to the builtin free function.
1645 static Instruction *CreateFree(Value *Source, Instruction *InsertBefore);
1646 static Instruction *CreateFree(Value *Source, BasicBlock *InsertAtEnd);
1647 static Instruction *CreateFree(Value *Source,
1648 ArrayRef<OperandBundleDef> Bundles,
1649 Instruction *InsertBefore);
1650 static Instruction *CreateFree(Value *Source,
1651 ArrayRef<OperandBundleDef> Bundles,
1652 BasicBlock *InsertAtEnd);
1653
1654 // Note that 'musttail' implies 'tail'.
1655 enum TailCallKind : unsigned {
1656 TCK_None = 0,
1657 TCK_Tail = 1,
1658 TCK_MustTail = 2,
1659 TCK_NoTail = 3,
1660 TCK_LAST = TCK_NoTail
1661 };
1662
1663 using TailCallKindField = Bitfield::Element<TailCallKind, 0, 2, TCK_LAST>;
1664 static_assert(
1665 Bitfield::areContiguous<TailCallKindField, CallBase::CallingConvField>(),
1666 "Bitfields must be contiguous");
1667
1668 TailCallKind getTailCallKind() const {
1669 return getSubclassData<TailCallKindField>();
1670 }
1671
1672 bool isTailCall() const {
1673 TailCallKind Kind = getTailCallKind();
1674 return Kind == TCK_Tail || Kind == TCK_MustTail;
1675 }
1676
1677 bool isMustTailCall() const { return getTailCallKind() == TCK_MustTail; }
1678
1679 bool isNoTailCall() const { return getTailCallKind() == TCK_NoTail; }
1680
1681 void setTailCallKind(TailCallKind TCK) {
1682 setSubclassData<TailCallKindField>(TCK);
1683 }
1684
1685 void setTailCall(bool IsTc = true) {
1686 setTailCallKind(IsTc ? TCK_Tail : TCK_None);
1687 }
1688
1689 /// Return true if the call can return twice
1690 bool canReturnTwice() const { return hasFnAttr(Attribute::ReturnsTwice); }
1691 void setCanReturnTwice() { addFnAttr(Attribute::ReturnsTwice); }
1692
1693 // Methods for support type inquiry through isa, cast, and dyn_cast:
1694 static bool classof(const Instruction *I) {
1695 return I->getOpcode() == Instruction::Call;
1696 }
1697 static bool classof(const Value *V) {
1698 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1699 }
1700
1701 /// Updates profile metadata by scaling it by \p S / \p T.
1702 void updateProfWeight(uint64_t S, uint64_t T);
1703
1704private:
1705 // Shadow Instruction::setInstructionSubclassData with a private forwarding
1706 // method so that subclasses cannot accidentally use it.
1707 template <typename Bitfield>
1708 void setSubclassData(typename Bitfield::Type Value) {
1709 Instruction::setSubclassData<Bitfield>(Value);
1710 }
1711};
1712
1713CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1714 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1715 BasicBlock *InsertAtEnd)
1716 : CallBase(Ty->getReturnType(), Instruction::Call,
1717 OperandTraits<CallBase>::op_end(this) -
1718 (Args.size() + CountBundleInputs(Bundles) + 1),
1719 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1720 InsertAtEnd) {
1721 init(Ty, Func, Args, Bundles, NameStr);
1722}
1723
1724CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1725 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1726 Instruction *InsertBefore)
1727 : CallBase(Ty->getReturnType(), Instruction::Call,
1728 OperandTraits<CallBase>::op_end(this) -
1729 (Args.size() + CountBundleInputs(Bundles) + 1),
1730 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1731 InsertBefore) {
1732 init(Ty, Func, Args, Bundles, NameStr);
1733}
1734
1735//===----------------------------------------------------------------------===//
1736// SelectInst Class
1737//===----------------------------------------------------------------------===//
1738
1739/// This class represents the LLVM 'select' instruction.
1740///
1741class SelectInst : public Instruction {
1742 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1743 Instruction *InsertBefore)
1744 : Instruction(S1->getType(), Instruction::Select,
1745 &Op<0>(), 3, InsertBefore) {
1746 init(C, S1, S2);
1747 setName(NameStr);
1748 }
1749
1750 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1751 BasicBlock *InsertAtEnd)
1752 : Instruction(S1->getType(), Instruction::Select,
1753 &Op<0>(), 3, InsertAtEnd) {
1754 init(C, S1, S2);
1755 setName(NameStr);
1756 }
1757
1758 void init(Value *C, Value *S1, Value *S2) {
1759 assert(!areInvalidOperands(C, S1, S2) && "Invalid operands for select")(static_cast <bool> (!areInvalidOperands(C, S1, S2) &&
"Invalid operands for select") ? void (0) : __assert_fail ("!areInvalidOperands(C, S1, S2) && \"Invalid operands for select\""
, "llvm/include/llvm/IR/Instructions.h", 1759, __extension__ __PRETTY_FUNCTION__
))
;
1760 Op<0>() = C;
1761 Op<1>() = S1;
1762 Op<2>() = S2;
1763 }
1764
1765protected:
1766 // Note: Instruction needs to be a friend here to call cloneImpl.
1767 friend class Instruction;
1768
1769 SelectInst *cloneImpl() const;
1770
1771public:
1772 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1773 const Twine &NameStr = "",
1774 Instruction *InsertBefore = nullptr,
1775 Instruction *MDFrom = nullptr) {
1776 SelectInst *Sel = new(3) SelectInst(C, S1, S2, NameStr, InsertBefore);
1777 if (MDFrom)
1778 Sel->copyMetadata(*MDFrom);
1779 return Sel;
1780 }
1781
1782 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1783 const Twine &NameStr,
1784 BasicBlock *InsertAtEnd) {
1785 return new(3) SelectInst(C, S1, S2, NameStr, InsertAtEnd);
1786 }
1787
1788 const Value *getCondition() const { return Op<0>(); }
1789 const Value *getTrueValue() const { return Op<1>(); }
1790 const Value *getFalseValue() const { return Op<2>(); }
1791 Value *getCondition() { return Op<0>(); }
1792 Value *getTrueValue() { return Op<1>(); }
1793 Value *getFalseValue() { return Op<2>(); }
1794
1795 void setCondition(Value *V) { Op<0>() = V; }
1796 void setTrueValue(Value *V) { Op<1>() = V; }
1797 void setFalseValue(Value *V) { Op<2>() = V; }
1798
1799 /// Swap the true and false values of the select instruction.
1800 /// This doesn't swap prof metadata.
1801 void swapValues() { Op<1>().swap(Op<2>()); }
1802
1803 /// Return a string if the specified operands are invalid
1804 /// for a select operation, otherwise return null.
1805 static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
1806
1807 /// Transparently provide more efficient getOperand methods.
1808 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1809
1810 OtherOps getOpcode() const {
1811 return static_cast<OtherOps>(Instruction::getOpcode());
1812 }
1813
1814 // Methods for support type inquiry through isa, cast, and dyn_cast:
1815 static bool classof(const Instruction *I) {
1816 return I->getOpcode() == Instruction::Select;
1817 }
1818 static bool classof(const Value *V) {
1819 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1820 }
1821};
1822
1823template <>
1824struct OperandTraits<SelectInst> : public FixedNumOperandTraits<SelectInst, 3> {
1825};
1826
1827DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)SelectInst::op_iterator SelectInst::op_begin() { return OperandTraits
<SelectInst>::op_begin(this); } SelectInst::const_op_iterator
SelectInst::op_begin() const { return OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this)); } SelectInst
::op_iterator SelectInst::op_end() { return OperandTraits<
SelectInst>::op_end(this); } SelectInst::const_op_iterator
SelectInst::op_end() const { return OperandTraits<SelectInst
>::op_end(const_cast<SelectInst*>(this)); } Value *SelectInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SelectInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1827, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this))[i_nocapture
].get()); } void SelectInst::setOperand(unsigned i_nocapture,
Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<SelectInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1827, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<SelectInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned SelectInst::getNumOperands() const
{ return OperandTraits<SelectInst>::operands(this); } template
<int Idx_nocapture> Use &SelectInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &SelectInst::Op() const { return
this->OpFrom<Idx_nocapture>(this); }
1828
1829//===----------------------------------------------------------------------===//
1830// VAArgInst Class
1831//===----------------------------------------------------------------------===//
1832
1833/// This class represents the va_arg llvm instruction, which returns
1834/// an argument of the specified type given a va_list and increments that list
1835///
1836class VAArgInst : public UnaryInstruction {
1837protected:
1838 // Note: Instruction needs to be a friend here to call cloneImpl.
1839 friend class Instruction;
1840
1841 VAArgInst *cloneImpl() const;
1842
1843public:
1844 VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
1845 Instruction *InsertBefore = nullptr)
1846 : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
1847 setName(NameStr);
1848 }
1849
1850 VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
1851 BasicBlock *InsertAtEnd)
1852 : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
1853 setName(NameStr);
1854 }
1855
1856 Value *getPointerOperand() { return getOperand(0); }
1857 const Value *getPointerOperand() const { return getOperand(0); }
1858 static unsigned getPointerOperandIndex() { return 0U; }
1859
1860 // Methods for support type inquiry through isa, cast, and dyn_cast:
1861 static bool classof(const Instruction *I) {
1862 return I->getOpcode() == VAArg;
1863 }
1864 static bool classof(const Value *V) {
1865 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1866 }
1867};
1868
1869//===----------------------------------------------------------------------===//
1870// ExtractElementInst Class
1871//===----------------------------------------------------------------------===//
1872
1873/// This instruction extracts a single (scalar)
1874/// element from a VectorType value
1875///
1876class ExtractElementInst : public Instruction {
1877 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
1878 Instruction *InsertBefore = nullptr);
1879 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
1880 BasicBlock *InsertAtEnd);
1881
1882protected:
1883 // Note: Instruction needs to be a friend here to call cloneImpl.
1884 friend class Instruction;
1885
1886 ExtractElementInst *cloneImpl() const;
1887
1888public:
1889 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1890 const Twine &NameStr = "",
1891 Instruction *InsertBefore = nullptr) {
1892 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertBefore);
1893 }
1894
1895 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1896 const Twine &NameStr,
1897 BasicBlock *InsertAtEnd) {
1898 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertAtEnd);
1899 }
1900
1901 /// Return true if an extractelement instruction can be
1902 /// formed with the specified operands.
1903 static bool isValidOperands(const Value *Vec, const Value *Idx);
1904
1905 Value *getVectorOperand() { return Op<0>(); }
1906 Value *getIndexOperand() { return Op<1>(); }
1907 const Value *getVectorOperand() const { return Op<0>(); }
1908 const Value *getIndexOperand() const { return Op<1>(); }
1909
1910 VectorType *getVectorOperandType() const {
1911 return cast<VectorType>(getVectorOperand()->getType());
1912 }
1913
1914 /// Transparently provide more efficient getOperand methods.
1915 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1916
1917 // Methods for support type inquiry through isa, cast, and dyn_cast:
1918 static bool classof(const Instruction *I) {
1919 return I->getOpcode() == Instruction::ExtractElement;
1920 }
1921 static bool classof(const Value *V) {
1922 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1923 }
1924};
1925
1926template <>
1927struct OperandTraits<ExtractElementInst> :
1928 public FixedNumOperandTraits<ExtractElementInst, 2> {
1929};
1930
1931DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)ExtractElementInst::op_iterator ExtractElementInst::op_begin(
) { return OperandTraits<ExtractElementInst>::op_begin(
this); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_begin() const { return OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this)); }
ExtractElementInst::op_iterator ExtractElementInst::op_end()
{ return OperandTraits<ExtractElementInst>::op_end(this
); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_end() const { return OperandTraits<ExtractElementInst
>::op_end(const_cast<ExtractElementInst*>(this)); } Value
*ExtractElementInst::getOperand(unsigned i_nocapture) const {
(static_cast <bool> (i_nocapture < OperandTraits<
ExtractElementInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1931, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this))[i_nocapture
].get()); } void ExtractElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ExtractElementInst>::operands(this)
&& "setOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1931, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ExtractElementInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ExtractElementInst::getNumOperands
() const { return OperandTraits<ExtractElementInst>::operands
(this); } template <int Idx_nocapture> Use &ExtractElementInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &ExtractElementInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1932
1933//===----------------------------------------------------------------------===//
1934// InsertElementInst Class
1935//===----------------------------------------------------------------------===//
1936
1937/// This instruction inserts a single (scalar)
1938/// element into a VectorType value
1939///
1940class InsertElementInst : public Instruction {
1941 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
1942 const Twine &NameStr = "",
1943 Instruction *InsertBefore = nullptr);
1944 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr,
1945 BasicBlock *InsertAtEnd);
1946
1947protected:
1948 // Note: Instruction needs to be a friend here to call cloneImpl.
1949 friend class Instruction;
1950
1951 InsertElementInst *cloneImpl() const;
1952
1953public:
1954 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1955 const Twine &NameStr = "",
1956 Instruction *InsertBefore = nullptr) {
1957 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertBefore);
1958 }
1959
1960 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1961 const Twine &NameStr,
1962 BasicBlock *InsertAtEnd) {
1963 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertAtEnd);
1964 }
1965
1966 /// Return true if an insertelement instruction can be
1967 /// formed with the specified operands.
1968 static bool isValidOperands(const Value *Vec, const Value *NewElt,
1969 const Value *Idx);
1970
1971 /// Overload to return most specific vector type.
1972 ///
1973 VectorType *getType() const {
1974 return cast<VectorType>(Instruction::getType());
1975 }
1976
1977 /// Transparently provide more efficient getOperand methods.
1978 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1979
1980 // Methods for support type inquiry through isa, cast, and dyn_cast:
1981 static bool classof(const Instruction *I) {
1982 return I->getOpcode() == Instruction::InsertElement;
1983 }
1984 static bool classof(const Value *V) {
1985 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1986 }
1987};
1988
1989template <>
1990struct OperandTraits<InsertElementInst> :
1991 public FixedNumOperandTraits<InsertElementInst, 3> {
1992};
1993
1994DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)InsertElementInst::op_iterator InsertElementInst::op_begin() {
return OperandTraits<InsertElementInst>::op_begin(this
); } InsertElementInst::const_op_iterator InsertElementInst::
op_begin() const { return OperandTraits<InsertElementInst>
::op_begin(const_cast<InsertElementInst*>(this)); } InsertElementInst
::op_iterator InsertElementInst::op_end() { return OperandTraits
<InsertElementInst>::op_end(this); } InsertElementInst::
const_op_iterator InsertElementInst::op_end() const { return OperandTraits
<InsertElementInst>::op_end(const_cast<InsertElementInst
*>(this)); } Value *InsertElementInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<InsertElementInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1994, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<InsertElementInst
>::op_begin(const_cast<InsertElementInst*>(this))[i_nocapture
].get()); } void InsertElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertElementInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 1994, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<InsertElementInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned InsertElementInst::getNumOperands
() const { return OperandTraits<InsertElementInst>::operands
(this); } template <int Idx_nocapture> Use &InsertElementInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &InsertElementInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1995
1996//===----------------------------------------------------------------------===//
1997// ShuffleVectorInst Class
1998//===----------------------------------------------------------------------===//
1999
2000constexpr int UndefMaskElem = -1;
2001
2002/// This instruction constructs a fixed permutation of two
2003/// input vectors.
2004///
2005/// For each element of the result vector, the shuffle mask selects an element
2006/// from one of the input vectors to copy to the result. Non-negative elements
2007/// in the mask represent an index into the concatenated pair of input vectors.
2008/// UndefMaskElem (-1) specifies that the result element is undefined.
2009///
2010/// For scalable vectors, all the elements of the mask must be 0 or -1. This
2011/// requirement may be relaxed in the future.
2012class ShuffleVectorInst : public Instruction {
2013 SmallVector<int, 4> ShuffleMask;
2014 Constant *ShuffleMaskForBitcode;
2015
2016protected:
2017 // Note: Instruction needs to be a friend here to call cloneImpl.
2018 friend class Instruction;
2019
2020 ShuffleVectorInst *cloneImpl() const;
2021
2022public:
2023 ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr = "",
2024 Instruction *InsertBefore = nullptr);
2025 ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr,
2026 BasicBlock *InsertAtEnd);
2027 ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr = "",
2028 Instruction *InsertBefore = nullptr);
2029 ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr,
2030 BasicBlock *InsertAtEnd);
2031 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2032 const Twine &NameStr = "",
2033 Instruction *InsertBefor = nullptr);
2034 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2035 const Twine &NameStr, BasicBlock *InsertAtEnd);
2036 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2037 const Twine &NameStr = "",
2038 Instruction *InsertBefor = nullptr);
2039 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2040 const Twine &NameStr, BasicBlock *InsertAtEnd);
2041
2042 void *operator new(size_t S) { return User::operator new(S, 2); }
2043 void operator delete(void *Ptr) { return User::operator delete(Ptr); }
2044
2045 /// Swap the operands and adjust the mask to preserve the semantics
2046 /// of the instruction.
2047 void commute();
2048
2049 /// Return true if a shufflevector instruction can be
2050 /// formed with the specified operands.
2051 static bool isValidOperands(const Value *V1, const Value *V2,
2052 const Value *Mask);
2053 static bool isValidOperands(const Value *V1, const Value *V2,
2054 ArrayRef<int> Mask);
2055
2056 /// Overload to return most specific vector type.
2057 ///
2058 VectorType *getType() const {
2059 return cast<VectorType>(Instruction::getType());
2060 }
2061
2062 /// Transparently provide more efficient getOperand methods.
2063 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2064
2065 /// Return the shuffle mask value of this instruction for the given element
2066 /// index. Return UndefMaskElem if the element is undef.
2067 int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
2068
2069 /// Convert the input shuffle mask operand to a vector of integers. Undefined
2070 /// elements of the mask are returned as UndefMaskElem.
2071 static void getShuffleMask(const Constant *Mask,
2072 SmallVectorImpl<int> &Result);
2073
2074 /// Return the mask for this instruction as a vector of integers. Undefined
2075 /// elements of the mask are returned as UndefMaskElem.
2076 void getShuffleMask(SmallVectorImpl<int> &Result) const {
2077 Result.assign(ShuffleMask.begin(), ShuffleMask.end());
2078 }
2079
2080 /// Return the mask for this instruction, for use in bitcode.
2081 ///
2082 /// TODO: This is temporary until we decide a new bitcode encoding for
2083 /// shufflevector.
2084 Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
2085
2086 static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
2087 Type *ResultTy);
2088
2089 void setShuffleMask(ArrayRef<int> Mask);
2090
2091 ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
2092
2093 /// Return true if this shuffle returns a vector with a different number of
2094 /// elements than its source vectors.
2095 /// Examples: shufflevector <4 x n> A, <4 x n> B, <1,2,3>
2096 /// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
2097 bool changesLength() const {
2098 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2099 ->getElementCount()
2100 .getKnownMinValue();
2101 unsigned NumMaskElts = ShuffleMask.size();
2102 return NumSourceElts != NumMaskElts;
2103 }
2104
2105 /// Return true if this shuffle returns a vector with a greater number of
2106 /// elements than its source vectors.
2107 /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
2108 bool increasesLength() const {
2109 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2110 ->getElementCount()
2111 .getKnownMinValue();
2112 unsigned NumMaskElts = ShuffleMask.size();
2113 return NumSourceElts < NumMaskElts;
2114 }
2115
2116 /// Return true if this shuffle mask chooses elements from exactly one source
2117 /// vector.
2118 /// Example: <7,5,undef,7>
2119 /// This assumes that vector operands are the same length as the mask.
2120 static bool isSingleSourceMask(ArrayRef<int> Mask);
2121 static bool isSingleSourceMask(const Constant *Mask) {
2122 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2122, __extension__ __PRETTY_FUNCTION__
))
;
2123 SmallVector<int, 16> MaskAsInts;
2124 getShuffleMask(Mask, MaskAsInts);
2125 return isSingleSourceMask(MaskAsInts);
2126 }
2127
2128 /// Return true if this shuffle chooses elements from exactly one source
2129 /// vector without changing the length of that vector.
2130 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
2131 /// TODO: Optionally allow length-changing shuffles.
2132 bool isSingleSource() const {
2133 return !changesLength() && isSingleSourceMask(ShuffleMask);
2134 }
2135
2136 /// Return true if this shuffle mask chooses elements from exactly one source
2137 /// vector without lane crossings. A shuffle using this mask is not
2138 /// necessarily a no-op because it may change the number of elements from its
2139 /// input vectors or it may provide demanded bits knowledge via undef lanes.
2140 /// Example: <undef,undef,2,3>
2141 static bool isIdentityMask(ArrayRef<int> Mask);
2142 static bool isIdentityMask(const Constant *Mask) {
2143 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2143, __extension__ __PRETTY_FUNCTION__
))
;
2144 SmallVector<int, 16> MaskAsInts;
2145 getShuffleMask(Mask, MaskAsInts);
2146 return isIdentityMask(MaskAsInts);
2147 }
2148
2149 /// Return true if this shuffle chooses elements from exactly one source
2150 /// vector without lane crossings and does not change the number of elements
2151 /// from its input vectors.
2152 /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
2153 bool isIdentity() const {
2154 return !changesLength() && isIdentityMask(ShuffleMask);
2155 }
2156
2157 /// Return true if this shuffle lengthens exactly one source vector with
2158 /// undefs in the high elements.
2159 bool isIdentityWithPadding() const;
2160
2161 /// Return true if this shuffle extracts the first N elements of exactly one
2162 /// source vector.
2163 bool isIdentityWithExtract() const;
2164
2165 /// Return true if this shuffle concatenates its 2 source vectors. This
2166 /// returns false if either input is undefined. In that case, the shuffle is
2167 /// is better classified as an identity with padding operation.
2168 bool isConcat() const;
2169
2170 /// Return true if this shuffle mask chooses elements from its source vectors
2171 /// without lane crossings. A shuffle using this mask would be
2172 /// equivalent to a vector select with a constant condition operand.
2173 /// Example: <4,1,6,undef>
2174 /// This returns false if the mask does not choose from both input vectors.
2175 /// In that case, the shuffle is better classified as an identity shuffle.
2176 /// This assumes that vector operands are the same length as the mask
2177 /// (a length-changing shuffle can never be equivalent to a vector select).
2178 static bool isSelectMask(ArrayRef<int> Mask);
2179 static bool isSelectMask(const Constant *Mask) {
2180 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2180, __extension__ __PRETTY_FUNCTION__
))
;
2181 SmallVector<int, 16> MaskAsInts;
2182 getShuffleMask(Mask, MaskAsInts);
2183 return isSelectMask(MaskAsInts);
2184 }
2185
2186 /// Return true if this shuffle chooses elements from its source vectors
2187 /// without lane crossings and all operands have the same number of elements.
2188 /// In other words, this shuffle is equivalent to a vector select with a
2189 /// constant condition operand.
2190 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,1,6,3>
2191 /// This returns false if the mask does not choose from both input vectors.
2192 /// In that case, the shuffle is better classified as an identity shuffle.
2193 /// TODO: Optionally allow length-changing shuffles.
2194 bool isSelect() const {
2195 return !changesLength() && isSelectMask(ShuffleMask);
2196 }
2197
2198 /// Return true if this shuffle mask swaps the order of elements from exactly
2199 /// one source vector.
2200 /// Example: <7,6,undef,4>
2201 /// This assumes that vector operands are the same length as the mask.
2202 static bool isReverseMask(ArrayRef<int> Mask);
2203 static bool isReverseMask(const Constant *Mask) {
2204 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2204, __extension__ __PRETTY_FUNCTION__
))
;
2205 SmallVector<int, 16> MaskAsInts;
2206 getShuffleMask(Mask, MaskAsInts);
2207 return isReverseMask(MaskAsInts);
2208 }
2209
2210 /// Return true if this shuffle swaps the order of elements from exactly
2211 /// one source vector.
2212 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
2213 /// TODO: Optionally allow length-changing shuffles.
2214 bool isReverse() const {
2215 return !changesLength() && isReverseMask(ShuffleMask);
2216 }
2217
2218 /// Return true if this shuffle mask chooses all elements with the same value
2219 /// as the first element of exactly one source vector.
2220 /// Example: <4,undef,undef,4>
2221 /// This assumes that vector operands are the same length as the mask.
2222 static bool isZeroEltSplatMask(ArrayRef<int> Mask);
2223 static bool isZeroEltSplatMask(const Constant *Mask) {
2224 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2224, __extension__ __PRETTY_FUNCTION__
))
;
2225 SmallVector<int, 16> MaskAsInts;
2226 getShuffleMask(Mask, MaskAsInts);
2227 return isZeroEltSplatMask(MaskAsInts);
2228 }
2229
2230 /// Return true if all elements of this shuffle are the same value as the
2231 /// first element of exactly one source vector without changing the length
2232 /// of that vector.
2233 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,0,undef,0>
2234 /// TODO: Optionally allow length-changing shuffles.
2235 /// TODO: Optionally allow splats from other elements.
2236 bool isZeroEltSplat() const {
2237 return !changesLength() && isZeroEltSplatMask(ShuffleMask);
2238 }
2239
2240 /// Return true if this shuffle mask is a transpose mask.
2241 /// Transpose vector masks transpose a 2xn matrix. They read corresponding
2242 /// even- or odd-numbered vector elements from two n-dimensional source
2243 /// vectors and write each result into consecutive elements of an
2244 /// n-dimensional destination vector. Two shuffles are necessary to complete
2245 /// the transpose, one for the even elements and another for the odd elements.
2246 /// This description closely follows how the TRN1 and TRN2 AArch64
2247 /// instructions operate.
2248 ///
2249 /// For example, a simple 2x2 matrix can be transposed with:
2250 ///
2251 /// ; Original matrix
2252 /// m0 = < a, b >
2253 /// m1 = < c, d >
2254 ///
2255 /// ; Transposed matrix
2256 /// t0 = < a, c > = shufflevector m0, m1, < 0, 2 >
2257 /// t1 = < b, d > = shufflevector m0, m1, < 1, 3 >
2258 ///
2259 /// For matrices having greater than n columns, the resulting nx2 transposed
2260 /// matrix is stored in two result vectors such that one vector contains
2261 /// interleaved elements from all the even-numbered rows and the other vector
2262 /// contains interleaved elements from all the odd-numbered rows. For example,
2263 /// a 2x4 matrix can be transposed with:
2264 ///
2265 /// ; Original matrix
2266 /// m0 = < a, b, c, d >
2267 /// m1 = < e, f, g, h >
2268 ///
2269 /// ; Transposed matrix
2270 /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
2271 /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
2272 static bool isTransposeMask(ArrayRef<int> Mask);
2273 static bool isTransposeMask(const Constant *Mask) {
2274 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2274, __extension__ __PRETTY_FUNCTION__
))
;
2275 SmallVector<int, 16> MaskAsInts;
2276 getShuffleMask(Mask, MaskAsInts);
2277 return isTransposeMask(MaskAsInts);
2278 }
2279
2280 /// Return true if this shuffle transposes the elements of its inputs without
2281 /// changing the length of the vectors. This operation may also be known as a
2282 /// merge or interleave. See the description for isTransposeMask() for the
2283 /// exact specification.
2284 /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
2285 bool isTranspose() const {
2286 return !changesLength() && isTransposeMask(ShuffleMask);
2287 }
2288
2289 /// Return true if this shuffle mask is an extract subvector mask.
2290 /// A valid extract subvector mask returns a smaller vector from a single
2291 /// source operand. The base extraction index is returned as well.
2292 static bool isExtractSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2293 int &Index);
2294 static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
2295 int &Index) {
2296 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2296, __extension__ __PRETTY_FUNCTION__
))
;
2297 // Not possible to express a shuffle mask for a scalable vector for this
2298 // case.
2299 if (isa<ScalableVectorType>(Mask->getType()))
2300 return false;
2301 SmallVector<int, 16> MaskAsInts;
2302 getShuffleMask(Mask, MaskAsInts);
2303 return isExtractSubvectorMask(MaskAsInts, NumSrcElts, Index);
2304 }
2305
2306 /// Return true if this shuffle mask is an extract subvector mask.
2307 bool isExtractSubvectorMask(int &Index) const {
2308 // Not possible to express a shuffle mask for a scalable vector for this
2309 // case.
2310 if (isa<ScalableVectorType>(getType()))
2311 return false;
2312
2313 int NumSrcElts =
2314 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2315 return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
2316 }
2317
2318 /// Return true if this shuffle mask is an insert subvector mask.
2319 /// A valid insert subvector mask inserts the lowest elements of a second
2320 /// source operand into an in-place first source operand operand.
2321 /// Both the sub vector width and the insertion index is returned.
2322 static bool isInsertSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2323 int &NumSubElts, int &Index);
2324 static bool isInsertSubvectorMask(const Constant *Mask, int NumSrcElts,
2325 int &NumSubElts, int &Index) {
2326 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2326, __extension__ __PRETTY_FUNCTION__
))
;
2327 // Not possible to express a shuffle mask for a scalable vector for this
2328 // case.
2329 if (isa<ScalableVectorType>(Mask->getType()))
2330 return false;
2331 SmallVector<int, 16> MaskAsInts;
2332 getShuffleMask(Mask, MaskAsInts);
2333 return isInsertSubvectorMask(MaskAsInts, NumSrcElts, NumSubElts, Index);
2334 }
2335
2336 /// Return true if this shuffle mask is an insert subvector mask.
2337 bool isInsertSubvectorMask(int &NumSubElts, int &Index) const {
2338 // Not possible to express a shuffle mask for a scalable vector for this
2339 // case.
2340 if (isa<ScalableVectorType>(getType()))
2341 return false;
2342
2343 int NumSrcElts =
2344 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2345 return isInsertSubvectorMask(ShuffleMask, NumSrcElts, NumSubElts, Index);
2346 }
2347
2348 /// Return true if this shuffle mask replicates each of the \p VF elements
2349 /// in a vector \p ReplicationFactor times.
2350 /// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is:
2351 /// <0,0,0,1,1,1,2,2,2,3,3,3>
2352 static bool isReplicationMask(ArrayRef<int> Mask, int &ReplicationFactor,
2353 int &VF);
2354 static bool isReplicationMask(const Constant *Mask, int &ReplicationFactor,
2355 int &VF) {
2356 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "llvm/include/llvm/IR/Instructions.h", 2356, __extension__ __PRETTY_FUNCTION__
))
;
2357 // Not possible to express a shuffle mask for a scalable vector for this
2358 // case.
2359 if (isa<ScalableVectorType>(Mask->getType()))
2360 return false;
2361 SmallVector<int, 16> MaskAsInts;
2362 getShuffleMask(Mask, MaskAsInts);
2363 return isReplicationMask(MaskAsInts, ReplicationFactor, VF);
2364 }
2365
2366 /// Return true if this shuffle mask is a replication mask.
2367 bool isReplicationMask(int &ReplicationFactor, int &VF) const;
2368
2369 /// Change values in a shuffle permute mask assuming the two vector operands
2370 /// of length InVecNumElts have swapped position.
2371 static void commuteShuffleMask(MutableArrayRef<int> Mask,
2372 unsigned InVecNumElts) {
2373 for (int &Idx : Mask) {
2374 if (Idx == -1)
2375 continue;
2376 Idx = Idx < (int)InVecNumElts ? Idx + InVecNumElts : Idx - InVecNumElts;
2377 assert(Idx >= 0 && Idx < (int)InVecNumElts * 2 &&(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "llvm/include/llvm/IR/Instructions.h", 2378, __extension__ __PRETTY_FUNCTION__
))
2378 "shufflevector mask index out of range")(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "llvm/include/llvm/IR/Instructions.h", 2378, __extension__ __PRETTY_FUNCTION__
))
;
2379 }
2380 }
2381
2382 // Methods for support type inquiry through isa, cast, and dyn_cast:
2383 static bool classof(const Instruction *I) {
2384 return I->getOpcode() == Instruction::ShuffleVector;
2385 }
2386 static bool classof(const Value *V) {
2387 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2388 }
2389};
2390
2391template <>
2392struct OperandTraits<ShuffleVectorInst>
2393 : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
2394
2395DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)ShuffleVectorInst::op_iterator ShuffleVectorInst::op_begin() {
return OperandTraits<ShuffleVectorInst>::op_begin(this
); } ShuffleVectorInst::const_op_iterator ShuffleVectorInst::
op_begin() const { return OperandTraits<ShuffleVectorInst>
::op_begin(const_cast<ShuffleVectorInst*>(this)); } ShuffleVectorInst
::op_iterator ShuffleVectorInst::op_end() { return OperandTraits
<ShuffleVectorInst>::op_end(this); } ShuffleVectorInst::
const_op_iterator ShuffleVectorInst::op_end() const { return OperandTraits
<ShuffleVectorInst>::op_end(const_cast<ShuffleVectorInst
*>(this)); } Value *ShuffleVectorInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<ShuffleVectorInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2395, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<ShuffleVectorInst
>::op_begin(const_cast<ShuffleVectorInst*>(this))[i_nocapture
].get()); } void ShuffleVectorInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ShuffleVectorInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2395, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<ShuffleVectorInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned ShuffleVectorInst::getNumOperands
() const { return OperandTraits<ShuffleVectorInst>::operands
(this); } template <int Idx_nocapture> Use &ShuffleVectorInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &ShuffleVectorInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2396
2397//===----------------------------------------------------------------------===//
2398// ExtractValueInst Class
2399//===----------------------------------------------------------------------===//
2400
2401/// This instruction extracts a struct member or array
2402/// element value from an aggregate value.
2403///
2404class ExtractValueInst : public UnaryInstruction {
2405 SmallVector<unsigned, 4> Indices;
2406
2407 ExtractValueInst(const ExtractValueInst &EVI);
2408
2409 /// Constructors - Create a extractvalue instruction with a base aggregate
2410 /// value and a list of indices. The first ctor can optionally insert before
2411 /// an existing instruction, the second appends the new instruction to the
2412 /// specified BasicBlock.
2413 inline ExtractValueInst(Value *Agg,
2414 ArrayRef<unsigned> Idxs,
2415 const Twine &NameStr,
2416 Instruction *InsertBefore);
2417 inline ExtractValueInst(Value *Agg,
2418 ArrayRef<unsigned> Idxs,
2419 const Twine &NameStr, BasicBlock *InsertAtEnd);
2420
2421 void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
2422
2423protected:
2424 // Note: Instruction needs to be a friend here to call cloneImpl.
2425 friend class Instruction;
2426
2427 ExtractValueInst *cloneImpl() const;
2428
2429public:
2430 static ExtractValueInst *Create(Value *Agg,
2431 ArrayRef<unsigned> Idxs,
2432 const Twine &NameStr = "",
2433 Instruction *InsertBefore = nullptr) {
2434 return new
2435 ExtractValueInst(Agg, Idxs, NameStr, InsertBefore);
2436 }
2437
2438 static ExtractValueInst *Create(Value *Agg,
2439 ArrayRef<unsigned> Idxs,
2440 const Twine &NameStr,
2441 BasicBlock *InsertAtEnd) {
2442 return new ExtractValueInst(Agg, Idxs, NameStr, InsertAtEnd);
2443 }
2444
2445 /// Returns the type of the element that would be extracted
2446 /// with an extractvalue instruction with the specified parameters.
2447 ///
2448 /// Null is returned if the indices are invalid for the specified type.
2449 static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
2450
2451 using idx_iterator = const unsigned*;
2452
2453 inline idx_iterator idx_begin() const { return Indices.begin(); }
2454 inline idx_iterator idx_end() const { return Indices.end(); }
2455 inline iterator_range<idx_iterator> indices() const {
2456 return make_range(idx_begin(), idx_end());
2457 }
2458
2459 Value *getAggregateOperand() {
2460 return getOperand(0);
2461 }
2462 const Value *getAggregateOperand() const {
2463 return getOperand(0);
2464 }
2465 static unsigned getAggregateOperandIndex() {
2466 return 0U; // get index for modifying correct operand
2467 }
2468
2469 ArrayRef<unsigned> getIndices() const {
2470 return Indices;
2471 }
2472
2473 unsigned getNumIndices() const {
2474 return (unsigned)Indices.size();
2475 }
2476
2477 bool hasIndices() const {
2478 return true;
2479 }
2480
2481 // Methods for support type inquiry through isa, cast, and dyn_cast:
2482 static bool classof(const Instruction *I) {
2483 return I->getOpcode() == Instruction::ExtractValue;
2484 }
2485 static bool classof(const Value *V) {
2486 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2487 }
2488};
2489
2490ExtractValueInst::ExtractValueInst(Value *Agg,
2491 ArrayRef<unsigned> Idxs,
2492 const Twine &NameStr,
2493 Instruction *InsertBefore)
2494 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2495 ExtractValue, Agg, InsertBefore) {
2496 init(Idxs, NameStr);
2497}
2498
2499ExtractValueInst::ExtractValueInst(Value *Agg,
2500 ArrayRef<unsigned> Idxs,
2501 const Twine &NameStr,
2502 BasicBlock *InsertAtEnd)
2503 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2504 ExtractValue, Agg, InsertAtEnd) {
2505 init(Idxs, NameStr);
2506}
2507
2508//===----------------------------------------------------------------------===//
2509// InsertValueInst Class
2510//===----------------------------------------------------------------------===//
2511
2512/// This instruction inserts a struct field of array element
2513/// value into an aggregate value.
2514///
2515class InsertValueInst : public Instruction {
2516 SmallVector<unsigned, 4> Indices;
2517
2518 InsertValueInst(const InsertValueInst &IVI);
2519
2520 /// Constructors - Create a insertvalue instruction with a base aggregate
2521 /// value, a value to insert, and a list of indices. The first ctor can
2522 /// optionally insert before an existing instruction, the second appends
2523 /// the new instruction to the specified BasicBlock.
2524 inline InsertValueInst(Value *Agg, Value *Val,
2525 ArrayRef<unsigned> Idxs,
2526 const Twine &NameStr,
2527 Instruction *InsertBefore);
2528 inline InsertValueInst(Value *Agg, Value *Val,
2529 ArrayRef<unsigned> Idxs,
2530 const Twine &NameStr, BasicBlock *InsertAtEnd);
2531
2532 /// Constructors - These two constructors are convenience methods because one
2533 /// and two index insertvalue instructions are so common.
2534 InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
2535 const Twine &NameStr = "",
2536 Instruction *InsertBefore = nullptr);
2537 InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr,
2538 BasicBlock *InsertAtEnd);
2539
2540 void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2541 const Twine &NameStr);
2542
2543protected:
2544 // Note: Instruction needs to be a friend here to call cloneImpl.
2545 friend class Instruction;
2546
2547 InsertValueInst *cloneImpl() const;
2548
2549public:
2550 // allocate space for exactly two operands
2551 void *operator new(size_t S) { return User::operator new(S, 2); }
2552 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2553
2554 static InsertValueInst *Create(Value *Agg, Value *Val,
2555 ArrayRef<unsigned> Idxs,
2556 const Twine &NameStr = "",
2557 Instruction *InsertBefore = nullptr) {
2558 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertBefore);
2559 }
2560
2561 static InsertValueInst *Create(Value *Agg, Value *Val,
2562 ArrayRef<unsigned> Idxs,
2563 const Twine &NameStr,
2564 BasicBlock *InsertAtEnd) {
2565 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertAtEnd);
2566 }
2567
2568 /// Transparently provide more efficient getOperand methods.
2569 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2570
2571 using idx_iterator = const unsigned*;
2572
2573 inline idx_iterator idx_begin() const { return Indices.begin(); }
2574 inline idx_iterator idx_end() const { return Indices.end(); }
2575 inline iterator_range<idx_iterator> indices() const {
2576 return make_range(idx_begin(), idx_end());
2577 }
2578
2579 Value *getAggregateOperand() {
2580 return getOperand(0);
2581 }
2582 const Value *getAggregateOperand() const {
2583 return getOperand(0);
2584 }
2585 static unsigned getAggregateOperandIndex() {
2586 return 0U; // get index for modifying correct operand
2587 }
2588
2589 Value *getInsertedValueOperand() {
2590 return getOperand(1);
2591 }
2592 const Value *getInsertedValueOperand() const {
2593 return getOperand(1);
2594 }
2595 static unsigned getInsertedValueOperandIndex() {
2596 return 1U; // get index for modifying correct operand
2597 }
2598
2599 ArrayRef<unsigned> getIndices() const {
2600 return Indices;
2601 }
2602
2603 unsigned getNumIndices() const {
2604 return (unsigned)Indices.size();
2605 }
2606
2607 bool hasIndices() const {
2608 return true;
2609 }
2610
2611 // Methods for support type inquiry through isa, cast, and dyn_cast:
2612 static bool classof(const Instruction *I) {
2613 return I->getOpcode() == Instruction::InsertValue;
2614 }
2615 static bool classof(const Value *V) {
2616 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2617 }
2618};
2619
2620template <>
2621struct OperandTraits<InsertValueInst> :
2622 public FixedNumOperandTraits<InsertValueInst, 2> {
2623};
2624
2625InsertValueInst::InsertValueInst(Value *Agg,
2626 Value *Val,
2627 ArrayRef<unsigned> Idxs,
2628 const Twine &NameStr,
2629 Instruction *InsertBefore)
2630 : Instruction(Agg->getType(), InsertValue,
2631 OperandTraits<InsertValueInst>::op_begin(this),
2632 2, InsertBefore) {
2633 init(Agg, Val, Idxs, NameStr);
2634}
2635
2636InsertValueInst::InsertValueInst(Value *Agg,
2637 Value *Val,
2638 ArrayRef<unsigned> Idxs,
2639 const Twine &NameStr,
2640 BasicBlock *InsertAtEnd)
2641 : Instruction(Agg->getType(), InsertValue,
2642 OperandTraits<InsertValueInst>::op_begin(this),
2643 2, InsertAtEnd) {
2644 init(Agg, Val, Idxs, NameStr);
2645}
2646
2647DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)InsertValueInst::op_iterator InsertValueInst::op_begin() { return
OperandTraits<InsertValueInst>::op_begin(this); } InsertValueInst
::const_op_iterator InsertValueInst::op_begin() const { return
OperandTraits<InsertValueInst>::op_begin(const_cast<
InsertValueInst*>(this)); } InsertValueInst::op_iterator InsertValueInst
::op_end() { return OperandTraits<InsertValueInst>::op_end
(this); } InsertValueInst::const_op_iterator InsertValueInst::
op_end() const { return OperandTraits<InsertValueInst>::
op_end(const_cast<InsertValueInst*>(this)); } Value *InsertValueInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<InsertValueInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2647, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<InsertValueInst
>::op_begin(const_cast<InsertValueInst*>(this))[i_nocapture
].get()); } void InsertValueInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertValueInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2647, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<InsertValueInst>::op_begin(this)[i_nocapture
] = Val_nocapture; } unsigned InsertValueInst::getNumOperands
() const { return OperandTraits<InsertValueInst>::operands
(this); } template <int Idx_nocapture> Use &InsertValueInst
::Op() { return this->OpFrom<Idx_nocapture>(this); }
template <int Idx_nocapture> const Use &InsertValueInst
::Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2648
2649//===----------------------------------------------------------------------===//
2650// PHINode Class
2651//===----------------------------------------------------------------------===//
2652
2653// PHINode - The PHINode class is used to represent the magical mystical PHI
2654// node, that can not exist in nature, but can be synthesized in a computer
2655// scientist's overactive imagination.
2656//
2657class PHINode : public Instruction {
2658 /// The number of operands actually allocated. NumOperands is
2659 /// the number actually in use.
2660 unsigned ReservedSpace;
2661
2662 PHINode(const PHINode &PN);
2663
2664 explicit PHINode(Type *Ty, unsigned NumReservedValues,
2665 const Twine &NameStr = "",
2666 Instruction *InsertBefore = nullptr)
2667 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
2668 ReservedSpace(NumReservedValues) {
2669 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "llvm/include/llvm/IR/Instructions.h", 2669, __extension__ __PRETTY_FUNCTION__
))
;
2670 setName(NameStr);
2671 allocHungoffUses(ReservedSpace);
2672 }
2673
2674 PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
2675 BasicBlock *InsertAtEnd)
2676 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
2677 ReservedSpace(NumReservedValues) {
2678 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "llvm/include/llvm/IR/Instructions.h", 2678, __extension__ __PRETTY_FUNCTION__
))
;
2679 setName(NameStr);
2680 allocHungoffUses(ReservedSpace);
2681 }
2682
2683protected:
2684 // Note: Instruction needs to be a friend here to call cloneImpl.
2685 friend class Instruction;
2686
2687 PHINode *cloneImpl() const;
2688
2689 // allocHungoffUses - this is more complicated than the generic
2690 // User::allocHungoffUses, because we have to allocate Uses for the incoming
2691 // values and pointers to the incoming blocks, all in one allocation.
2692 void allocHungoffUses(unsigned N) {
2693 User::allocHungoffUses(N, /* IsPhi */ true);
2694 }
2695
2696public:
2697 /// Constructors - NumReservedValues is a hint for the number of incoming
2698 /// edges that this phi node will have (use 0 if you really have no idea).
2699 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2700 const Twine &NameStr = "",
2701 Instruction *InsertBefore = nullptr) {
2702 return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
2703 }
2704
2705 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2706 const Twine &NameStr, BasicBlock *InsertAtEnd) {
2707 return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
2708 }
2709
2710 /// Provide fast operand accessors
2711 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2712
2713 // Block iterator interface. This provides access to the list of incoming
2714 // basic blocks, which parallels the list of incoming values.
2715
2716 using block_iterator = BasicBlock **;
2717 using const_block_iterator = BasicBlock * const *;
2718
2719 block_iterator block_begin() {
2720 return reinterpret_cast<block_iterator>(op_begin() + ReservedSpace);
2721 }
2722
2723 const_block_iterator block_begin() const {
2724 return reinterpret_cast<const_block_iterator>(op_begin() + ReservedSpace);
2725 }
2726
2727 block_iterator block_end() {
2728 return block_begin() + getNumOperands();
2729 }
2730
2731 const_block_iterator block_end() const {
2732 return block_begin() + getNumOperands();
2733 }
2734
2735 iterator_range<block_iterator> blocks() {
2736 return make_range(block_begin(), block_end());
2737 }
2738
2739 iterator_range<const_block_iterator> blocks() const {
2740 return make_range(block_begin(), block_end());
2741 }
2742
2743 op_range incoming_values() { return operands(); }
2744
2745 const_op_range incoming_values() const { return operands(); }
2746
2747 /// Return the number of incoming edges
2748 ///
2749 unsigned getNumIncomingValues() const { return getNumOperands(); }
2750
2751 /// Return incoming value number x
2752 ///
2753 Value *getIncomingValue(unsigned i) const {
2754 return getOperand(i);
2755 }
2756 void setIncomingValue(unsigned i, Value *V) {
2757 assert(V && "PHI node got a null value!")(static_cast <bool> (V && "PHI node got a null value!"
) ? void (0) : __assert_fail ("V && \"PHI node got a null value!\""
, "llvm/include/llvm/IR/Instructions.h", 2757, __extension__ __PRETTY_FUNCTION__
))
;
2758 assert(getType() == V->getType() &&(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "llvm/include/llvm/IR/Instructions.h", 2759, __extension__ __PRETTY_FUNCTION__
))
2759 "All operands to PHI node must be the same type as the PHI node!")(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "llvm/include/llvm/IR/Instructions.h", 2759, __extension__ __PRETTY_FUNCTION__
))
;
2760 setOperand(i, V);
2761 }
2762
2763 static unsigned getOperandNumForIncomingValue(unsigned i) {
2764 return i;
2765 }
2766
2767 static unsigned getIncomingValueNumForOperand(unsigned i) {
2768 return i;
2769 }
2770
2771 /// Return incoming basic block number @p i.
2772 ///
2773 BasicBlock *getIncomingBlock(unsigned i) const {
2774 return block_begin()[i];
2775 }
2776
2777 /// Return incoming basic block corresponding
2778 /// to an operand of the PHI.
2779 ///
2780 BasicBlock *getIncomingBlock(const Use &U) const {
2781 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?")(static_cast <bool> (this == U.getUser() && "Iterator doesn't point to PHI's Uses?"
) ? void (0) : __assert_fail ("this == U.getUser() && \"Iterator doesn't point to PHI's Uses?\""
, "llvm/include/llvm/IR/Instructions.h", 2781, __extension__ __PRETTY_FUNCTION__
))
;
2782 return getIncomingBlock(unsigned(&U - op_begin()));
2783 }
2784
2785 /// Return incoming basic block corresponding
2786 /// to value use iterator.
2787 ///
2788 BasicBlock *getIncomingBlock(Value::const_user_iterator I) const {
2789 return getIncomingBlock(I.getUse());
2790 }
2791
2792 void setIncomingBlock(unsigned i, BasicBlock *BB) {
2793 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "llvm/include/llvm/IR/Instructions.h", 2793, __extension__ __PRETTY_FUNCTION__
))
;
2794 block_begin()[i] = BB;
2795 }
2796
2797 /// Replace every incoming basic block \p Old to basic block \p New.
2798 void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
2799 assert(New && Old && "PHI node got a null basic block!")(static_cast <bool> (New && Old && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("New && Old && \"PHI node got a null basic block!\""
, "llvm/include/llvm/IR/Instructions.h", 2799, __extension__ __PRETTY_FUNCTION__
))
;
2800 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2801 if (getIncomingBlock(Op) == Old)
2802 setIncomingBlock(Op, New);
2803 }
2804
2805 /// Add an incoming value to the end of the PHI list
2806 ///
2807 void addIncoming(Value *V, BasicBlock *BB) {
2808 if (getNumOperands() == ReservedSpace)
2809 growOperands(); // Get more space!
2810 // Initialize some new operands.
2811 setNumHungOffUseOperands(getNumOperands() + 1);
2812 setIncomingValue(getNumOperands() - 1, V);
2813 setIncomingBlock(getNumOperands() - 1, BB);
2814 }
2815
2816 /// Remove an incoming value. This is useful if a
2817 /// predecessor basic block is deleted. The value removed is returned.
2818 ///
2819 /// If the last incoming value for a PHI node is removed (and DeletePHIIfEmpty
2820 /// is true), the PHI node is destroyed and any uses of it are replaced with
2821 /// dummy values. The only time there should be zero incoming values to a PHI
2822 /// node is when the block is dead, so this strategy is sound.
2823 ///
2824 Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
2825
2826 Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
2827 int Idx = getBasicBlockIndex(BB);
2828 assert(Idx >= 0 && "Invalid basic block argument to remove!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument to remove!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument to remove!\""
, "llvm/include/llvm/IR/Instructions.h", 2828, __extension__ __PRETTY_FUNCTION__
))
;
2829 return removeIncomingValue(Idx, DeletePHIIfEmpty);
2830 }
2831
2832 /// Return the first index of the specified basic
2833 /// block in the value list for this PHI. Returns -1 if no instance.
2834 ///
2835 int getBasicBlockIndex(const BasicBlock *BB) const {
2836 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
2837 if (block_begin()[i] == BB)
2838 return i;
2839 return -1;
2840 }
2841
2842 Value *getIncomingValueForBlock(const BasicBlock *BB) const {
2843 int Idx = getBasicBlockIndex(BB);
2844 assert(Idx >= 0 && "Invalid basic block argument!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument!\""
, "llvm/include/llvm/IR/Instructions.h", 2844, __extension__ __PRETTY_FUNCTION__
))
;
2845 return getIncomingValue(Idx);
2846 }
2847
2848 /// Set every incoming value(s) for block \p BB to \p V.
2849 void setIncomingValueForBlock(const BasicBlock *BB, Value *V) {
2850 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "llvm/include/llvm/IR/Instructions.h", 2850, __extension__ __PRETTY_FUNCTION__
))
;
2851 bool Found = false;
2852 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2853 if (getIncomingBlock(Op) == BB) {
2854 Found = true;
2855 setIncomingValue(Op, V);
2856 }
2857 (void)Found;
2858 assert(Found && "Invalid basic block argument to set!")(static_cast <bool> (Found && "Invalid basic block argument to set!"
) ? void (0) : __assert_fail ("Found && \"Invalid basic block argument to set!\""
, "llvm/include/llvm/IR/Instructions.h", 2858, __extension__ __PRETTY_FUNCTION__
))
;
2859 }
2860
2861 /// If the specified PHI node always merges together the
2862 /// same value, return the value, otherwise return null.
2863 Value *hasConstantValue() const;
2864
2865 /// Whether the specified PHI node always merges
2866 /// together the same value, assuming undefs are equal to a unique
2867 /// non-undef value.
2868 bool hasConstantOrUndefValue() const;
2869
2870 /// If the PHI node is complete which means all of its parent's predecessors
2871 /// have incoming value in this PHI, return true, otherwise return false.
2872 bool isComplete() const {
2873 return llvm::all_of(predecessors(getParent()),
2874 [this](const BasicBlock *Pred) {
2875 return getBasicBlockIndex(Pred) >= 0;
2876 });
2877 }
2878
2879 /// Methods for support type inquiry through isa, cast, and dyn_cast:
2880 static bool classof(const Instruction *I) {
2881 return I->getOpcode() == Instruction::PHI;
2882 }
2883 static bool classof(const Value *V) {
2884 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2885 }
2886
2887private:
2888 void growOperands();
2889};
2890
2891template <>
2892struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
2893};
2894
2895DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)PHINode::op_iterator PHINode::op_begin() { return OperandTraits
<PHINode>::op_begin(this); } PHINode::const_op_iterator
PHINode::op_begin() const { return OperandTraits<PHINode>
::op_begin(const_cast<PHINode*>(this)); } PHINode::op_iterator
PHINode::op_end() { return OperandTraits<PHINode>::op_end
(this); } PHINode::const_op_iterator PHINode::op_end() const {
return OperandTraits<PHINode>::op_end(const_cast<PHINode
*>(this)); } Value *PHINode::getOperand(unsigned i_nocapture
) const { (static_cast <bool> (i_nocapture < OperandTraits
<PHINode>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"getOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2895, __extension__ __PRETTY_FUNCTION__
)); return cast_or_null<Value>( OperandTraits<PHINode
>::op_begin(const_cast<PHINode*>(this))[i_nocapture]
.get()); } void PHINode::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<PHINode>::operands(this) && "setOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"setOperand() out of range!\""
, "llvm/include/llvm/IR/Instructions.h", 2895, __extension__ __PRETTY_FUNCTION__
)); OperandTraits<PHINode>::op_begin(this)[i_nocapture]
= Val_nocapture; } unsigned PHINode::getNumOperands() const {
return OperandTraits<PHINode>::operands(this); } template
<int Idx_nocapture> Use &PHINode::Op() { return this
->OpFrom<Idx_nocapture>(this); } template <int Idx_nocapture
> const Use &PHINode::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
2896
2897//===----------------------------------------------------------------------===//
2898// LandingPadInst Class
2899//===----------------------------------------------------------------------===//
2900
2901//===---------------------------------------------------------------------------
2902/// The landingpad instruction holds all of the information
2903/// necessary to generate correct exception handling. The landingpad instruction
2904/// cannot be moved from the top of a landing pad block, which itself is
2905/// accessible only from the 'unwind' edge of an invoke. This uses the
2906/// SubclassData field in Value to store whether or not the landingpad is a
2907/// cleanup.
2908///
2909class LandingPadInst : public Instruction {
2910 using CleanupField = BoolBitfieldElementT<0>;
2911
2912 /// The number of operands actually allocated. NumOperands is
2913 /// the number actually in use.
2914 unsigned ReservedSpace;
2915
2916 LandingPadInst(const LandingPadInst &LP);
2917
2918public:
2919 enum ClauseType { Catch, Filter };
2920
2921private:
2922 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2923 const Twine &NameStr, Instruction *InsertBefore);
2924 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2925 const Twine &NameStr, BasicBlock *InsertAtEnd);
2926
2927 // Allocate space for exactly zero operands.
2928 void *operator new(size_t S) { return User::operator new(S); }
2929
2930 void growOperands(unsigned Size);
2931 void init(unsigned NumReservedValues, const Twine &NameStr);
2932
2933protected:
2934 // Note: Instruction needs to be a friend here to call cloneImpl.
2935 friend class Instruction;
2936
2937 LandingPadInst *cloneImpl() const;
2938
2939public:
2940 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2941
2942 /// Constructors - NumReservedClauses is a hint for the number of incoming
2943 /// clauses that this landingpad will have (use 0 if you really have no idea).
2944 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2945 const Twine &NameStr = "",
2946 Instruction *InsertBefore = nullptr);
2947 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2948 const Twine &NameStr, BasicBlock *InsertAtEnd);
2949
2950 /// Provide fast operand accessors
2951 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2952
2953 /// Return 'true' if this landingpad instruction is a
2954 /// cleanup. I.e., it should be run when unwinding even if its landing pad
2955 /// doesn't catch the exception.
2956 bool isCleanup() const { return getSubclassData<CleanupField>(); }
2957
2958 /// Indicate that this landingpad instruction is a cleanup.
2959 void setCleanup(bool V) { setSubclassData<CleanupField>(V); }
2960
2961 /// Add a catch or filter clause to the landing pad.
2962 void addClause(Constant *ClauseVal);
2963
2964 /// Get the value of the clause at index Idx. Use isCatch/isFilter to
2965 /// determine what type of clause this is.
2966 Constant *getClause(unsigned Idx) const {
2967 return cast<Constant>(getOperandList()[Idx]);
2968 }
2969
2970 /// Return 'true' if the clause and index Idx is a catch clause.
2971 bool isCatch(unsigned Idx) const {
2972 return !isa<ArrayType>(getOperandList()[Idx]->getType());
2973 }
2974
2975 /// Return 'true' if the clause and index Idx is a filter clause.
2976 bool isFilter(unsigned Idx) const {
2977 return isa<ArrayType>(getOperandList()[Idx]->getType());
2978 }
2979
2980 /// Get the number of clauses for this landing pad.
2981 unsigned getNumClauses() const { return getNumOperands(); }
2982
2983 /// Grow the size of the operand list to accommodate the new
2984 /// number of clauses.
2985 void reserveClauses(unsigned Size) { growOperands(Size); }
2986
2987 // Methods for support type inquiry through isa, cast, and dyn_cast:
2988 static bool classof(const Instruction *I) {
2989 return I->getOpcode() == Instruction::LandingPad;
2990 }
2991 static bool classof(const Value *V) {
2992 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2993 }
2994};