Bug Summary

File:llvm/lib/Transforms/Scalar/LICM.cpp
Warning:line 1201, column 33
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LICM.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/Transforms/Scalar -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/build-llvm/lib/Transforms/Scalar -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-06-21-164211-33944-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp

/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp

1//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs loop invariant code motion, attempting to remove as much
10// code from the body of a loop as possible. It does this by either hoisting
11// code into the preheader block, or by sinking code to the exit blocks if it is
12// safe. This pass also promotes must-aliased memory locations in the loop to
13// live in registers, thus hoisting and sinking "invariant" loads and stores.
14//
15// Hoisting operations out of loops is a canonicalization transform. It
16// enables and simplifies subsequent optimizations in the middle-end.
17// Rematerialization of hoisted instructions to reduce register pressure is the
18// responsibility of the back-end, which has more accurate information about
19// register pressure and also handles other optimizations than LICM that
20// increase live-ranges.
21//
22// This pass uses alias analysis for two purposes:
23//
24// 1. Moving loop invariant loads and calls out of loops. If we can determine
25// that a load or call inside of a loop never aliases anything stored to,
26// we can hoist it or sink it like any other instruction.
27// 2. Scalar Promotion of Memory - If there is a store instruction inside of
28// the loop, we try to move the store to happen AFTER the loop instead of
29// inside of the loop. This can only happen if a few conditions are true:
30// A. The pointer stored through is loop invariant
31// B. There are no stores or loads in the loop which _may_ alias the
32// pointer. There are no calls in the loop which mod/ref the pointer.
33// If these conditions are true, we can promote the loads and stores in the
34// loop of the pointer to use a temporary alloca'd variable. We then use
35// the SSAUpdater to construct the appropriate SSA form for the value.
36//
37//===----------------------------------------------------------------------===//
38
39#include "llvm/Transforms/Scalar/LICM.h"
40#include "llvm/ADT/SetOperations.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/Analysis/AliasAnalysis.h"
43#include "llvm/Analysis/AliasSetTracker.h"
44#include "llvm/Analysis/BasicAliasAnalysis.h"
45#include "llvm/Analysis/BlockFrequencyInfo.h"
46#include "llvm/Analysis/CaptureTracking.h"
47#include "llvm/Analysis/ConstantFolding.h"
48#include "llvm/Analysis/GlobalsModRef.h"
49#include "llvm/Analysis/GuardUtils.h"
50#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
51#include "llvm/Analysis/Loads.h"
52#include "llvm/Analysis/LoopInfo.h"
53#include "llvm/Analysis/LoopIterator.h"
54#include "llvm/Analysis/LoopPass.h"
55#include "llvm/Analysis/MemoryBuiltins.h"
56#include "llvm/Analysis/MemorySSA.h"
57#include "llvm/Analysis/MemorySSAUpdater.h"
58#include "llvm/Analysis/MustExecute.h"
59#include "llvm/Analysis/OptimizationRemarkEmitter.h"
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
62#include "llvm/Analysis/TargetLibraryInfo.h"
63#include "llvm/Analysis/ValueTracking.h"
64#include "llvm/IR/CFG.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DerivedTypes.h"
69#include "llvm/IR/Dominators.h"
70#include "llvm/IR/Instructions.h"
71#include "llvm/IR/IntrinsicInst.h"
72#include "llvm/IR/LLVMContext.h"
73#include "llvm/IR/Metadata.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/PredIteratorCache.h"
76#include "llvm/InitializePasses.h"
77#include "llvm/Support/CommandLine.h"
78#include "llvm/Support/Debug.h"
79#include "llvm/Support/raw_ostream.h"
80#include "llvm/Transforms/Scalar.h"
81#include "llvm/Transforms/Scalar/LoopPassManager.h"
82#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
83#include "llvm/Transforms/Utils/BasicBlockUtils.h"
84#include "llvm/Transforms/Utils/Local.h"
85#include "llvm/Transforms/Utils/LoopUtils.h"
86#include "llvm/Transforms/Utils/SSAUpdater.h"
87#include <algorithm>
88#include <utility>
89using namespace llvm;
90
91#define DEBUG_TYPE"licm" "licm"
92
93STATISTIC(NumCreatedBlocks, "Number of blocks created")static llvm::Statistic NumCreatedBlocks = {"licm", "NumCreatedBlocks"
, "Number of blocks created"}
;
94STATISTIC(NumClonedBranches, "Number of branches cloned")static llvm::Statistic NumClonedBranches = {"licm", "NumClonedBranches"
, "Number of branches cloned"}
;
95STATISTIC(NumSunk, "Number of instructions sunk out of loop")static llvm::Statistic NumSunk = {"licm", "NumSunk", "Number of instructions sunk out of loop"
}
;
96STATISTIC(NumHoisted, "Number of instructions hoisted out of loop")static llvm::Statistic NumHoisted = {"licm", "NumHoisted", "Number of instructions hoisted out of loop"
}
;
97STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk")static llvm::Statistic NumMovedLoads = {"licm", "NumMovedLoads"
, "Number of load insts hoisted or sunk"}
;
98STATISTIC(NumMovedCalls, "Number of call insts hoisted or sunk")static llvm::Statistic NumMovedCalls = {"licm", "NumMovedCalls"
, "Number of call insts hoisted or sunk"}
;
99STATISTIC(NumPromoted, "Number of memory locations promoted to registers")static llvm::Statistic NumPromoted = {"licm", "NumPromoted", "Number of memory locations promoted to registers"
}
;
100
101/// Memory promotion is enabled by default.
102static cl::opt<bool>
103 DisablePromotion("disable-licm-promotion", cl::Hidden, cl::init(false),
104 cl::desc("Disable memory promotion in LICM pass"));
105
106static cl::opt<bool> ControlFlowHoisting(
107 "licm-control-flow-hoisting", cl::Hidden, cl::init(false),
108 cl::desc("Enable control flow (and PHI) hoisting in LICM"));
109
110static cl::opt<unsigned> HoistSinkColdnessThreshold(
111 "licm-coldness-threshold", cl::Hidden, cl::init(4),
112 cl::desc("Relative coldness Threshold of hoisting/sinking destination "
113 "block for LICM to be considered beneficial"));
114
115static cl::opt<uint32_t> MaxNumUsesTraversed(
116 "licm-max-num-uses-traversed", cl::Hidden, cl::init(8),
117 cl::desc("Max num uses visited for identifying load "
118 "invariance in loop using invariant start (default = 8)"));
119
120// Default value of zero implies we use the regular alias set tracker mechanism
121// instead of the cross product using AA to identify aliasing of the memory
122// location we are interested in.
123static cl::opt<int>
124LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0),
125 cl::desc("How many instruction to cross product using AA"));
126
127// Experimental option to allow imprecision in LICM in pathological cases, in
128// exchange for faster compile. This is to be removed if MemorySSA starts to
129// address the same issue. This flag applies only when LICM uses MemorySSA
130// instead on AliasSetTracker. LICM calls MemorySSAWalker's
131// getClobberingMemoryAccess, up to the value of the Cap, getting perfect
132// accuracy. Afterwards, LICM will call into MemorySSA's getDefiningAccess,
133// which may not be precise, since optimizeUses is capped. The result is
134// correct, but we may not get as "far up" as possible to get which access is
135// clobbering the one queried.
136cl::opt<unsigned> llvm::SetLicmMssaOptCap(
137 "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
138 cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
139 "for faster compile. Caps the MemorySSA clobbering calls."));
140
141// Experimentally, memory promotion carries less importance than sinking and
142// hoisting. Limit when we do promotion when using MemorySSA, in order to save
143// compile time.
144cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
145 "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
146 cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
147 "effect. When MSSA in LICM is enabled, then this is the maximum "
148 "number of accesses allowed to be present in a loop in order to "
149 "enable memory promotion."));
150
151static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
152static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
153 const LoopSafetyInfo *SafetyInfo,
154 TargetTransformInfo *TTI, bool &FreeInLoop);
155static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
156 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
157 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
158 OptimizationRemarkEmitter *ORE);
159static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
160 BlockFrequencyInfo *BFI, const Loop *CurLoop,
161 ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
162 OptimizationRemarkEmitter *ORE);
163static bool isSafeToExecuteUnconditionally(Instruction &Inst,
164 const DominatorTree *DT,
165 const TargetLibraryInfo *TLI,
166 const Loop *CurLoop,
167 const LoopSafetyInfo *SafetyInfo,
168 OptimizationRemarkEmitter *ORE,
169 const Instruction *CtxI = nullptr);
170static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
171 AliasSetTracker *CurAST, Loop *CurLoop,
172 AAResults *AA);
173static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
174 Loop *CurLoop, Instruction &I,
175 SinkAndHoistLICMFlags &Flags);
176static bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
177 MemoryUse &MU);
178static Instruction *cloneInstructionInExitBlock(
179 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
180 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
181
182static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
183 AliasSetTracker *AST, MemorySSAUpdater *MSSAU);
184
185static void moveInstructionBefore(Instruction &I, Instruction &Dest,
186 ICFLoopSafetyInfo &SafetyInfo,
187 MemorySSAUpdater *MSSAU, ScalarEvolution *SE);
188
189static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
190 function_ref<void(Instruction *)> Fn);
191static SmallVector<SmallSetVector<Value *, 8>, 0>
192collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L);
193
194namespace {
195struct LoopInvariantCodeMotion {
196 bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
197 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
198 TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
199 OptimizationRemarkEmitter *ORE);
200
201 LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
202 unsigned LicmMssaNoAccForPromotionCap)
203 : LicmMssaOptCap(LicmMssaOptCap),
204 LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}
205
206private:
207 unsigned LicmMssaOptCap;
208 unsigned LicmMssaNoAccForPromotionCap;
209
210 std::unique_ptr<AliasSetTracker>
211 collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AAResults *AA);
212};
213
214struct LegacyLICMPass : public LoopPass {
215 static char ID; // Pass identification, replacement for typeid
216 LegacyLICMPass(
217 unsigned LicmMssaOptCap = SetLicmMssaOptCap,
218 unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
219 : LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
220 initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
221 }
222
223 bool runOnLoop(Loop *L, LPPassManager &LPM) override {
224 if (skipLoop(L))
225 return false;
226
227 LLVM_DEBUG(dbgs() << "Perform LICM on Loop with header at block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
<< L->getHeader()->getNameOrAsOperand() <<
"\n"; } } while (false)
228 << L->getHeader()->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
<< L->getHeader()->getNameOrAsOperand() <<
"\n"; } } while (false)
;
229
230 auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
231 MemorySSA *MSSA = EnableMSSALoopDependency
232 ? (&getAnalysis<MemorySSAWrapperPass>().getMSSA())
233 : nullptr;
234 bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
235 BlockFrequencyInfo *BFI =
236 hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
237 : nullptr;
238 // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
239 // pass. Function analyses need to be preserved across loop transformations
240 // but ORE cannot be preserved (see comment before the pass definition).
241 OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
242 return LICM.runOnLoop(
243 L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
244 &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
245 &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
246 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
247 *L->getHeader()->getParent()),
248 &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
249 *L->getHeader()->getParent()),
250 SE ? &SE->getSE() : nullptr, MSSA, &ORE);
251 }
252
253 /// This transformation requires natural loop information & requires that
254 /// loop preheaders be inserted into the CFG...
255 ///
256 void getAnalysisUsage(AnalysisUsage &AU) const override {
257 AU.addPreserved<DominatorTreeWrapperPass>();
258 AU.addPreserved<LoopInfoWrapperPass>();
259 AU.addRequired<TargetLibraryInfoWrapperPass>();
260 if (EnableMSSALoopDependency) {
261 AU.addRequired<MemorySSAWrapperPass>();
262 AU.addPreserved<MemorySSAWrapperPass>();
263 }
264 AU.addRequired<TargetTransformInfoWrapperPass>();
265 getLoopAnalysisUsage(AU);
266 LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
267 AU.addPreserved<LazyBlockFrequencyInfoPass>();
268 AU.addPreserved<LazyBranchProbabilityInfoPass>();
269 }
270
271private:
272 LoopInvariantCodeMotion LICM;
273};
274} // namespace
275
276PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
277 LoopStandardAnalysisResults &AR, LPMUpdater &) {
278 // For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
279 // pass. Function analyses need to be preserved across loop transformations
280 // but ORE cannot be preserved (see comment before the pass definition).
281 OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
282
283 LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
284 if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
285 &AR.SE, AR.MSSA, &ORE))
286 return PreservedAnalyses::all();
287
288 auto PA = getLoopPassPreservedAnalyses();
289
290 PA.preserve<DominatorTreeAnalysis>();
291 PA.preserve<LoopAnalysis>();
292 if (AR.MSSA)
293 PA.preserve<MemorySSAAnalysis>();
294
295 return PA;
296}
297
298char LegacyLICMPass::ID = 0;
299INITIALIZE_PASS_BEGIN(LegacyLICMPass, "licm", "Loop Invariant Code Motion",static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
300 false, false)static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
301INITIALIZE_PASS_DEPENDENCY(LoopPass)initializeLoopPassPass(Registry);
302INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
303INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
304INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry);
305INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)initializeLazyBFIPassPass(Registry);
306INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
307 false)PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
308
309Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
310Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
311 unsigned LicmMssaNoAccForPromotionCap) {
312 return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
313}
314
315llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(bool IsSink, Loop *L,
316 MemorySSA *MSSA)
317 : SinkAndHoistLICMFlags(SetLicmMssaOptCap, SetLicmMssaNoAccForPromotionCap,
318 IsSink, L, MSSA) {}
319
320llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(
321 unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
322 Loop *L, MemorySSA *MSSA)
323 : LicmMssaOptCap(LicmMssaOptCap),
324 LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap),
325 IsSink(IsSink) {
326 assert(((L != nullptr) == (MSSA != nullptr)) &&(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 327, __extension__ __PRETTY_FUNCTION__))
327 "Unexpected values for SinkAndHoistLICMFlags")(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 327, __extension__ __PRETTY_FUNCTION__))
;
328 if (!MSSA)
329 return;
330
331 unsigned AccessCapCount = 0;
332 for (auto *BB : L->getBlocks())
333 if (const auto *Accesses = MSSA->getBlockAccesses(BB))
334 for (const auto &MA : *Accesses) {
335 (void)MA;
336 ++AccessCapCount;
337 if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
338 NoOfMemAccTooLarge = true;
339 return;
340 }
341 }
342}
343
344/// Hoist expressions out of the specified loop. Note, alias info for inner
345/// loop is not preserved so it is not a good idea to run LICM multiple
346/// times on one loop.
347bool LoopInvariantCodeMotion::runOnLoop(
348 Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
349 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
350 ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
351 bool Changed = false;
352
353 assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop is not in LCSSA form."
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop is not in LCSSA form.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 353, __extension__ __PRETTY_FUNCTION__))
;
354
355 // If this loop has metadata indicating that LICM is not to be performed then
356 // just exit.
357 if (hasDisableLICMTransformsHint(L)) {
358 return false;
359 }
360
361 std::unique_ptr<AliasSetTracker> CurAST;
362 std::unique_ptr<MemorySSAUpdater> MSSAU;
363 std::unique_ptr<SinkAndHoistLICMFlags> Flags;
364
365 // Don't sink stores from loops with coroutine suspend instructions.
366 // LICM would sink instructions into the default destination of
367 // the coroutine switch. The default destination of the switch is to
368 // handle the case where the coroutine is suspended, by which point the
369 // coroutine frame may have been destroyed. No instruction can be sunk there.
370 // FIXME: This would unfortunately hurt the performance of coroutines, however
371 // there is currently no general solution for this. Similar issues could also
372 // potentially happen in other passes where instructions are being moved
373 // across that edge.
374 bool HasCoroSuspendInst = llvm::any_of(L->getBlocks(), [](BasicBlock *BB) {
375 return llvm::any_of(*BB, [](Instruction &I) {
376 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
377 return II && II->getIntrinsicID() == Intrinsic::coro_suspend;
378 });
379 });
380
381 if (!MSSA) {
382 LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Using Alias Set Tracker.\n"
; } } while (false)
;
383 CurAST = collectAliasInfoForLoop(L, LI, AA);
384 Flags = std::make_unique<SinkAndHoistLICMFlags>(
385 LicmMssaOptCap, LicmMssaNoAccForPromotionCap, /*IsSink=*/true);
386 } else {
387 LLVM_DEBUG(dbgs() << "LICM: Using MemorySSA.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Using MemorySSA.\n"; } } while
(false)
;
388 MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
389 Flags = std::make_unique<SinkAndHoistLICMFlags>(
390 LicmMssaOptCap, LicmMssaNoAccForPromotionCap, /*IsSink=*/true, L, MSSA);
391 }
392
393 // Get the preheader block to move instructions into...
394 BasicBlock *Preheader = L->getLoopPreheader();
395
396 // Compute loop safety information.
397 ICFLoopSafetyInfo SafetyInfo;
398 SafetyInfo.computeLoopSafetyInfo(L);
399
400 // We want to visit all of the instructions in this loop... that are not parts
401 // of our subloops (they have already had their invariants hoisted out of
402 // their loop, into this loop, so there is no need to process the BODIES of
403 // the subloops).
404 //
405 // Traverse the body of the loop in depth first order on the dominator tree so
406 // that we are guaranteed to see definitions before we see uses. This allows
407 // us to sink instructions in one pass, without iteration. After sinking
408 // instructions, we perform another pass to hoist them out of the loop.
409 if (L->hasDedicatedExits())
410 Changed |=
411 sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, TTI, L,
412 CurAST.get(), MSSAU.get(), &SafetyInfo, *Flags.get(), ORE);
413 Flags->setIsSink(false);
414 if (Preheader)
415 Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, L,
416 CurAST.get(), MSSAU.get(), SE, &SafetyInfo,
417 *Flags.get(), ORE);
418
419 // Now that all loop invariants have been removed from the loop, promote any
420 // memory references to scalars that we can.
421 // Don't sink stores from loops without dedicated block exits. Exits
422 // containing indirect branches are not transformed by loop simplify,
423 // make sure we catch that. An additional load may be generated in the
424 // preheader for SSA updater, so also avoid sinking when no preheader
425 // is available.
426 if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
427 !Flags->tooManyMemoryAccesses() && !HasCoroSuspendInst) {
428 // Figure out the loop exits and their insertion points
429 SmallVector<BasicBlock *, 8> ExitBlocks;
430 L->getUniqueExitBlocks(ExitBlocks);
431
432 // We can't insert into a catchswitch.
433 bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
434 return isa<CatchSwitchInst>(Exit->getTerminator());
435 });
436
437 if (!HasCatchSwitch) {
438 SmallVector<Instruction *, 8> InsertPts;
439 SmallVector<MemoryAccess *, 8> MSSAInsertPts;
440 InsertPts.reserve(ExitBlocks.size());
441 if (MSSAU)
442 MSSAInsertPts.reserve(ExitBlocks.size());
443 for (BasicBlock *ExitBlock : ExitBlocks) {
444 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
445 if (MSSAU)
446 MSSAInsertPts.push_back(nullptr);
447 }
448
449 PredIteratorCache PIC;
450
451 bool Promoted = false;
452 if (CurAST.get()) {
453 // Loop over all of the alias sets in the tracker object.
454 for (AliasSet &AS : *CurAST) {
455 // We can promote this alias set if it has a store, if it is a "Must"
456 // alias set, if the pointer is loop invariant, and if we are not
457 // eliminating any volatile loads or stores.
458 if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
459 !L->isLoopInvariant(AS.begin()->getValue()))
460 continue;
461
462 assert((static_cast <bool> (!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? void (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 464, __extension__ __PRETTY_FUNCTION__))
463 !AS.empty() &&(static_cast <bool> (!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? void (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 464, __extension__ __PRETTY_FUNCTION__))
464 "Must alias set should have at least one pointer element in it!")(static_cast <bool> (!AS.empty() && "Must alias set should have at least one pointer element in it!"
) ? void (0) : __assert_fail ("!AS.empty() && \"Must alias set should have at least one pointer element in it!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 464, __extension__ __PRETTY_FUNCTION__))
;
465
466 SmallSetVector<Value *, 8> PointerMustAliases;
467 for (const auto &ASI : AS)
468 PointerMustAliases.insert(ASI.getValue());
469
470 Promoted |= promoteLoopAccessesToScalars(
471 PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
472 DT, TLI, L, CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
473 }
474 } else {
475 // Promoting one set of accesses may make the pointers for another set
476 // loop invariant, so run this in a loop (with the MaybePromotable set
477 // decreasing in size over time).
478 bool LocalPromoted;
479 do {
480 LocalPromoted = false;
481 for (const SmallSetVector<Value *, 8> &PointerMustAliases :
482 collectPromotionCandidates(MSSA, AA, L)) {
483 LocalPromoted |= promoteLoopAccessesToScalars(
484 PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC,
485 LI, DT, TLI, L, /*AST*/nullptr, MSSAU.get(), &SafetyInfo, ORE);
486 }
487 Promoted |= LocalPromoted;
488 } while (LocalPromoted);
489 }
490
491 // Once we have promoted values across the loop body we have to
492 // recursively reform LCSSA as any nested loop may now have values defined
493 // within the loop used in the outer loop.
494 // FIXME: This is really heavy handed. It would be a bit better to use an
495 // SSAUpdater strategy during promotion that was LCSSA aware and reformed
496 // it as it went.
497 if (Promoted)
498 formLCSSARecursively(*L, *DT, LI, SE);
499
500 Changed |= Promoted;
501 }
502 }
503
504 // Check that neither this loop nor its parent have had LCSSA broken. LICM is
505 // specifically moving instructions across the loop boundary and so it is
506 // especially in need of sanity checking here.
507 assert(L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 507, __extension__ __PRETTY_FUNCTION__))
;
508 assert((L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) &&(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 509, __extension__ __PRETTY_FUNCTION__))
509 "Parent loop not left in LCSSA form after LICM!")(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 509, __extension__ __PRETTY_FUNCTION__))
;
510
511 if (MSSAU.get() && VerifyMemorySSA)
512 MSSAU->getMemorySSA()->verifyMemorySSA();
513
514 if (Changed && SE)
515 SE->forgetLoopDispositions(L);
516 return Changed;
517}
518
519/// Walk the specified region of the CFG (defined by all blocks dominated by
520/// the specified block, and that are in the current loop) in reverse depth
521/// first order w.r.t the DominatorTree. This allows us to visit uses before
522/// definitions, allowing us to sink a loop body in one pass without iteration.
523///
524bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
525 DominatorTree *DT, BlockFrequencyInfo *BFI,
526 TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
527 Loop *CurLoop, AliasSetTracker *CurAST,
528 MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
529 SinkAndHoistLICMFlags &Flags,
530 OptimizationRemarkEmitter *ORE) {
531
532 // Verify inputs.
533 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && SafetyInfo != nullptr &&
"Unexpected input to sinkRegion.") ? void (0) : __assert_fail
("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 535, __extension__ __PRETTY_FUNCTION__))
534 CurLoop != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && SafetyInfo != nullptr &&
"Unexpected input to sinkRegion.") ? void (0) : __assert_fail
("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 535, __extension__ __PRETTY_FUNCTION__))
535 "Unexpected input to sinkRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && SafetyInfo != nullptr &&
"Unexpected input to sinkRegion.") ? void (0) : __assert_fail
("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 535, __extension__ __PRETTY_FUNCTION__))
;
536 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 537, __extension__ __PRETTY_FUNCTION__))
537 "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 537, __extension__ __PRETTY_FUNCTION__))
;
538
539 // We want to visit children before parents. We will enque all the parents
540 // before their children in the worklist and process the worklist in reverse
541 // order.
542 SmallVector<DomTreeNode *, 16> Worklist = collectChildrenInLoop(N, CurLoop);
543
544 bool Changed = false;
545 for (DomTreeNode *DTN : reverse(Worklist)) {
546 BasicBlock *BB = DTN->getBlock();
547 // Only need to process the contents of this block if it is not part of a
548 // subloop (which would already have been processed).
549 if (inSubLoop(BB, CurLoop, LI))
550 continue;
551
552 for (BasicBlock::iterator II = BB->end(); II != BB->begin();) {
553 Instruction &I = *--II;
554
555 // The instruction is not used in the loop if it is dead. In this case,
556 // we just delete it instead of sinking it.
557 if (isInstructionTriviallyDead(&I, TLI)) {
558 LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM deleting dead inst: " <<
I << '\n'; } } while (false)
;
559 salvageKnowledge(&I);
560 salvageDebugInfo(I);
561 ++II;
562 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
563 Changed = true;
564 continue;
565 }
566
567 // Check to see if we can sink this instruction to the exit blocks
568 // of the loop. We can do this if the all users of the instruction are
569 // outside of the loop. In this case, it doesn't even matter if the
570 // operands of the instruction are loop invariant.
571 //
572 bool FreeInLoop = false;
573 if (!I.mayHaveSideEffects() &&
574 isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
575 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
576 ORE)) {
577 if (sink(I, LI, DT, BFI, CurLoop, SafetyInfo, MSSAU, ORE)) {
578 if (!FreeInLoop) {
579 ++II;
580 salvageDebugInfo(I);
581 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
582 }
583 Changed = true;
584 }
585 }
586 }
587 }
588 if (MSSAU && VerifyMemorySSA)
589 MSSAU->getMemorySSA()->verifyMemorySSA();
590 return Changed;
591}
592
593namespace {
594// This is a helper class for hoistRegion to make it able to hoist control flow
595// in order to be able to hoist phis. The way this works is that we initially
596// start hoisting to the loop preheader, and when we see a loop invariant branch
597// we make note of this. When we then come to hoist an instruction that's
598// conditional on such a branch we duplicate the branch and the relevant control
599// flow, then hoist the instruction into the block corresponding to its original
600// block in the duplicated control flow.
601class ControlFlowHoister {
602private:
603 // Information about the loop we are hoisting from
604 LoopInfo *LI;
605 DominatorTree *DT;
606 Loop *CurLoop;
607 MemorySSAUpdater *MSSAU;
608
609 // A map of blocks in the loop to the block their instructions will be hoisted
610 // to.
611 DenseMap<BasicBlock *, BasicBlock *> HoistDestinationMap;
612
613 // The branches that we can hoist, mapped to the block that marks a
614 // convergence point of their control flow.
615 DenseMap<BranchInst *, BasicBlock *> HoistableBranches;
616
617public:
618 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
619 MemorySSAUpdater *MSSAU)
620 : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}
621
622 void registerPossiblyHoistableBranch(BranchInst *BI) {
623 // We can only hoist conditional branches with loop invariant operands.
624 if (!ControlFlowHoisting || !BI->isConditional() ||
625 !CurLoop->hasLoopInvariantOperands(BI))
626 return;
627
628 // The branch destinations need to be in the loop, and we don't gain
629 // anything by duplicating conditional branches with duplicate successors,
630 // as it's essentially the same as an unconditional branch.
631 BasicBlock *TrueDest = BI->getSuccessor(0);
632 BasicBlock *FalseDest = BI->getSuccessor(1);
633 if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
634 TrueDest == FalseDest)
635 return;
636
637 // We can hoist BI if one branch destination is the successor of the other,
638 // or both have common successor which we check by seeing if the
639 // intersection of their successors is non-empty.
640 // TODO: This could be expanded to allowing branches where both ends
641 // eventually converge to a single block.
642 SmallPtrSet<BasicBlock *, 4> TrueDestSucc, FalseDestSucc;
643 TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
644 FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
645 BasicBlock *CommonSucc = nullptr;
646 if (TrueDestSucc.count(FalseDest)) {
647 CommonSucc = FalseDest;
648 } else if (FalseDestSucc.count(TrueDest)) {
649 CommonSucc = TrueDest;
650 } else {
651 set_intersect(TrueDestSucc, FalseDestSucc);
652 // If there's one common successor use that.
653 if (TrueDestSucc.size() == 1)
654 CommonSucc = *TrueDestSucc.begin();
655 // If there's more than one pick whichever appears first in the block list
656 // (we can't use the value returned by TrueDestSucc.begin() as it's
657 // unpredicatable which element gets returned).
658 else if (!TrueDestSucc.empty()) {
659 Function *F = TrueDest->getParent();
660 auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
661 auto It = llvm::find_if(*F, IsSucc);
662 assert(It != F->end() && "Could not find successor in function")(static_cast <bool> (It != F->end() && "Could not find successor in function"
) ? void (0) : __assert_fail ("It != F->end() && \"Could not find successor in function\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 662, __extension__ __PRETTY_FUNCTION__))
;
663 CommonSucc = &*It;
664 }
665 }
666 // The common successor has to be dominated by the branch, as otherwise
667 // there will be some other path to the successor that will not be
668 // controlled by this branch so any phi we hoist would be controlled by the
669 // wrong condition. This also takes care of avoiding hoisting of loop back
670 // edges.
671 // TODO: In some cases this could be relaxed if the successor is dominated
672 // by another block that's been hoisted and we can guarantee that the
673 // control flow has been replicated exactly.
674 if (CommonSucc && DT->dominates(BI, CommonSucc))
675 HoistableBranches[BI] = CommonSucc;
676 }
677
678 bool canHoistPHI(PHINode *PN) {
679 // The phi must have loop invariant operands.
680 if (!ControlFlowHoisting || !CurLoop->hasLoopInvariantOperands(PN))
681 return false;
682 // We can hoist phis if the block they are in is the target of hoistable
683 // branches which cover all of the predecessors of the block.
684 SmallPtrSet<BasicBlock *, 8> PredecessorBlocks;
685 BasicBlock *BB = PN->getParent();
686 for (BasicBlock *PredBB : predecessors(BB))
687 PredecessorBlocks.insert(PredBB);
688 // If we have less predecessor blocks than predecessors then the phi will
689 // have more than one incoming value for the same block which we can't
690 // handle.
691 // TODO: This could be handled be erasing some of the duplicate incoming
692 // values.
693 if (PredecessorBlocks.size() != pred_size(BB))
694 return false;
695 for (auto &Pair : HoistableBranches) {
696 if (Pair.second == BB) {
697 // Which blocks are predecessors via this branch depends on if the
698 // branch is triangle-like or diamond-like.
699 if (Pair.first->getSuccessor(0) == BB) {
700 PredecessorBlocks.erase(Pair.first->getParent());
701 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
702 } else if (Pair.first->getSuccessor(1) == BB) {
703 PredecessorBlocks.erase(Pair.first->getParent());
704 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
705 } else {
706 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
707 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
708 }
709 }
710 }
711 // PredecessorBlocks will now be empty if for every predecessor of BB we
712 // found a hoistable branch source.
713 return PredecessorBlocks.empty();
714 }
715
716 BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
717 if (!ControlFlowHoisting)
718 return CurLoop->getLoopPreheader();
719 // If BB has already been hoisted, return that
720 if (HoistDestinationMap.count(BB))
721 return HoistDestinationMap[BB];
722
723 // Check if this block is conditional based on a pending branch
724 auto HasBBAsSuccessor =
725 [&](DenseMap<BranchInst *, BasicBlock *>::value_type &Pair) {
726 return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
727 Pair.first->getSuccessor(1) == BB);
728 };
729 auto It = llvm::find_if(HoistableBranches, HasBBAsSuccessor);
730
731 // If not involved in a pending branch, hoist to preheader
732 BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
733 if (It == HoistableBranches.end()) {
734 LLVM_DEBUG(dbgs() << "LICM using "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
735 << InitialPreheader->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
736 << " as hoist destination for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
737 << BB->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
;
738 HoistDestinationMap[BB] = InitialPreheader;
739 return InitialPreheader;
740 }
741 BranchInst *BI = It->first;
742 assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 744, __extension__ __PRETTY_FUNCTION__))
743 HoistableBranches.end() &&(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 744, __extension__ __PRETTY_FUNCTION__))
744 "BB is expected to be the target of at most one branch")(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 744, __extension__ __PRETTY_FUNCTION__))
;
745
746 LLVMContext &C = BB->getContext();
747 BasicBlock *TrueDest = BI->getSuccessor(0);
748 BasicBlock *FalseDest = BI->getSuccessor(1);
749 BasicBlock *CommonSucc = HoistableBranches[BI];
750 BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());
751
752 // Create hoisted versions of blocks that currently don't have them
753 auto CreateHoistedBlock = [&](BasicBlock *Orig) {
754 if (HoistDestinationMap.count(Orig))
755 return HoistDestinationMap[Orig];
756 BasicBlock *New =
757 BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
758 HoistDestinationMap[Orig] = New;
759 DT->addNewBlock(New, HoistTarget);
760 if (CurLoop->getParentLoop())
761 CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
762 ++NumCreatedBlocks;
763 LLVM_DEBUG(dbgs() << "LICM created " << New->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
764 << " as hoist destination for " << Orig->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
765 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
;
766 return New;
767 };
768 BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
769 BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
770 BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);
771
772 // Link up these blocks with branches.
773 if (!HoistCommonSucc->getTerminator()) {
774 // The new common successor we've generated will branch to whatever that
775 // hoist target branched to.
776 BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
777 assert(TargetSucc && "Expected hoist target to have a single successor")(static_cast <bool> (TargetSucc && "Expected hoist target to have a single successor"
) ? void (0) : __assert_fail ("TargetSucc && \"Expected hoist target to have a single successor\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 777, __extension__ __PRETTY_FUNCTION__))
;
778 HoistCommonSucc->moveBefore(TargetSucc);
779 BranchInst::Create(TargetSucc, HoistCommonSucc);
780 }
781 if (!HoistTrueDest->getTerminator()) {
782 HoistTrueDest->moveBefore(HoistCommonSucc);
783 BranchInst::Create(HoistCommonSucc, HoistTrueDest);
784 }
785 if (!HoistFalseDest->getTerminator()) {
786 HoistFalseDest->moveBefore(HoistCommonSucc);
787 BranchInst::Create(HoistCommonSucc, HoistFalseDest);
788 }
789
790 // If BI is being cloned to what was originally the preheader then
791 // HoistCommonSucc will now be the new preheader.
792 if (HoistTarget == InitialPreheader) {
793 // Phis in the loop header now need to use the new preheader.
794 InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
795 if (MSSAU)
796 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
797 HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
798 // The new preheader dominates the loop header.
799 DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
800 DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
801 DT->changeImmediateDominator(HeaderNode, PreheaderNode);
802 // The preheader hoist destination is now the new preheader, with the
803 // exception of the hoist destination of this branch.
804 for (auto &Pair : HoistDestinationMap)
805 if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
806 Pair.second = HoistCommonSucc;
807 }
808
809 // Now finally clone BI.
810 ReplaceInstWithInst(
811 HoistTarget->getTerminator(),
812 BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
813 ++NumClonedBranches;
814
815 assert(CurLoop->getLoopPreheader() &&(static_cast <bool> (CurLoop->getLoopPreheader() &&
"Hoisting blocks should not have destroyed preheader") ? void
(0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 816, __extension__ __PRETTY_FUNCTION__))
816 "Hoisting blocks should not have destroyed preheader")(static_cast <bool> (CurLoop->getLoopPreheader() &&
"Hoisting blocks should not have destroyed preheader") ? void
(0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 816, __extension__ __PRETTY_FUNCTION__))
;
817 return HoistDestinationMap[BB];
818 }
819};
820} // namespace
821
822// Hoisting/sinking instruction out of a loop isn't always beneficial. It's only
823// only worthwhile if the destination block is actually colder than current
824// block.
825static bool worthSinkOrHoistInst(Instruction &I, BasicBlock *DstBlock,
826 OptimizationRemarkEmitter *ORE,
827 BlockFrequencyInfo *BFI) {
828 // Check block frequency only when runtime profile is available
829 // to avoid pathological cases. With static profile, lean towards
830 // hosting because it helps canonicalize the loop for vectorizer.
831 if (!DstBlock->getParent()->hasProfileData())
832 return true;
833
834 if (!HoistSinkColdnessThreshold || !BFI)
835 return true;
836
837 BasicBlock *SrcBlock = I.getParent();
838 if (BFI->getBlockFreq(DstBlock).getFrequency() / HoistSinkColdnessThreshold >
839 BFI->getBlockFreq(SrcBlock).getFrequency()) {
840 ORE->emit([&]() {
841 return OptimizationRemarkMissed(DEBUG_TYPE"licm", "SinkHoistInst", &I)
842 << "failed to sink or hoist instruction because containing block "
843 "has lower frequency than destination block";
844 });
845 return false;
846 }
847
848 return true;
849}
850
851/// Walk the specified region of the CFG (defined by all blocks dominated by
852/// the specified block, and that are in the current loop) in depth first
853/// order w.r.t the DominatorTree. This allows us to visit definitions before
854/// uses, allowing us to hoist a loop body in one pass without iteration.
855///
856bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
857 DominatorTree *DT, BlockFrequencyInfo *BFI,
858 TargetLibraryInfo *TLI, Loop *CurLoop,
859 AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
860 ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo,
861 SinkAndHoistLICMFlags &Flags,
862 OptimizationRemarkEmitter *ORE) {
863 // Verify inputs.
864 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && SafetyInfo != nullptr &&
"Unexpected input to hoistRegion.") ? void (0) : __assert_fail
("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 866, __extension__ __PRETTY_FUNCTION__))
865 CurLoop != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && SafetyInfo != nullptr &&
"Unexpected input to hoistRegion.") ? void (0) : __assert_fail
("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 866, __extension__ __PRETTY_FUNCTION__))
866 "Unexpected input to hoistRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && SafetyInfo != nullptr &&
"Unexpected input to hoistRegion.") ? void (0) : __assert_fail
("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 866, __extension__ __PRETTY_FUNCTION__))
;
867 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 868, __extension__ __PRETTY_FUNCTION__))
868 "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 868, __extension__ __PRETTY_FUNCTION__))
;
869
870 ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);
871
872 // Keep track of instructions that have been hoisted, as they may need to be
873 // re-hoisted if they end up not dominating all of their uses.
874 SmallVector<Instruction *, 16> HoistedInstructions;
875
876 // For PHI hoisting to work we need to hoist blocks before their successors.
877 // We can do this by iterating through the blocks in the loop in reverse
878 // post-order.
879 LoopBlocksRPO Worklist(CurLoop);
880 Worklist.perform(LI);
881 bool Changed = false;
882 for (BasicBlock *BB : Worklist) {
883 // Only need to process the contents of this block if it is not part of a
884 // subloop (which would already have been processed).
885 if (inSubLoop(BB, CurLoop, LI))
886 continue;
887
888 for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E;) {
889 Instruction &I = *II++;
890 // Try constant folding this instruction. If all the operands are
891 // constants, it is technically hoistable, but it would be better to
892 // just fold it.
893 if (Constant *C = ConstantFoldInstruction(
894 &I, I.getModule()->getDataLayout(), TLI)) {
895 LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << " --> " << *Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
" --> " << *C << '\n'; } } while (false)
896 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
" --> " << *C << '\n'; } } while (false)
;
897 if (CurAST)
898 CurAST->copyValue(&I, C);
899 // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
900 I.replaceAllUsesWith(C);
901 if (isInstructionTriviallyDead(&I, TLI))
902 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
903 Changed = true;
904 continue;
905 }
906
907 // Try hoisting the instruction out to the preheader. We can only do
908 // this if all of the operands of the instruction are loop invariant and
909 // if it is safe to hoist the instruction. We also check block frequency
910 // to make sure instruction only gets hoisted into colder blocks.
911 // TODO: It may be safe to hoist if we are hoisting to a conditional block
912 // and we have accurately duplicated the control flow from the loop header
913 // to that block.
914 if (CurLoop->hasLoopInvariantOperands(&I) &&
915 canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
916 ORE) &&
917 worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
918 isSafeToExecuteUnconditionally(
919 I, DT, TLI, CurLoop, SafetyInfo, ORE,
920 CurLoop->getLoopPreheader()->getTerminator())) {
921 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
922 MSSAU, SE, ORE);
923 HoistedInstructions.push_back(&I);
924 Changed = true;
925 continue;
926 }
927
928 // Attempt to remove floating point division out of the loop by
929 // converting it to a reciprocal multiplication.
930 if (I.getOpcode() == Instruction::FDiv && I.hasAllowReciprocal() &&
931 CurLoop->isLoopInvariant(I.getOperand(1))) {
932 auto Divisor = I.getOperand(1);
933 auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
934 auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
935 ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
936 SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
937 ReciprocalDivisor->insertBefore(&I);
938
939 auto Product =
940 BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
941 Product->setFastMathFlags(I.getFastMathFlags());
942 SafetyInfo->insertInstructionTo(Product, I.getParent());
943 Product->insertAfter(&I);
944 I.replaceAllUsesWith(Product);
945 eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
946
947 hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
948 SafetyInfo, MSSAU, SE, ORE);
949 HoistedInstructions.push_back(ReciprocalDivisor);
950 Changed = true;
951 continue;
952 }
953
954 auto IsInvariantStart = [&](Instruction &I) {
955 using namespace PatternMatch;
956 return I.use_empty() &&
957 match(&I, m_Intrinsic<Intrinsic::invariant_start>());
958 };
959 auto MustExecuteWithoutWritesBefore = [&](Instruction &I) {
960 return SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
961 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop);
962 };
963 if ((IsInvariantStart(I) || isGuard(&I)) &&
964 CurLoop->hasLoopInvariantOperands(&I) &&
965 MustExecuteWithoutWritesBefore(I)) {
966 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
967 MSSAU, SE, ORE);
968 HoistedInstructions.push_back(&I);
969 Changed = true;
970 continue;
971 }
972
973 if (PHINode *PN = dyn_cast<PHINode>(&I)) {
974 if (CFH.canHoistPHI(PN)) {
975 // Redirect incoming blocks first to ensure that we create hoisted
976 // versions of those blocks before we hoist the phi.
977 for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
978 PN->setIncomingBlock(
979 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
980 hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
981 MSSAU, SE, ORE);
982 assert(DT->dominates(PN, BB) && "Conditional PHIs not expected")(static_cast <bool> (DT->dominates(PN, BB) &&
"Conditional PHIs not expected") ? void (0) : __assert_fail (
"DT->dominates(PN, BB) && \"Conditional PHIs not expected\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 982, __extension__ __PRETTY_FUNCTION__))
;
983 Changed = true;
984 continue;
985 }
986 }
987
988 // Remember possibly hoistable branches so we can actually hoist them
989 // later if needed.
990 if (BranchInst *BI = dyn_cast<BranchInst>(&I))
991 CFH.registerPossiblyHoistableBranch(BI);
992 }
993 }
994
995 // If we hoisted instructions to a conditional block they may not dominate
996 // their uses that weren't hoisted (such as phis where some operands are not
997 // loop invariant). If so make them unconditional by moving them to their
998 // immediate dominator. We iterate through the instructions in reverse order
999 // which ensures that when we rehoist an instruction we rehoist its operands,
1000 // and also keep track of where in the block we are rehoisting to to make sure
1001 // that we rehoist instructions before the instructions that use them.
1002 Instruction *HoistPoint = nullptr;
1003 if (ControlFlowHoisting) {
1004 for (Instruction *I : reverse(HoistedInstructions)) {
1005 if (!llvm::all_of(I->uses(),
1006 [&](Use &U) { return DT->dominates(I, U); })) {
1007 BasicBlock *Dominator =
1008 DT->getNode(I->getParent())->getIDom()->getBlock();
1009 if (!HoistPoint || !DT->dominates(HoistPoint->getParent(), Dominator)) {
1010 if (HoistPoint)
1011 assert(DT->dominates(Dominator, HoistPoint->getParent()) &&(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1012, __extension__ __PRETTY_FUNCTION__))
1012 "New hoist point expected to dominate old hoist point")(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1012, __extension__ __PRETTY_FUNCTION__))
;
1013 HoistPoint = Dominator->getTerminator();
1014 }
1015 LLVM_DEBUG(dbgs() << "LICM rehoisting to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
1016 << HoistPoint->getParent()->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
1017 << ": " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
;
1018 moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
1019 HoistPoint = I;
1020 Changed = true;
1021 }
1022 }
1023 }
1024 if (MSSAU && VerifyMemorySSA)
1025 MSSAU->getMemorySSA()->verifyMemorySSA();
1026
1027 // Now that we've finished hoisting make sure that LI and DT are still
1028 // valid.
1029#ifdef EXPENSIVE_CHECKS
1030 if (Changed) {
1031 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
(0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1032, __extension__ __PRETTY_FUNCTION__))
1032 "Dominator tree verification failed")(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
(0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1032, __extension__ __PRETTY_FUNCTION__))
;
1033 LI->verify(*DT);
1034 }
1035#endif
1036
1037 return Changed;
1038}
1039
1040// Return true if LI is invariant within scope of the loop. LI is invariant if
1041// CurLoop is dominated by an invariant.start representing the same memory
1042// location and size as the memory location LI loads from, and also the
1043// invariant.start has no uses.
1044static bool isLoadInvariantInLoop(LoadInst *LI, DominatorTree *DT,
1045 Loop *CurLoop) {
1046 Value *Addr = LI->getOperand(0);
1047 const DataLayout &DL = LI->getModule()->getDataLayout();
1048 const TypeSize LocSizeInBits = DL.getTypeSizeInBits(LI->getType());
1049
1050 // It is not currently possible for clang to generate an invariant.start
1051 // intrinsic with scalable vector types because we don't support thread local
1052 // sizeless types and we don't permit sizeless types in structs or classes.
1053 // Furthermore, even if support is added for this in future the intrinsic
1054 // itself is defined to have a size of -1 for variable sized objects. This
1055 // makes it impossible to verify if the intrinsic envelops our region of
1056 // interest. For example, both <vscale x 32 x i8> and <vscale x 16 x i8>
1057 // types would have a -1 parameter, but the former is clearly double the size
1058 // of the latter.
1059 if (LocSizeInBits.isScalable())
30
Calling 'LinearPolySize::isScalable'
33
Returning from 'LinearPolySize::isScalable'
34
Taking true branch
1060 return false;
35
Returning zero, which participates in a condition later
1061
1062 // if the type is i8 addrspace(x)*, we know this is the type of
1063 // llvm.invariant.start operand
1064 auto *PtrInt8Ty = PointerType::get(Type::getInt8Ty(LI->getContext()),
1065 LI->getPointerAddressSpace());
1066 unsigned BitcastsVisited = 0;
1067 // Look through bitcasts until we reach the i8* type (this is invariant.start
1068 // operand type).
1069 while (Addr->getType() != PtrInt8Ty) {
1070 auto *BC = dyn_cast<BitCastInst>(Addr);
1071 // Avoid traversing high number of bitcast uses.
1072 if (++BitcastsVisited > MaxNumUsesTraversed || !BC)
1073 return false;
1074 Addr = BC->getOperand(0);
1075 }
1076
1077 unsigned UsesVisited = 0;
1078 // Traverse all uses of the load operand value, to see if invariant.start is
1079 // one of the uses, and whether it dominates the load instruction.
1080 for (auto *U : Addr->users()) {
1081 // Avoid traversing for Load operand with high number of users.
1082 if (++UsesVisited > MaxNumUsesTraversed)
1083 return false;
1084 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
1085 // If there are escaping uses of invariant.start instruction, the load maybe
1086 // non-invariant.
1087 if (!II || II->getIntrinsicID() != Intrinsic::invariant_start ||
1088 !II->use_empty())
1089 continue;
1090 ConstantInt *InvariantSize = cast<ConstantInt>(II->getArgOperand(0));
1091 // The intrinsic supports having a -1 argument for variable sized objects
1092 // so we should check for that here.
1093 if (InvariantSize->isNegative())
1094 continue;
1095 uint64_t InvariantSizeInBits = InvariantSize->getSExtValue() * 8;
1096 // Confirm the invariant.start location size contains the load operand size
1097 // in bits. Also, the invariant.start should dominate the load, and we
1098 // should not hoist the load out of a loop that contains this dominating
1099 // invariant.start.
1100 if (LocSizeInBits.getFixedSize() <= InvariantSizeInBits &&
1101 DT->properlyDominates(II->getParent(), CurLoop->getHeader()))
1102 return true;
1103 }
1104
1105 return false;
1106}
1107
1108namespace {
1109/// Return true if-and-only-if we know how to (mechanically) both hoist and
1110/// sink a given instruction out of a loop. Does not address legality
1111/// concerns such as aliasing or speculation safety.
1112bool isHoistableAndSinkableInst(Instruction &I) {
1113 // Only these instructions are hoistable/sinkable.
1114 return (isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) ||
5
Assuming 'I' is a 'LoadInst'
6
Returning the value 1, which participates in a condition later
1115 isa<FenceInst>(I) || isa<CastInst>(I) || isa<UnaryOperator>(I) ||
1116 isa<BinaryOperator>(I) || isa<SelectInst>(I) ||
1117 isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
1118 isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
1119 isa<ShuffleVectorInst>(I) || isa<ExtractValueInst>(I) ||
1120 isa<InsertValueInst>(I) || isa<FreezeInst>(I));
1121}
1122/// Return true if all of the alias sets within this AST are known not to
1123/// contain a Mod, or if MSSA knows there are no MemoryDefs in the loop.
1124bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
1125 const Loop *L) {
1126 if (CurAST) {
1127 for (AliasSet &AS : *CurAST) {
1128 if (!AS.isForwardingAliasSet() && AS.isMod()) {
1129 return false;
1130 }
1131 }
1132 return true;
1133 } else { /*MSSAU*/
1134 for (auto *BB : L->getBlocks())
1135 if (MSSAU->getMemorySSA()->getBlockDefs(BB))
1136 return false;
1137 return true;
1138 }
1139}
1140
1141/// Return true if I is the only Instruction with a MemoryAccess in L.
1142bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
1143 const MemorySSAUpdater *MSSAU) {
1144 for (auto *BB : L->getBlocks())
1145 if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
1146 int NotAPhi = 0;
1147 for (const auto &Acc : *Accs) {
1148 if (isa<MemoryPhi>(&Acc))
1149 continue;
1150 const auto *MUD = cast<MemoryUseOrDef>(&Acc);
1151 if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
1152 return false;
1153 }
1154 }
1155 return true;
1156}
1157}
1158
1159bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
1160 Loop *CurLoop, AliasSetTracker *CurAST,
1161 MemorySSAUpdater *MSSAU,
1162 bool TargetExecutesOncePerLoop,
1163 SinkAndHoistLICMFlags *Flags,
1164 OptimizationRemarkEmitter *ORE) {
1165 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1166, __extension__ __PRETTY_FUNCTION__))
1
Assuming pointer value is null
2
Assuming the condition is true
3
'?' condition is true
1166 "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1166, __extension__ __PRETTY_FUNCTION__))
;
1167
1168 // If we don't understand the instruction, bail early.
1169 if (!isHoistableAndSinkableInst(I))
4
Calling 'isHoistableAndSinkableInst'
7
Returning from 'isHoistableAndSinkableInst'
8
Taking false branch
1170 return false;
1171
1172 MemorySSA *MSSA = MSSAU
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
? MSSAU->getMemorySSA() : nullptr;
9
'?' condition is true
10
'MSSA' initialized here
1173 if (MSSA)
11
Assuming 'MSSA' is null
12
Taking false branch
1174 assert(Flags != nullptr && "Flags cannot be null.")(static_cast <bool> (Flags != nullptr && "Flags cannot be null."
) ? void (0) : __assert_fail ("Flags != nullptr && \"Flags cannot be null.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1174, __extension__ __PRETTY_FUNCTION__))
;
1175
1176 // Loads have extra constraints we have to verify before we can hoist them.
1177 if (LoadInst *LI
13.1
'LI' is non-null
13.1
'LI' is non-null
13.1
'LI' is non-null
13.1
'LI' is non-null
= dyn_cast<LoadInst>(&I)) {
13
Assuming the object is a 'LoadInst'
14
Taking true branch
1178 if (!LI->isUnordered())
15
Calling 'LoadInst::isUnordered'
19
Returning from 'LoadInst::isUnordered'
20
Taking false branch
1179 return false; // Don't sink/hoist volatile or ordered atomic loads!
1180
1181 // Loads from constant memory are always safe to move, even if they end up
1182 // in the same alias set as something that ends up being modified.
1183 if (AA->pointsToConstantMemory(LI->getOperand(0)))
21
Assuming the condition is false
22
Taking false branch
1184 return true;
1185 if (LI->hasMetadata(LLVMContext::MD_invariant_load))
23
Calling 'Instruction::hasMetadata'
26
Returning from 'Instruction::hasMetadata'
27
Taking false branch
1186 return true;
1187
1188 if (LI->isAtomic() && !TargetExecutesOncePerLoop)
28
Assuming the condition is false
1189 return false; // Don't risk duplicating unordered loads
1190
1191 // This checks for an invariant.start dominating the load.
1192 if (isLoadInvariantInLoop(LI, DT, CurLoop))
29
Calling 'isLoadInvariantInLoop'
36
Returning from 'isLoadInvariantInLoop'
37
Taking false branch
1193 return true;
1194
1195 bool Invalidated;
1196 if (CurAST
37.1
'CurAST' is null
37.1
'CurAST' is null
37.1
'CurAST' is null
37.1
'CurAST' is null
)
38
Taking false branch
1197 Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
1198 CurLoop, AA);
1199 else
1200 Invalidated = pointerInvalidatedByLoopWithMSSA(
1201 MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, I, *Flags);
39
Called C++ object pointer is null
1202 // Check loop-invariant address because this may also be a sinkable load
1203 // whose address is not necessarily loop-invariant.
1204 if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
1205 ORE->emit([&]() {
1206 return OptimizationRemarkMissed(
1207 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressInvalidated", LI)
1208 << "failed to move load with loop-invariant address "
1209 "because the loop may invalidate its value";
1210 });
1211
1212 return !Invalidated;
1213 } else if (CallInst *CI = dyn_cast<CallInst>(&I)) {
1214 // Don't sink or hoist dbg info; it's legal, but not useful.
1215 if (isa<DbgInfoIntrinsic>(I))
1216 return false;
1217
1218 // Don't sink calls which can throw.
1219 if (CI->mayThrow())
1220 return false;
1221
1222 // Convergent attribute has been used on operations that involve
1223 // inter-thread communication which results are implicitly affected by the
1224 // enclosing control flows. It is not safe to hoist or sink such operations
1225 // across control flow.
1226 if (CI->isConvergent())
1227 return false;
1228
1229 using namespace PatternMatch;
1230 if (match(CI, m_Intrinsic<Intrinsic::assume>()))
1231 // Assumes don't actually alias anything or throw
1232 return true;
1233
1234 if (match(CI, m_Intrinsic<Intrinsic::experimental_widenable_condition>()))
1235 // Widenable conditions don't actually alias anything or throw
1236 return true;
1237
1238 // Handle simple cases by querying alias analysis.
1239 FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
1240 if (Behavior == FMRB_DoesNotAccessMemory)
1241 return true;
1242 if (AAResults::onlyReadsMemory(Behavior)) {
1243 // A readonly argmemonly function only reads from memory pointed to by
1244 // it's arguments with arbitrary offsets. If we can prove there are no
1245 // writes to this memory in the loop, we can hoist or sink.
1246 if (AAResults::onlyAccessesArgPointees(Behavior)) {
1247 // TODO: expand to writeable arguments
1248 for (Value *Op : CI->arg_operands())
1249 if (Op->getType()->isPointerTy()) {
1250 bool Invalidated;
1251 if (CurAST)
1252 Invalidated = pointerInvalidatedByLoop(
1253 MemoryLocation::getBeforeOrAfter(Op), CurAST, CurLoop, AA);
1254 else
1255 Invalidated = pointerInvalidatedByLoopWithMSSA(
1256 MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I,
1257 *Flags);
1258 if (Invalidated)
1259 return false;
1260 }
1261 return true;
1262 }
1263
1264 // If this call only reads from memory and there are no writes to memory
1265 // in the loop, we can hoist or sink the call as appropriate.
1266 if (isReadOnly(CurAST, MSSAU, CurLoop))
1267 return true;
1268 }
1269
1270 // FIXME: This should use mod/ref information to see if we can hoist or
1271 // sink the call.
1272
1273 return false;
1274 } else if (auto *FI = dyn_cast<FenceInst>(&I)) {
1275 // Fences alias (most) everything to provide ordering. For the moment,
1276 // just give up if there are any other memory operations in the loop.
1277 if (CurAST) {
1278 auto Begin = CurAST->begin();
1279 assert(Begin != CurAST->end() && "must contain FI")(static_cast <bool> (Begin != CurAST->end() &&
"must contain FI") ? void (0) : __assert_fail ("Begin != CurAST->end() && \"must contain FI\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1279, __extension__ __PRETTY_FUNCTION__))
;
1280 if (std::next(Begin) != CurAST->end())
1281 // constant memory for instance, TODO: handle better
1282 return false;
1283 auto *UniqueI = Begin->getUniqueInstruction();
1284 if (!UniqueI)
1285 // other memory op, give up
1286 return false;
1287 (void)FI; // suppress unused variable warning
1288 assert(UniqueI == FI && "AS must contain FI")(static_cast <bool> (UniqueI == FI && "AS must contain FI"
) ? void (0) : __assert_fail ("UniqueI == FI && \"AS must contain FI\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1288, __extension__ __PRETTY_FUNCTION__))
;
1289 return true;
1290 } else // MSSAU
1291 return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
1292 } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
1293 if (!SI->isUnordered())
1294 return false; // Don't sink/hoist volatile or ordered atomic store!
1295
1296 // We can only hoist a store that we can prove writes a value which is not
1297 // read or overwritten within the loop. For those cases, we fallback to
1298 // load store promotion instead. TODO: We can extend this to cases where
1299 // there is exactly one write to the location and that write dominates an
1300 // arbitrary number of reads in the loop.
1301 if (CurAST) {
1302 auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
1303
1304 if (AS.isRef() || !AS.isMustAlias())
1305 // Quick exit test, handled by the full path below as well.
1306 return false;
1307 auto *UniqueI = AS.getUniqueInstruction();
1308 if (!UniqueI)
1309 // other memory op, give up
1310 return false;
1311 assert(UniqueI == SI && "AS must contain SI")(static_cast <bool> (UniqueI == SI && "AS must contain SI"
) ? void (0) : __assert_fail ("UniqueI == SI && \"AS must contain SI\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1311, __extension__ __PRETTY_FUNCTION__))
;
1312 return true;
1313 } else { // MSSAU
1314 if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
1315 return true;
1316 // If there are more accesses than the Promotion cap or no "quota" to
1317 // check clobber, then give up as we're not walking a list that long.
1318 if (Flags->tooManyMemoryAccesses() || Flags->tooManyClobberingCalls())
1319 return false;
1320 // If there are interfering Uses (i.e. their defining access is in the
1321 // loop), or ordered loads (stored as Defs!), don't move this store.
1322 // Could do better here, but this is conservatively correct.
1323 // TODO: Cache set of Uses on the first walk in runOnLoop, update when
1324 // moving accesses. Can also extend to dominating uses.
1325 auto *SIMD = MSSA->getMemoryAccess(SI);
1326 for (auto *BB : CurLoop->getBlocks())
1327 if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
1328 for (const auto &MA : *Accesses)
1329 if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
1330 auto *MD = MU->getDefiningAccess();
1331 if (!MSSA->isLiveOnEntryDef(MD) &&
1332 CurLoop->contains(MD->getBlock()))
1333 return false;
1334 // Disable hoisting past potentially interfering loads. Optimized
1335 // Uses may point to an access outside the loop, as getClobbering
1336 // checks the previous iteration when walking the backedge.
1337 // FIXME: More precise: no Uses that alias SI.
1338 if (!Flags->getIsSink() && !MSSA->dominates(SIMD, MU))
1339 return false;
1340 } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
1341 if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
1342 (void)LI; // Silence warning.
1343 assert(!LI->isUnordered() && "Expected unordered load")(static_cast <bool> (!LI->isUnordered() && "Expected unordered load"
) ? void (0) : __assert_fail ("!LI->isUnordered() && \"Expected unordered load\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1343, __extension__ __PRETTY_FUNCTION__))
;
1344 return false;
1345 }
1346 // Any call, while it may not be clobbering SI, it may be a use.
1347 if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
1348 // Check if the call may read from the memory location written
1349 // to by SI. Check CI's attributes and arguments; the number of
1350 // such checks performed is limited above by NoOfMemAccTooLarge.
1351 ModRefInfo MRI = AA->getModRefInfo(CI, MemoryLocation::get(SI));
1352 if (isModOrRefSet(MRI))
1353 return false;
1354 }
1355 }
1356 }
1357 auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
1358 Flags->incrementClobberingCalls();
1359 // If there are no clobbering Defs in the loop, store is safe to hoist.
1360 return MSSA->isLiveOnEntryDef(Source) ||
1361 !CurLoop->contains(Source->getBlock());
1362 }
1363 }
1364
1365 assert(!I.mayReadOrWriteMemory() && "unhandled aliasing")(static_cast <bool> (!I.mayReadOrWriteMemory() &&
"unhandled aliasing") ? void (0) : __assert_fail ("!I.mayReadOrWriteMemory() && \"unhandled aliasing\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1365, __extension__ __PRETTY_FUNCTION__))
;
1366
1367 // We've established mechanical ability and aliasing, it's up to the caller
1368 // to check fault safety
1369 return true;
1370}
1371
1372/// Returns true if a PHINode is a trivially replaceable with an
1373/// Instruction.
1374/// This is true when all incoming values are that instruction.
1375/// This pattern occurs most often with LCSSA PHI nodes.
1376///
1377static bool isTriviallyReplaceablePHI(const PHINode &PN, const Instruction &I) {
1378 for (const Value *IncValue : PN.incoming_values())
1379 if (IncValue != &I)
1380 return false;
1381
1382 return true;
1383}
1384
1385/// Return true if the instruction is free in the loop.
1386static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
1387 const TargetTransformInfo *TTI) {
1388
1389 if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I)) {
1390 if (TTI->getUserCost(GEP, TargetTransformInfo::TCK_SizeAndLatency) !=
1391 TargetTransformInfo::TCC_Free)
1392 return false;
1393 // For a GEP, we cannot simply use getUserCost because currently it
1394 // optimistically assume that a GEP will fold into addressing mode
1395 // regardless of its users.
1396 const BasicBlock *BB = GEP->getParent();
1397 for (const User *U : GEP->users()) {
1398 const Instruction *UI = cast<Instruction>(U);
1399 if (CurLoop->contains(UI) &&
1400 (BB != UI->getParent() ||
1401 (!isa<StoreInst>(UI) && !isa<LoadInst>(UI))))
1402 return false;
1403 }
1404 return true;
1405 } else
1406 return TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) ==
1407 TargetTransformInfo::TCC_Free;
1408}
1409
1410/// Return true if the only users of this instruction are outside of
1411/// the loop. If this is true, we can sink the instruction to the exit
1412/// blocks of the loop.
1413///
1414/// We also return true if the instruction could be folded away in lowering.
1415/// (e.g., a GEP can be folded into a load as an addressing mode in the loop).
1416static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
1417 const LoopSafetyInfo *SafetyInfo,
1418 TargetTransformInfo *TTI, bool &FreeInLoop) {
1419 const auto &BlockColors = SafetyInfo->getBlockColors();
1420 bool IsFree = isFreeInLoop(I, CurLoop, TTI);
1421 for (const User *U : I.users()) {
1422 const Instruction *UI = cast<Instruction>(U);
1423 if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
1424 const BasicBlock *BB = PN->getParent();
1425 // We cannot sink uses in catchswitches.
1426 if (isa<CatchSwitchInst>(BB->getTerminator()))
1427 return false;
1428
1429 // We need to sink a callsite to a unique funclet. Avoid sinking if the
1430 // phi use is too muddled.
1431 if (isa<CallInst>(I))
1432 if (!BlockColors.empty() &&
1433 BlockColors.find(const_cast<BasicBlock *>(BB))->second.size() != 1)
1434 return false;
1435 }
1436
1437 if (CurLoop->contains(UI)) {
1438 if (IsFree) {
1439 FreeInLoop = true;
1440 continue;
1441 }
1442 return false;
1443 }
1444 }
1445 return true;
1446}
1447
1448static Instruction *cloneInstructionInExitBlock(
1449 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
1450 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
1451 Instruction *New;
1452 if (auto *CI = dyn_cast<CallInst>(&I)) {
1453 const auto &BlockColors = SafetyInfo->getBlockColors();
1454
1455 // Sinking call-sites need to be handled differently from other
1456 // instructions. The cloned call-site needs a funclet bundle operand
1457 // appropriate for its location in the CFG.
1458 SmallVector<OperandBundleDef, 1> OpBundles;
1459 for (unsigned BundleIdx = 0, BundleEnd = CI->getNumOperandBundles();
1460 BundleIdx != BundleEnd; ++BundleIdx) {
1461 OperandBundleUse Bundle = CI->getOperandBundleAt(BundleIdx);
1462 if (Bundle.getTagID() == LLVMContext::OB_funclet)
1463 continue;
1464
1465 OpBundles.emplace_back(Bundle);
1466 }
1467
1468 if (!BlockColors.empty()) {
1469 const ColorVector &CV = BlockColors.find(&ExitBlock)->second;
1470 assert(CV.size() == 1 && "non-unique color for exit block!")(static_cast <bool> (CV.size() == 1 && "non-unique color for exit block!"
) ? void (0) : __assert_fail ("CV.size() == 1 && \"non-unique color for exit block!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1470, __extension__ __PRETTY_FUNCTION__))
;
1471 BasicBlock *BBColor = CV.front();
1472 Instruction *EHPad = BBColor->getFirstNonPHI();
1473 if (EHPad->isEHPad())
1474 OpBundles.emplace_back("funclet", EHPad);
1475 }
1476
1477 New = CallInst::Create(CI, OpBundles);
1478 } else {
1479 New = I.clone();
1480 }
1481
1482 ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
1483 if (!I.getName().empty())
1484 New->setName(I.getName() + ".le");
1485
1486 if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
1487 // Create a new MemoryAccess and let MemorySSA set its defining access.
1488 MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
1489 New, nullptr, New->getParent(), MemorySSA::Beginning);
1490 if (NewMemAcc) {
1491 if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
1492 MSSAU->insertDef(MemDef, /*RenameUses=*/true);
1493 else {
1494 auto *MemUse = cast<MemoryUse>(NewMemAcc);
1495 MSSAU->insertUse(MemUse, /*RenameUses=*/true);
1496 }
1497 }
1498 }
1499
1500 // Build LCSSA PHI nodes for any in-loop operands (if legal). Note that
1501 // this is particularly cheap because we can rip off the PHI node that we're
1502 // replacing for the number and blocks of the predecessors.
1503 // OPT: If this shows up in a profile, we can instead finish sinking all
1504 // invariant instructions, and then walk their operands to re-establish
1505 // LCSSA. That will eliminate creating PHI nodes just to nuke them when
1506 // sinking bottom-up.
1507 for (Use &Op : New->operands())
1508 if (LI->wouldBeOutOfLoopUseRequiringLCSSA(Op.get(), PN.getParent())) {
1509 auto *OInst = cast<Instruction>(Op.get());
1510 PHINode *OpPN =
1511 PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
1512 OInst->getName() + ".lcssa", &ExitBlock.front());
1513 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1514 OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
1515 Op = OpPN;
1516 }
1517 return New;
1518}
1519
1520static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
1521 AliasSetTracker *AST, MemorySSAUpdater *MSSAU) {
1522 if (AST)
1523 AST->deleteValue(&I);
1524 if (MSSAU)
1525 MSSAU->removeMemoryAccess(&I);
1526 SafetyInfo.removeInstruction(&I);
1527 I.eraseFromParent();
1528}
1529
1530static void moveInstructionBefore(Instruction &I, Instruction &Dest,
1531 ICFLoopSafetyInfo &SafetyInfo,
1532 MemorySSAUpdater *MSSAU,
1533 ScalarEvolution *SE) {
1534 SafetyInfo.removeInstruction(&I);
1535 SafetyInfo.insertInstructionTo(&I, Dest.getParent());
1536 I.moveBefore(&Dest);
1537 if (MSSAU)
1538 if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
1539 MSSAU->getMemorySSA()->getMemoryAccess(&I)))
1540 MSSAU->moveToPlace(OldMemAcc, Dest.getParent(),
1541 MemorySSA::BeforeTerminator);
1542 if (SE)
1543 SE->forgetValue(&I);
1544}
1545
1546static Instruction *sinkThroughTriviallyReplaceablePHI(
1547 PHINode *TPN, Instruction *I, LoopInfo *LI,
1548 SmallDenseMap<BasicBlock *, Instruction *, 32> &SunkCopies,
1549 const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
1550 MemorySSAUpdater *MSSAU) {
1551 assert(isTriviallyReplaceablePHI(*TPN, *I) &&(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1552, __extension__ __PRETTY_FUNCTION__))
1552 "Expect only trivially replaceable PHI")(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1552, __extension__ __PRETTY_FUNCTION__))
;
1553 BasicBlock *ExitBlock = TPN->getParent();
1554 Instruction *New;
1555 auto It = SunkCopies.find(ExitBlock);
1556 if (It != SunkCopies.end())
1557 New = It->second;
1558 else
1559 New = SunkCopies[ExitBlock] = cloneInstructionInExitBlock(
1560 *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
1561 return New;
1562}
1563
1564static bool canSplitPredecessors(PHINode *PN, LoopSafetyInfo *SafetyInfo) {
1565 BasicBlock *BB = PN->getParent();
1566 if (!BB->canSplitPredecessors())
1567 return false;
1568 // It's not impossible to split EHPad blocks, but if BlockColors already exist
1569 // it require updating BlockColors for all offspring blocks accordingly. By
1570 // skipping such corner case, we can make updating BlockColors after splitting
1571 // predecessor fairly simple.
1572 if (!SafetyInfo->getBlockColors().empty() && BB->getFirstNonPHI()->isEHPad())
1573 return false;
1574 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
1575 BasicBlock *BBPred = *PI;
1576 if (isa<IndirectBrInst>(BBPred->getTerminator()) ||
1577 isa<CallBrInst>(BBPred->getTerminator()))
1578 return false;
1579 }
1580 return true;
1581}
1582
1583static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
1584 LoopInfo *LI, const Loop *CurLoop,
1585 LoopSafetyInfo *SafetyInfo,
1586 MemorySSAUpdater *MSSAU) {
1587#ifndef NDEBUG
1588 SmallVector<BasicBlock *, 32> ExitBlocks;
1589 CurLoop->getUniqueExitBlocks(ExitBlocks);
1590 SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
1591 ExitBlocks.end());
1592#endif
1593 BasicBlock *ExitBB = PN->getParent();
1594 assert(ExitBlockSet.count(ExitBB) && "Expect the PHI is in an exit block.")(static_cast <bool> (ExitBlockSet.count(ExitBB) &&
"Expect the PHI is in an exit block.") ? void (0) : __assert_fail
("ExitBlockSet.count(ExitBB) && \"Expect the PHI is in an exit block.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1594, __extension__ __PRETTY_FUNCTION__))
;
1595
1596 // Split predecessors of the loop exit to make instructions in the loop are
1597 // exposed to exit blocks through trivially replaceable PHIs while keeping the
1598 // loop in the canonical form where each predecessor of each exit block should
1599 // be contained within the loop. For example, this will convert the loop below
1600 // from
1601 //
1602 // LB1:
1603 // %v1 =
1604 // br %LE, %LB2
1605 // LB2:
1606 // %v2 =
1607 // br %LE, %LB1
1608 // LE:
1609 // %p = phi [%v1, %LB1], [%v2, %LB2] <-- non-trivially replaceable
1610 //
1611 // to
1612 //
1613 // LB1:
1614 // %v1 =
1615 // br %LE.split, %LB2
1616 // LB2:
1617 // %v2 =
1618 // br %LE.split2, %LB1
1619 // LE.split:
1620 // %p1 = phi [%v1, %LB1] <-- trivially replaceable
1621 // br %LE
1622 // LE.split2:
1623 // %p2 = phi [%v2, %LB2] <-- trivially replaceable
1624 // br %LE
1625 // LE:
1626 // %p = phi [%p1, %LE.split], [%p2, %LE.split2]
1627 //
1628 const auto &BlockColors = SafetyInfo->getBlockColors();
1629 SmallSetVector<BasicBlock *, 8> PredBBs(pred_begin(ExitBB), pred_end(ExitBB));
1630 while (!PredBBs.empty()) {
1631 BasicBlock *PredBB = *PredBBs.begin();
1632 assert(CurLoop->contains(PredBB) &&(static_cast <bool> (CurLoop->contains(PredBB) &&
"Expect all predecessors are in the loop") ? void (0) : __assert_fail
("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1633, __extension__ __PRETTY_FUNCTION__))
1633 "Expect all predecessors are in the loop")(static_cast <bool> (CurLoop->contains(PredBB) &&
"Expect all predecessors are in the loop") ? void (0) : __assert_fail
("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1633, __extension__ __PRETTY_FUNCTION__))
;
1634 if (PN->getBasicBlockIndex(PredBB) >= 0) {
1635 BasicBlock *NewPred = SplitBlockPredecessors(
1636 ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
1637 // Since we do not allow splitting EH-block with BlockColors in
1638 // canSplitPredecessors(), we can simply assign predecessor's color to
1639 // the new block.
1640 if (!BlockColors.empty())
1641 // Grab a reference to the ColorVector to be inserted before getting the
1642 // reference to the vector we are copying because inserting the new
1643 // element in BlockColors might cause the map to be reallocated.
1644 SafetyInfo->copyColors(NewPred, PredBB);
1645 }
1646 PredBBs.remove(PredBB);
1647 }
1648}
1649
1650/// When an instruction is found to only be used outside of the loop, this
1651/// function moves it to the exit blocks and patches up SSA form as needed.
1652/// This method is guaranteed to remove the original instruction from its
1653/// position, and may either delete it or move it to outside of the loop.
1654///
1655static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
1656 BlockFrequencyInfo *BFI, const Loop *CurLoop,
1657 ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
1658 OptimizationRemarkEmitter *ORE) {
1659 bool Changed = false;
1660 LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM sinking instruction: " <<
I << "\n"; } } while (false)
;
1661
1662 // Iterate over users to be ready for actual sinking. Replace users via
1663 // unreachable blocks with undef and make all user PHIs trivially replaceable.
1664 SmallPtrSet<Instruction *, 8> VisitedUsers;
1665 for (Value::user_iterator UI = I.user_begin(), UE = I.user_end(); UI != UE;) {
1666 auto *User = cast<Instruction>(*UI);
1667 Use &U = UI.getUse();
1668 ++UI;
1669
1670 if (VisitedUsers.count(User) || CurLoop->contains(User))
1671 continue;
1672
1673 if (!DT->isReachableFromEntry(User->getParent())) {
1674 U = UndefValue::get(I.getType());
1675 Changed = true;
1676 continue;
1677 }
1678
1679 // The user must be a PHI node.
1680 PHINode *PN = cast<PHINode>(User);
1681
1682 // Surprisingly, instructions can be used outside of loops without any
1683 // exits. This can only happen in PHI nodes if the incoming block is
1684 // unreachable.
1685 BasicBlock *BB = PN->getIncomingBlock(U);
1686 if (!DT->isReachableFromEntry(BB)) {
1687 U = UndefValue::get(I.getType());
1688 Changed = true;
1689 continue;
1690 }
1691
1692 VisitedUsers.insert(PN);
1693 if (isTriviallyReplaceablePHI(*PN, I))
1694 continue;
1695
1696 if (!canSplitPredecessors(PN, SafetyInfo))
1697 return Changed;
1698
1699 // Split predecessors of the PHI so that we can make users trivially
1700 // replaceable.
1701 splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);
1702
1703 // Should rebuild the iterators, as they may be invalidated by
1704 // splitPredecessorsOfLoopExit().
1705 UI = I.user_begin();
1706 UE = I.user_end();
1707 }
1708
1709 if (VisitedUsers.empty())
1710 return Changed;
1711
1712 ORE->emit([&]() {
1713 return OptimizationRemark(DEBUG_TYPE"licm", "InstSunk", &I)
1714 << "sinking " << ore::NV("Inst", &I);
1715 });
1716 if (isa<LoadInst>(I))
1717 ++NumMovedLoads;
1718 else if (isa<CallInst>(I))
1719 ++NumMovedCalls;
1720 ++NumSunk;
1721
1722#ifndef NDEBUG
1723 SmallVector<BasicBlock *, 32> ExitBlocks;
1724 CurLoop->getUniqueExitBlocks(ExitBlocks);
1725 SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
1726 ExitBlocks.end());
1727#endif
1728
1729 // Clones of this instruction. Don't create more than one per exit block!
1730 SmallDenseMap<BasicBlock *, Instruction *, 32> SunkCopies;
1731
1732 // If this instruction is only used outside of the loop, then all users are
1733 // PHI nodes in exit blocks due to LCSSA form. Just RAUW them with clones of
1734 // the instruction.
1735 // First check if I is worth sinking for all uses. Sink only when it is worth
1736 // across all uses.
1737 SmallSetVector<User*, 8> Users(I.user_begin(), I.user_end());
1738 SmallVector<PHINode *, 8> ExitPNs;
1739 for (auto *UI : Users) {
1740 auto *User = cast<Instruction>(UI);
1741
1742 if (CurLoop->contains(User))
1743 continue;
1744
1745 PHINode *PN = cast<PHINode>(User);
1746 assert(ExitBlockSet.count(PN->getParent()) &&(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
(0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1747, __extension__ __PRETTY_FUNCTION__))
1747 "The LCSSA PHI is not in an exit block!")(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
(0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1747, __extension__ __PRETTY_FUNCTION__))
;
1748 if (!worthSinkOrHoistInst(I, PN->getParent(), ORE, BFI)) {
1749 return Changed;
1750 }
1751
1752 ExitPNs.push_back(PN);
1753 }
1754
1755 for (auto *PN : ExitPNs) {
1756
1757 // The PHI must be trivially replaceable.
1758 Instruction *New = sinkThroughTriviallyReplaceablePHI(
1759 PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
1760 PN->replaceAllUsesWith(New);
1761 eraseInstruction(*PN, *SafetyInfo, nullptr, nullptr);
1762 Changed = true;
1763 }
1764 return Changed;
1765}
1766
1767/// When an instruction is found to only use loop invariant operands that
1768/// is safe to hoist, this instruction is called to do the dirty work.
1769///
1770static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
1771 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
1772 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
1773 OptimizationRemarkEmitter *ORE) {
1774 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getNameOrAsOperand() << ": "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
1775 << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
;
1776 ORE->emit([&]() {
1777 return OptimizationRemark(DEBUG_TYPE"licm", "Hoisted", &I) << "hoisting "
1778 << ore::NV("Inst", &I);
1779 });
1780
1781 // Metadata can be dependent on conditions we are hoisting above.
1782 // Conservatively strip all metadata on the instruction unless we were
1783 // guaranteed to execute I if we entered the loop, in which case the metadata
1784 // is valid in the loop preheader.
1785 if (I.hasMetadataOtherThanDebugLoc() &&
1786 // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
1787 // time in isGuaranteedToExecute if we don't actually have anything to
1788 // drop. It is a compile time optimization, not required for correctness.
1789 !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
1790 I.dropUnknownNonDebugMetadata();
1791
1792 if (isa<PHINode>(I))
1793 // Move the new node to the end of the phi list in the destination block.
1794 moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
1795 else
1796 // Move the new node to the destination block, before its terminator.
1797 moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);
1798
1799 I.updateLocationAfterHoist();
1800
1801 if (isa<LoadInst>(I))
1802 ++NumMovedLoads;
1803 else if (isa<CallInst>(I))
1804 ++NumMovedCalls;
1805 ++NumHoisted;
1806}
1807
1808/// Only sink or hoist an instruction if it is not a trapping instruction,
1809/// or if the instruction is known not to trap when moved to the preheader.
1810/// or if it is a trapping instruction and is guaranteed to execute.
1811static bool isSafeToExecuteUnconditionally(Instruction &Inst,
1812 const DominatorTree *DT,
1813 const TargetLibraryInfo *TLI,
1814 const Loop *CurLoop,
1815 const LoopSafetyInfo *SafetyInfo,
1816 OptimizationRemarkEmitter *ORE,
1817 const Instruction *CtxI) {
1818 if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
1819 return true;
1820
1821 bool GuaranteedToExecute =
1822 SafetyInfo->isGuaranteedToExecute(Inst, DT, CurLoop);
1823
1824 if (!GuaranteedToExecute) {
1825 auto *LI = dyn_cast<LoadInst>(&Inst);
1826 if (LI && CurLoop->isLoopInvariant(LI->getPointerOperand()))
1827 ORE->emit([&]() {
1828 return OptimizationRemarkMissed(
1829 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressCondExecuted", LI)
1830 << "failed to hoist load with loop-invariant address "
1831 "because load is conditionally executed";
1832 });
1833 }
1834
1835 return GuaranteedToExecute;
1836}
1837
1838namespace {
1839class LoopPromoter : public LoadAndStorePromoter {
1840 Value *SomePtr; // Designated pointer to store to.
1841 const SmallSetVector<Value *, 8> &PointerMustAliases;
1842 SmallVectorImpl<BasicBlock *> &LoopExitBlocks;
1843 SmallVectorImpl<Instruction *> &LoopInsertPts;
1844 SmallVectorImpl<MemoryAccess *> &MSSAInsertPts;
1845 PredIteratorCache &PredCache;
1846 AliasSetTracker *AST;
1847 MemorySSAUpdater *MSSAU;
1848 LoopInfo &LI;
1849 DebugLoc DL;
1850 int Alignment;
1851 bool UnorderedAtomic;
1852 AAMDNodes AATags;
1853 ICFLoopSafetyInfo &SafetyInfo;
1854
1855 // We're about to add a use of V in a loop exit block. Insert an LCSSA phi
1856 // (if legal) if doing so would add an out-of-loop use to an instruction
1857 // defined in-loop.
1858 Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
1859 if (!LI.wouldBeOutOfLoopUseRequiringLCSSA(V, BB))
1860 return V;
1861
1862 Instruction *I = cast<Instruction>(V);
1863 // We need to create an LCSSA PHI node for the incoming value and
1864 // store that.
1865 PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
1866 I->getName() + ".lcssa", &BB->front());
1867 for (BasicBlock *Pred : PredCache.get(BB))
1868 PN->addIncoming(I, Pred);
1869 return PN;
1870 }
1871
1872public:
1873 LoopPromoter(Value *SP, ArrayRef<const Instruction *> Insts, SSAUpdater &S,
1874 const SmallSetVector<Value *, 8> &PMA,
1875 SmallVectorImpl<BasicBlock *> &LEB,
1876 SmallVectorImpl<Instruction *> &LIP,
1877 SmallVectorImpl<MemoryAccess *> &MSSAIP, PredIteratorCache &PIC,
1878 AliasSetTracker *ast, MemorySSAUpdater *MSSAU, LoopInfo &li,
1879 DebugLoc dl, int alignment, bool UnorderedAtomic,
1880 const AAMDNodes &AATags, ICFLoopSafetyInfo &SafetyInfo)
1881 : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
1882 LoopExitBlocks(LEB), LoopInsertPts(LIP), MSSAInsertPts(MSSAIP),
1883 PredCache(PIC), AST(ast), MSSAU(MSSAU), LI(li), DL(std::move(dl)),
1884 Alignment(alignment), UnorderedAtomic(UnorderedAtomic), AATags(AATags),
1885 SafetyInfo(SafetyInfo) {}
1886
1887 bool isInstInList(Instruction *I,
1888 const SmallVectorImpl<Instruction *> &) const override {
1889 Value *Ptr;
1890 if (LoadInst *LI = dyn_cast<LoadInst>(I))
1891 Ptr = LI->getOperand(0);
1892 else
1893 Ptr = cast<StoreInst>(I)->getPointerOperand();
1894 return PointerMustAliases.count(Ptr);
1895 }
1896
1897 void doExtraRewritesBeforeFinalDeletion() override {
1898 // Insert stores after in the loop exit blocks. Each exit block gets a
1899 // store of the live-out values that feed them. Since we've already told
1900 // the SSA updater about the defs in the loop and the preheader
1901 // definition, it is all set and we can start using it.
1902 for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) {
1903 BasicBlock *ExitBlock = LoopExitBlocks[i];
1904 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
1905 LiveInValue = maybeInsertLCSSAPHI(LiveInValue, ExitBlock);
1906 Value *Ptr = maybeInsertLCSSAPHI(SomePtr, ExitBlock);
1907 Instruction *InsertPos = LoopInsertPts[i];
1908 StoreInst *NewSI = new StoreInst(LiveInValue, Ptr, InsertPos);
1909 if (UnorderedAtomic)
1910 NewSI->setOrdering(AtomicOrdering::Unordered);
1911 NewSI->setAlignment(Align(Alignment));
1912 NewSI->setDebugLoc(DL);
1913 if (AATags)
1914 NewSI->setAAMetadata(AATags);
1915
1916 if (MSSAU) {
1917 MemoryAccess *MSSAInsertPoint = MSSAInsertPts[i];
1918 MemoryAccess *NewMemAcc;
1919 if (!MSSAInsertPoint) {
1920 NewMemAcc = MSSAU->createMemoryAccessInBB(
1921 NewSI, nullptr, NewSI->getParent(), MemorySSA::Beginning);
1922 } else {
1923 NewMemAcc =
1924 MSSAU->createMemoryAccessAfter(NewSI, nullptr, MSSAInsertPoint);
1925 }
1926 MSSAInsertPts[i] = NewMemAcc;
1927 MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
1928 // FIXME: true for safety, false may still be correct.
1929 }
1930 }
1931 }
1932
1933 void replaceLoadWithValue(LoadInst *LI, Value *V) const override {
1934 // Update alias analysis.
1935 if (AST)
1936 AST->copyValue(LI, V);
1937 }
1938 void instructionDeleted(Instruction *I) const override {
1939 SafetyInfo.removeInstruction(I);
1940 if (AST)
1941 AST->deleteValue(I);
1942 if (MSSAU)
1943 MSSAU->removeMemoryAccess(I);
1944 }
1945};
1946
1947bool isNotCapturedBeforeOrInLoop(const Value *V, const Loop *L,
1948 DominatorTree *DT) {
1949 // We can perform the captured-before check against any instruction in the
1950 // loop header, as the loop header is reachable from any instruction inside
1951 // the loop.
1952 // TODO: ReturnCaptures=true shouldn't be necessary here.
1953 return !PointerMayBeCapturedBefore(V, /* ReturnCaptures */ true,
1954 /* StoreCaptures */ true,
1955 L->getHeader()->getTerminator(), DT);
1956}
1957
1958/// Return true iff we can prove that a caller of this function can not inspect
1959/// the contents of the provided object in a well defined program.
1960bool isKnownNonEscaping(Value *Object, const Loop *L,
1961 const TargetLibraryInfo *TLI, DominatorTree *DT) {
1962 if (isa<AllocaInst>(Object))
1963 // Since the alloca goes out of scope, we know the caller can't retain a
1964 // reference to it and be well defined. Thus, we don't need to check for
1965 // capture.
1966 return true;
1967
1968 // For all other objects we need to know that the caller can't possibly
1969 // have gotten a reference to the object. There are two components of
1970 // that:
1971 // 1) Object can't be escaped by this function. This is what
1972 // PointerMayBeCaptured checks.
1973 // 2) Object can't have been captured at definition site. For this, we
1974 // need to know the return value is noalias. At the moment, we use a
1975 // weaker condition and handle only AllocLikeFunctions (which are
1976 // known to be noalias). TODO
1977 return isAllocLikeFn(Object, TLI) &&
1978 isNotCapturedBeforeOrInLoop(Object, L, DT);
1979}
1980
1981} // namespace
1982
1983/// Try to promote memory values to scalars by sinking stores out of the
1984/// loop and moving loads to before the loop. We do this by looping over
1985/// the stores in the loop, looking for stores to Must pointers which are
1986/// loop invariant.
1987///
1988bool llvm::promoteLoopAccessesToScalars(
1989 const SmallSetVector<Value *, 8> &PointerMustAliases,
1990 SmallVectorImpl<BasicBlock *> &ExitBlocks,
1991 SmallVectorImpl<Instruction *> &InsertPts,
1992 SmallVectorImpl<MemoryAccess *> &MSSAInsertPts, PredIteratorCache &PIC,
1993 LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
1994 Loop *CurLoop, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
1995 ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
1996 // Verify inputs.
1997 assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1999, __extension__ __PRETTY_FUNCTION__))
1998 SafetyInfo != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1999, __extension__ __PRETTY_FUNCTION__))
1999 "Unexpected Input to promoteLoopAccessesToScalars")(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1999, __extension__ __PRETTY_FUNCTION__))
;
2000
2001 Value *SomePtr = *PointerMustAliases.begin();
2002 BasicBlock *Preheader = CurLoop->getLoopPreheader();
2003
2004 // It is not safe to promote a load/store from the loop if the load/store is
2005 // conditional. For example, turning:
2006 //
2007 // for () { if (c) *P += 1; }
2008 //
2009 // into:
2010 //
2011 // tmp = *P; for () { if (c) tmp +=1; } *P = tmp;
2012 //
2013 // is not safe, because *P may only be valid to access if 'c' is true.
2014 //
2015 // The safety property divides into two parts:
2016 // p1) The memory may not be dereferenceable on entry to the loop. In this
2017 // case, we can't insert the required load in the preheader.
2018 // p2) The memory model does not allow us to insert a store along any dynamic
2019 // path which did not originally have one.
2020 //
2021 // If at least one store is guaranteed to execute, both properties are
2022 // satisfied, and promotion is legal.
2023 //
2024 // This, however, is not a necessary condition. Even if no store/load is
2025 // guaranteed to execute, we can still establish these properties.
2026 // We can establish (p1) by proving that hoisting the load into the preheader
2027 // is safe (i.e. proving dereferenceability on all paths through the loop). We
2028 // can use any access within the alias set to prove dereferenceability,
2029 // since they're all must alias.
2030 //
2031 // There are two ways establish (p2):
2032 // a) Prove the location is thread-local. In this case the memory model
2033 // requirement does not apply, and stores are safe to insert.
2034 // b) Prove a store dominates every exit block. In this case, if an exit
2035 // blocks is reached, the original dynamic path would have taken us through
2036 // the store, so inserting a store into the exit block is safe. Note that this
2037 // is different from the store being guaranteed to execute. For instance,
2038 // if an exception is thrown on the first iteration of the loop, the original
2039 // store is never executed, but the exit blocks are not executed either.
2040
2041 bool DereferenceableInPH = false;
2042 bool SafeToInsertStore = false;
2043
2044 SmallVector<Instruction *, 64> LoopUses;
2045
2046 // We start with an alignment of one and try to find instructions that allow
2047 // us to prove better alignment.
2048 Align Alignment;
2049 // Keep track of which types of access we see
2050 bool SawUnorderedAtomic = false;
2051 bool SawNotAtomic = false;
2052 AAMDNodes AATags;
2053
2054 const DataLayout &MDL = Preheader->getModule()->getDataLayout();
2055
2056 bool IsKnownThreadLocalObject = false;
2057 if (SafetyInfo->anyBlockMayThrow()) {
2058 // If a loop can throw, we have to insert a store along each unwind edge.
2059 // That said, we can't actually make the unwind edge explicit. Therefore,
2060 // we have to prove that the store is dead along the unwind edge. We do
2061 // this by proving that the caller can't have a reference to the object
2062 // after return and thus can't possibly load from the object.
2063 Value *Object = getUnderlyingObject(SomePtr);
2064 if (!isKnownNonEscaping(Object, CurLoop, TLI, DT))
2065 return false;
2066 // Subtlety: Alloca's aren't visible to callers, but *are* potentially
2067 // visible to other threads if captured and used during their lifetimes.
2068 IsKnownThreadLocalObject = !isa<AllocaInst>(Object);
2069 }
2070
2071 // Check that all of the pointers in the alias set have the same type. We
2072 // cannot (yet) promote a memory location that is loaded and stored in
2073 // different sizes. While we are at it, collect alignment and AA info.
2074 for (Value *ASIV : PointerMustAliases) {
2075 // Check that all of the pointers in the alias set have the same type. We
2076 // cannot (yet) promote a memory location that is loaded and stored in
2077 // different sizes.
2078 if (SomePtr->getType() != ASIV->getType())
2079 return false;
2080
2081 for (User *U : ASIV->users()) {
2082 // Ignore instructions that are outside the loop.
2083 Instruction *UI = dyn_cast<Instruction>(U);
2084 if (!UI || !CurLoop->contains(UI))
2085 continue;
2086
2087 // If there is an non-load/store instruction in the loop, we can't promote
2088 // it.
2089 if (LoadInst *Load = dyn_cast<LoadInst>(UI)) {
2090 if (!Load->isUnordered())
2091 return false;
2092
2093 SawUnorderedAtomic |= Load->isAtomic();
2094 SawNotAtomic |= !Load->isAtomic();
2095
2096 Align InstAlignment = Load->getAlign();
2097
2098 // Note that proving a load safe to speculate requires proving
2099 // sufficient alignment at the target location. Proving it guaranteed
2100 // to execute does as well. Thus we can increase our guaranteed
2101 // alignment as well.
2102 if (!DereferenceableInPH || (InstAlignment > Alignment))
2103 if (isSafeToExecuteUnconditionally(*Load, DT, TLI, CurLoop,
2104 SafetyInfo, ORE,
2105 Preheader->getTerminator())) {
2106 DereferenceableInPH = true;
2107 Alignment = std::max(Alignment, InstAlignment);
2108 }
2109 } else if (const StoreInst *Store = dyn_cast<StoreInst>(UI)) {
2110 // Stores *of* the pointer are not interesting, only stores *to* the
2111 // pointer.
2112 if (UI->getOperand(1) != ASIV)
2113 continue;
2114 if (!Store->isUnordered())
2115 return false;
2116
2117 SawUnorderedAtomic |= Store->isAtomic();
2118 SawNotAtomic |= !Store->isAtomic();
2119
2120 // If the store is guaranteed to execute, both properties are satisfied.
2121 // We may want to check if a store is guaranteed to execute even if we
2122 // already know that promotion is safe, since it may have higher
2123 // alignment than any other guaranteed stores, in which case we can
2124 // raise the alignment on the promoted store.
2125 Align InstAlignment = Store->getAlign();
2126
2127 if (!DereferenceableInPH || !SafeToInsertStore ||
2128 (InstAlignment > Alignment)) {
2129 if (SafetyInfo->isGuaranteedToExecute(*UI, DT, CurLoop)) {
2130 DereferenceableInPH = true;
2131 SafeToInsertStore = true;
2132 Alignment = std::max(Alignment, InstAlignment);
2133 }
2134 }
2135
2136 // If a store dominates all exit blocks, it is safe to sink.
2137 // As explained above, if an exit block was executed, a dominating
2138 // store must have been executed at least once, so we are not
2139 // introducing stores on paths that did not have them.
2140 // Note that this only looks at explicit exit blocks. If we ever
2141 // start sinking stores into unwind edges (see above), this will break.
2142 if (!SafeToInsertStore)
2143 SafeToInsertStore = llvm::all_of(ExitBlocks, [&](BasicBlock *Exit) {
2144 return DT->dominates(Store->getParent(), Exit);
2145 });
2146
2147 // If the store is not guaranteed to execute, we may still get
2148 // deref info through it.
2149 if (!DereferenceableInPH) {
2150 DereferenceableInPH = isDereferenceableAndAlignedPointer(
2151 Store->getPointerOperand(), Store->getValueOperand()->getType(),
2152 Store->getAlign(), MDL, Preheader->getTerminator(), DT, TLI);
2153 }
2154 } else
2155 return false; // Not a load or store.
2156
2157 // Merge the AA tags.
2158 if (LoopUses.empty()) {
2159 // On the first load/store, just take its AA tags.
2160 UI->getAAMetadata(AATags);
2161 } else if (AATags) {
2162 UI->getAAMetadata(AATags, /* Merge = */ true);
2163 }
2164
2165 LoopUses.push_back(UI);
2166 }
2167 }
2168
2169 // If we found both an unordered atomic instruction and a non-atomic memory
2170 // access, bail. We can't blindly promote non-atomic to atomic since we
2171 // might not be able to lower the result. We can't downgrade since that
2172 // would violate memory model. Also, align 0 is an error for atomics.
2173 if (SawUnorderedAtomic && SawNotAtomic)
2174 return false;
2175
2176 // If we're inserting an atomic load in the preheader, we must be able to
2177 // lower it. We're only guaranteed to be able to lower naturally aligned
2178 // atomics.
2179 auto *SomePtrElemType = SomePtr->getType()->getPointerElementType();
2180 if (SawUnorderedAtomic &&
2181 Alignment < MDL.getTypeStoreSize(SomePtrElemType))
2182 return false;
2183
2184 // If we couldn't prove we can hoist the load, bail.
2185 if (!DereferenceableInPH)
2186 return false;
2187
2188 // We know we can hoist the load, but don't have a guaranteed store.
2189 // Check whether the location is thread-local. If it is, then we can insert
2190 // stores along paths which originally didn't have them without violating the
2191 // memory model.
2192 if (!SafeToInsertStore) {
2193 if (IsKnownThreadLocalObject)
2194 SafeToInsertStore = true;
2195 else {
2196 Value *Object = getUnderlyingObject(SomePtr);
2197 SafeToInsertStore =
2198 (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
2199 isNotCapturedBeforeOrInLoop(Object, CurLoop, DT);
2200 }
2201 }
2202
2203 // If we've still failed to prove we can sink the store, give up.
2204 if (!SafeToInsertStore)
2205 return false;
2206
2207 // Otherwise, this is safe to promote, lets do it!
2208 LLVM_DEBUG(dbgs() << "LICM: Promoting value stored to in loop: " << *SomePtrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
<< *SomePtr << '\n'; } } while (false)
2209 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
<< *SomePtr << '\n'; } } while (false)
;
2210 ORE->emit([&]() {
2211 return OptimizationRemark(DEBUG_TYPE"licm", "PromoteLoopAccessesToScalar",
2212 LoopUses[0])
2213 << "Moving accesses to memory location out of the loop";
2214 });
2215 ++NumPromoted;
2216
2217 // Look at all the loop uses, and try to merge their locations.
2218 std::vector<const DILocation *> LoopUsesLocs;
2219 for (auto U : LoopUses)
2220 LoopUsesLocs.push_back(U->getDebugLoc().get());
2221 auto DL = DebugLoc(DILocation::getMergedLocations(LoopUsesLocs));
2222
2223 // We use the SSAUpdater interface to insert phi nodes as required.
2224 SmallVector<PHINode *, 16> NewPHIs;
2225 SSAUpdater SSA(&NewPHIs);
2226 LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
2227 InsertPts, MSSAInsertPts, PIC, CurAST, MSSAU, *LI, DL,
2228 Alignment.value(), SawUnorderedAtomic, AATags,
2229 *SafetyInfo);
2230
2231 // Set up the preheader to have a definition of the value. It is the live-out
2232 // value from the preheader that uses in the loop will use.
2233 LoadInst *PreheaderLoad = new LoadInst(
2234 SomePtr->getType()->getPointerElementType(), SomePtr,
2235 SomePtr->getName() + ".promoted", Preheader->getTerminator());
2236 if (SawUnorderedAtomic)
2237 PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
2238 PreheaderLoad->setAlignment(Alignment);
2239 PreheaderLoad->setDebugLoc(DebugLoc());
2240 if (AATags)
2241 PreheaderLoad->setAAMetadata(AATags);
2242 SSA.AddAvailableValue(Preheader, PreheaderLoad);
2243
2244 if (MSSAU) {
2245 MemoryAccess *PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
2246 PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
2247 MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
2248 MSSAU->insertUse(NewMemUse, /*RenameUses=*/true);
2249 }
2250
2251 if (MSSAU && VerifyMemorySSA)
2252 MSSAU->getMemorySSA()->verifyMemorySSA();
2253 // Rewrite all the loads in the loop and remember all the definitions from
2254 // stores in the loop.
2255 Promoter.run(LoopUses);
2256
2257 if (MSSAU && VerifyMemorySSA)
2258 MSSAU->getMemorySSA()->verifyMemorySSA();
2259 // If the SSAUpdater didn't use the load in the preheader, just zap it now.
2260 if (PreheaderLoad->use_empty())
2261 eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST, MSSAU);
2262
2263 return true;
2264}
2265
2266static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
2267 function_ref<void(Instruction *)> Fn) {
2268 for (const BasicBlock *BB : L->blocks())
2269 if (const auto *Accesses = MSSA->getBlockAccesses(BB))
2270 for (const auto &Access : *Accesses)
2271 if (const auto *MUD = dyn_cast<MemoryUseOrDef>(&Access))
2272 Fn(MUD->getMemoryInst());
2273}
2274
2275static SmallVector<SmallSetVector<Value *, 8>, 0>
2276collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
2277 AliasSetTracker AST(*AA);
2278
2279 auto IsPotentiallyPromotable = [L](const Instruction *I) {
2280 if (const auto *SI = dyn_cast<StoreInst>(I))
2281 return L->isLoopInvariant(SI->getPointerOperand());
2282 if (const auto *LI = dyn_cast<LoadInst>(I))
2283 return L->isLoopInvariant(LI->getPointerOperand());
2284 return false;
2285 };
2286
2287 // Populate AST with potentially promotable accesses and remove them from
2288 // MaybePromotable, so they will not be checked again on the next iteration.
2289 SmallPtrSet<Value *, 16> AttemptingPromotion;
2290 foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
2291 if (IsPotentiallyPromotable(I)) {
2292 AttemptingPromotion.insert(I);
2293 AST.add(I);
2294 }
2295 });
2296
2297 // We're only interested in must-alias sets that contain a mod.
2298 SmallVector<const AliasSet *, 8> Sets;
2299 for (AliasSet &AS : AST)
2300 if (!AS.isForwardingAliasSet() && AS.isMod() && AS.isMustAlias())
2301 Sets.push_back(&AS);
2302
2303 if (Sets.empty())
2304 return {}; // Nothing to promote...
2305
2306 // Discard any sets for which there is an aliasing non-promotable access.
2307 foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
2308 if (AttemptingPromotion.contains(I))
2309 return;
2310
2311 llvm::erase_if(Sets, [&](const AliasSet *AS) {
2312 return AS->aliasesUnknownInst(I, *AA);
2313 });
2314 });
2315
2316 SmallVector<SmallSetVector<Value *, 8>, 0> Result;
2317 for (const AliasSet *Set : Sets) {
2318 SmallSetVector<Value *, 8> PointerMustAliases;
2319 for (const auto &ASI : *Set)
2320 PointerMustAliases.insert(ASI.getValue());
2321 Result.push_back(std::move(PointerMustAliases));
2322 }
2323
2324 return Result;
2325}
2326
2327/// Returns an owning pointer to an alias set which incorporates aliasing info
2328/// from L and all subloops of L.
2329std::unique_ptr<AliasSetTracker>
2330LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
2331 AAResults *AA) {
2332 auto CurAST = std::make_unique<AliasSetTracker>(*AA);
2333
2334 // Add everything from all the sub loops.
2335 for (Loop *InnerL : L->getSubLoops())
2336 for (BasicBlock *BB : InnerL->blocks())
2337 CurAST->add(*BB);
2338
2339 // And merge in this loop (without anything from inner loops).
2340 for (BasicBlock *BB : L->blocks())
2341 if (LI->getLoopFor(BB) == L)
2342 CurAST->add(*BB);
2343
2344 return CurAST;
2345}
2346
2347static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
2348 AliasSetTracker *CurAST, Loop *CurLoop,
2349 AAResults *AA) {
2350 // First check to see if any of the basic blocks in CurLoop invalidate *V.
2351 bool isInvalidatedAccordingToAST = CurAST->getAliasSetFor(MemLoc).isMod();
2352
2353 if (!isInvalidatedAccordingToAST || !LICMN2Theshold)
2354 return isInvalidatedAccordingToAST;
2355
2356 // Check with a diagnostic analysis if we can refine the information above.
2357 // This is to identify the limitations of using the AST.
2358 // The alias set mechanism used by LICM has a major weakness in that it
2359 // combines all things which may alias into a single set *before* asking
2360 // modref questions. As a result, a single readonly call within a loop will
2361 // collapse all loads and stores into a single alias set and report
2362 // invalidation if the loop contains any store. For example, readonly calls
2363 // with deopt states have this form and create a general alias set with all
2364 // loads and stores. In order to get any LICM in loops containing possible
2365 // deopt states we need a more precise invalidation of checking the mod ref
2366 // info of each instruction within the loop and LI. This has a complexity of
2367 // O(N^2), so currently, it is used only as a diagnostic tool since the
2368 // default value of LICMN2Threshold is zero.
2369
2370 // Don't look at nested loops.
2371 if (CurLoop->begin() != CurLoop->end())
2372 return true;
2373
2374 int N = 0;
2375 for (BasicBlock *BB : CurLoop->getBlocks())
2376 for (Instruction &I : *BB) {
2377 if (N >= LICMN2Theshold) {
2378 LLVM_DEBUG(dbgs() << "Alasing N2 threshold exhausted for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Alasing N2 threshold exhausted for "
<< *(MemLoc.Ptr) << "\n"; } } while (false)
2379 << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Alasing N2 threshold exhausted for "
<< *(MemLoc.Ptr) << "\n"; } } while (false)
;
2380 return true;
2381 }
2382 N++;
2383 auto Res = AA->getModRefInfo(&I, MemLoc);
2384 if (isModSet(Res)) {
2385 LLVM_DEBUG(dbgs() << "Aliasing failed on " << I << " for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing failed on " << I <<
" for " << *(MemLoc.Ptr) << "\n"; } } while (false
)
2386 << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing failed on " << I <<
" for " << *(MemLoc.Ptr) << "\n"; } } while (false
)
;
2387 return true;
2388 }
2389 }
2390 LLVM_DEBUG(dbgs() << "Aliasing okay for " << *(MemLoc.Ptr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Aliasing okay for " << *(MemLoc
.Ptr) << "\n"; } } while (false)
;
2391 return false;
2392}
2393
2394bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
2395 Loop *CurLoop, Instruction &I,
2396 SinkAndHoistLICMFlags &Flags) {
2397 // For hoisting, use the walker to determine safety
2398 if (!Flags.getIsSink()) {
2399 MemoryAccess *Source;
2400 // See declaration of SetLicmMssaOptCap for usage details.
2401 if (Flags.tooManyClobberingCalls())
2402 Source = MU->getDefiningAccess();
2403 else {
2404 Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
2405 Flags.incrementClobberingCalls();
2406 }
2407 return !MSSA->isLiveOnEntryDef(Source) &&
2408 CurLoop->contains(Source->getBlock());
2409 }
2410
2411 // For sinking, we'd need to check all Defs below this use. The getClobbering
2412 // call will look on the backedge of the loop, but will check aliasing with
2413 // the instructions on the previous iteration.
2414 // For example:
2415 // for (i ... )
2416 // load a[i] ( Use (LoE)
2417 // store a[i] ( 1 = Def (2), with 2 = Phi for the loop.
2418 // i++;
2419 // The load sees no clobbering inside the loop, as the backedge alias check
2420 // does phi translation, and will check aliasing against store a[i-1].
2421 // However sinking the load outside the loop, below the store is incorrect.
2422
2423 // For now, only sink if there are no Defs in the loop, and the existing ones
2424 // precede the use and are in the same block.
2425 // FIXME: Increase precision: Safe to sink if Use post dominates the Def;
2426 // needs PostDominatorTreeAnalysis.
2427 // FIXME: More precise: no Defs that alias this Use.
2428 if (Flags.tooManyMemoryAccesses())
2429 return true;
2430 for (auto *BB : CurLoop->getBlocks())
2431 if (pointerInvalidatedByBlockWithMSSA(*BB, *MSSA, *MU))
2432 return true;
2433 // When sinking, the source block may not be part of the loop so check it.
2434 if (!CurLoop->contains(&I))
2435 return pointerInvalidatedByBlockWithMSSA(*I.getParent(), *MSSA, *MU);
2436
2437 return false;
2438}
2439
2440bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
2441 MemoryUse &MU) {
2442 if (const auto *Accesses = MSSA.getBlockDefs(&BB))
2443 for (const auto &MA : *Accesses)
2444 if (const auto *MD = dyn_cast<MemoryDef>(&MA))
2445 if (MU.getBlock() != MD->getBlock() || !MSSA.locallyDominates(MD, &MU))
2446 return true;
2447 return false;
2448}
2449
2450/// Little predicate that returns true if the specified basic block is in
2451/// a subloop of the current one, not the current one itself.
2452///
2453static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI) {
2454 assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop")(static_cast <bool> (CurLoop->contains(BB) &&
"Only valid if BB is IN the loop") ? void (0) : __assert_fail
("CurLoop->contains(BB) && \"Only valid if BB is IN the loop\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2454, __extension__ __PRETTY_FUNCTION__))
;
2455 return LI->getLoopFor(BB) != CurLoop;
2456}

/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h

1//===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file exposes the class definitions of all of the subclasses of the
10// Instruction class. This is meant to be an easy way to get access to all
11// instruction subclasses.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRUCTIONS_H
16#define LLVM_IR_INSTRUCTIONS_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/Bitfields.h"
20#include "llvm/ADT/None.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/ADT/iterator.h"
26#include "llvm/ADT/iterator_range.h"
27#include "llvm/IR/Attributes.h"
28#include "llvm/IR/BasicBlock.h"
29#include "llvm/IR/CallingConv.h"
30#include "llvm/IR/CFG.h"
31#include "llvm/IR/Constant.h"
32#include "llvm/IR/DerivedTypes.h"
33#include "llvm/IR/Function.h"
34#include "llvm/IR/InstrTypes.h"
35#include "llvm/IR/Instruction.h"
36#include "llvm/IR/OperandTraits.h"
37#include "llvm/IR/Type.h"
38#include "llvm/IR/Use.h"
39#include "llvm/IR/User.h"
40#include "llvm/IR/Value.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/ErrorHandling.h"
44#include <cassert>
45#include <cstddef>
46#include <cstdint>
47#include <iterator>
48
49namespace llvm {
50
51class APInt;
52class ConstantInt;
53class DataLayout;
54class LLVMContext;
55
56//===----------------------------------------------------------------------===//
57// AllocaInst Class
58//===----------------------------------------------------------------------===//
59
60/// an instruction to allocate memory on the stack
61class AllocaInst : public UnaryInstruction {
62 Type *AllocatedType;
63
64 using AlignmentField = AlignmentBitfieldElementT<0>;
65 using UsedWithInAllocaField = BoolBitfieldElementT<AlignmentField::NextBit>;
66 using SwiftErrorField = BoolBitfieldElementT<UsedWithInAllocaField::NextBit>;
67 static_assert(Bitfield::areContiguous<AlignmentField, UsedWithInAllocaField,
68 SwiftErrorField>(),
69 "Bitfields must be contiguous");
70
71protected:
72 // Note: Instruction needs to be a friend here to call cloneImpl.
73 friend class Instruction;
74
75 AllocaInst *cloneImpl() const;
76
77public:
78 explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
79 const Twine &Name, Instruction *InsertBefore);
80 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
81 const Twine &Name, BasicBlock *InsertAtEnd);
82
83 AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
84 Instruction *InsertBefore);
85 AllocaInst(Type *Ty, unsigned AddrSpace,
86 const Twine &Name, BasicBlock *InsertAtEnd);
87
88 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
89 const Twine &Name = "", Instruction *InsertBefore = nullptr);
90 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
91 const Twine &Name, BasicBlock *InsertAtEnd);
92
93 /// Return true if there is an allocation size parameter to the allocation
94 /// instruction that is not 1.
95 bool isArrayAllocation() const;
96
97 /// Get the number of elements allocated. For a simple allocation of a single
98 /// element, this will return a constant 1 value.
99 const Value *getArraySize() const { return getOperand(0); }
100 Value *getArraySize() { return getOperand(0); }
101
102 /// Overload to return most specific pointer type.
103 PointerType *getType() const {
104 return cast<PointerType>(Instruction::getType());
105 }
106
107 /// Get allocation size in bits. Returns None if size can't be determined,
108 /// e.g. in case of a VLA.
109 Optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
110
111 /// Return the type that is being allocated by the instruction.
112 Type *getAllocatedType() const { return AllocatedType; }
113 /// for use only in special circumstances that need to generically
114 /// transform a whole instruction (eg: IR linking and vectorization).
115 void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
116
117 /// Return the alignment of the memory that is being allocated by the
118 /// instruction.
119 Align getAlign() const {
120 return Align(1ULL << getSubclassData<AlignmentField>());
121 }
122
123 void setAlignment(Align Align) {
124 setSubclassData<AlignmentField>(Log2(Align));
125 }
126
127 // FIXME: Remove this one transition to Align is over.
128 unsigned getAlignment() const { return getAlign().value(); }
129
130 /// Return true if this alloca is in the entry block of the function and is a
131 /// constant size. If so, the code generator will fold it into the
132 /// prolog/epilog code, so it is basically free.
133 bool isStaticAlloca() const;
134
135 /// Return true if this alloca is used as an inalloca argument to a call. Such
136 /// allocas are never considered static even if they are in the entry block.
137 bool isUsedWithInAlloca() const {
138 return getSubclassData<UsedWithInAllocaField>();
139 }
140
141 /// Specify whether this alloca is used to represent the arguments to a call.
142 void setUsedWithInAlloca(bool V) {
143 setSubclassData<UsedWithInAllocaField>(V);
144 }
145
146 /// Return true if this alloca is used as a swifterror argument to a call.
147 bool isSwiftError() const { return getSubclassData<SwiftErrorField>(); }
148 /// Specify whether this alloca is used to represent a swifterror.
149 void setSwiftError(bool V) { setSubclassData<SwiftErrorField>(V); }
150
151 // Methods for support type inquiry through isa, cast, and dyn_cast:
152 static bool classof(const Instruction *I) {
153 return (I->getOpcode() == Instruction::Alloca);
154 }
155 static bool classof(const Value *V) {
156 return isa<Instruction>(V) && classof(cast<Instruction>(V));
157 }
158
159private:
160 // Shadow Instruction::setInstructionSubclassData with a private forwarding
161 // method so that subclasses cannot accidentally use it.
162 template <typename Bitfield>
163 void setSubclassData(typename Bitfield::Type Value) {
164 Instruction::setSubclassData<Bitfield>(Value);
165 }
166};
167
168//===----------------------------------------------------------------------===//
169// LoadInst Class
170//===----------------------------------------------------------------------===//
171
172/// An instruction for reading from memory. This uses the SubclassData field in
173/// Value to store whether or not the load is volatile.
174class LoadInst : public UnaryInstruction {
175 using VolatileField = BoolBitfieldElementT<0>;
176 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
177 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
178 static_assert(
179 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
180 "Bitfields must be contiguous");
181
182 void AssertOK();
183
184protected:
185 // Note: Instruction needs to be a friend here to call cloneImpl.
186 friend class Instruction;
187
188 LoadInst *cloneImpl() const;
189
190public:
191 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
192 Instruction *InsertBefore);
193 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
194 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
195 Instruction *InsertBefore);
196 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
197 BasicBlock *InsertAtEnd);
198 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
199 Align Align, Instruction *InsertBefore = nullptr);
200 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
201 Align Align, BasicBlock *InsertAtEnd);
202 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
203 Align Align, AtomicOrdering Order,
204 SyncScope::ID SSID = SyncScope::System,
205 Instruction *InsertBefore = nullptr);
206 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
207 Align Align, AtomicOrdering Order, SyncScope::ID SSID,
208 BasicBlock *InsertAtEnd);
209
210 /// Return true if this is a load from a volatile memory location.
211 bool isVolatile() const { return getSubclassData<VolatileField>(); }
212
213 /// Specify whether this is a volatile load or not.
214 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
215
216 /// Return the alignment of the access that is being performed.
217 /// FIXME: Remove this function once transition to Align is over.
218 /// Use getAlign() instead.
219 unsigned getAlignment() const { return getAlign().value(); }
220
221 /// Return the alignment of the access that is being performed.
222 Align getAlign() const {
223 return Align(1ULL << (getSubclassData<AlignmentField>()));
224 }
225
226 void setAlignment(Align Align) {
227 setSubclassData<AlignmentField>(Log2(Align));
228 }
229
230 /// Returns the ordering constraint of this load instruction.
231 AtomicOrdering getOrdering() const {
232 return getSubclassData<OrderingField>();
233 }
234 /// Sets the ordering constraint of this load instruction. May not be Release
235 /// or AcquireRelease.
236 void setOrdering(AtomicOrdering Ordering) {
237 setSubclassData<OrderingField>(Ordering);
238 }
239
240 /// Returns the synchronization scope ID of this load instruction.
241 SyncScope::ID getSyncScopeID() const {
242 return SSID;
243 }
244
245 /// Sets the synchronization scope ID of this load instruction.
246 void setSyncScopeID(SyncScope::ID SSID) {
247 this->SSID = SSID;
248 }
249
250 /// Sets the ordering constraint and the synchronization scope ID of this load
251 /// instruction.
252 void setAtomic(AtomicOrdering Ordering,
253 SyncScope::ID SSID = SyncScope::System) {
254 setOrdering(Ordering);
255 setSyncScopeID(SSID);
256 }
257
258 bool isSimple() const { return !isAtomic() && !isVolatile(); }
259
260 bool isUnordered() const {
261 return (getOrdering() == AtomicOrdering::NotAtomic ||
16
Assuming the condition is true
18
Returning the value 1, which participates in a condition later
262 getOrdering() == AtomicOrdering::Unordered) &&
263 !isVolatile();
17
Assuming the condition is true
264 }
265
266 Value *getPointerOperand() { return getOperand(0); }
267 const Value *getPointerOperand() const { return getOperand(0); }
268 static unsigned getPointerOperandIndex() { return 0U; }
269 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
270
271 /// Returns the address space of the pointer operand.
272 unsigned getPointerAddressSpace() const {
273 return getPointerOperandType()->getPointerAddressSpace();
274 }
275
276 // Methods for support type inquiry through isa, cast, and dyn_cast:
277 static bool classof(const Instruction *I) {
278 return I->getOpcode() == Instruction::Load;
279 }
280 static bool classof(const Value *V) {
281 return isa<Instruction>(V) && classof(cast<Instruction>(V));
282 }
283
284private:
285 // Shadow Instruction::setInstructionSubclassData with a private forwarding
286 // method so that subclasses cannot accidentally use it.
287 template <typename Bitfield>
288 void setSubclassData(typename Bitfield::Type Value) {
289 Instruction::setSubclassData<Bitfield>(Value);
290 }
291
292 /// The synchronization scope ID of this load instruction. Not quite enough
293 /// room in SubClassData for everything, so synchronization scope ID gets its
294 /// own field.
295 SyncScope::ID SSID;
296};
297
298//===----------------------------------------------------------------------===//
299// StoreInst Class
300//===----------------------------------------------------------------------===//
301
302/// An instruction for storing to memory.
303class StoreInst : public Instruction {
304 using VolatileField = BoolBitfieldElementT<0>;
305 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
306 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
307 static_assert(
308 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
309 "Bitfields must be contiguous");
310
311 void AssertOK();
312
313protected:
314 // Note: Instruction needs to be a friend here to call cloneImpl.
315 friend class Instruction;
316
317 StoreInst *cloneImpl() const;
318
319public:
320 StoreInst(Value *Val, Value *Ptr, Instruction *InsertBefore);
321 StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
322 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Instruction *InsertBefore);
323 StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
324 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
325 Instruction *InsertBefore = nullptr);
326 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
327 BasicBlock *InsertAtEnd);
328 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
329 AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
330 Instruction *InsertBefore = nullptr);
331 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
332 AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd);
333
334 // allocate space for exactly two operands
335 void *operator new(size_t s) {
336 return User::operator new(s, 2);
337 }
338
339 /// Return true if this is a store to a volatile memory location.
340 bool isVolatile() const { return getSubclassData<VolatileField>(); }
341
342 /// Specify whether this is a volatile store or not.
343 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
344
345 /// Transparently provide more efficient getOperand methods.
346 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
347
348 /// Return the alignment of the access that is being performed
349 /// FIXME: Remove this function once transition to Align is over.
350 /// Use getAlign() instead.
351 unsigned getAlignment() const { return getAlign().value(); }
352
353 Align getAlign() const {
354 return Align(1ULL << (getSubclassData<AlignmentField>()));
355 }
356
357 void setAlignment(Align Align) {
358 setSubclassData<AlignmentField>(Log2(Align));
359 }
360
361 /// Returns the ordering constraint of this store instruction.
362 AtomicOrdering getOrdering() const {
363 return getSubclassData<OrderingField>();
364 }
365
366 /// Sets the ordering constraint of this store instruction. May not be
367 /// Acquire or AcquireRelease.
368 void setOrdering(AtomicOrdering Ordering) {
369 setSubclassData<OrderingField>(Ordering);
370 }
371
372 /// Returns the synchronization scope ID of this store instruction.
373 SyncScope::ID getSyncScopeID() const {
374 return SSID;
375 }
376
377 /// Sets the synchronization scope ID of this store instruction.
378 void setSyncScopeID(SyncScope::ID SSID) {
379 this->SSID = SSID;
380 }
381
382 /// Sets the ordering constraint and the synchronization scope ID of this
383 /// store instruction.
384 void setAtomic(AtomicOrdering Ordering,
385 SyncScope::ID SSID = SyncScope::System) {
386 setOrdering(Ordering);
387 setSyncScopeID(SSID);
388 }
389
390 bool isSimple() const { return !isAtomic() && !isVolatile(); }
391
392 bool isUnordered() const {
393 return (getOrdering() == AtomicOrdering::NotAtomic ||
394 getOrdering() == AtomicOrdering::Unordered) &&
395 !isVolatile();
396 }
397
398 Value *getValueOperand() { return getOperand(0); }
399 const Value *getValueOperand() const { return getOperand(0); }
400
401 Value *getPointerOperand() { return getOperand(1); }
402 const Value *getPointerOperand() const { return getOperand(1); }
403 static unsigned getPointerOperandIndex() { return 1U; }
404 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
405
406 /// Returns the address space of the pointer operand.
407 unsigned getPointerAddressSpace() const {
408 return getPointerOperandType()->getPointerAddressSpace();
409 }
410
411 // Methods for support type inquiry through isa, cast, and dyn_cast:
412 static bool classof(const Instruction *I) {
413 return I->getOpcode() == Instruction::Store;
414 }
415 static bool classof(const Value *V) {
416 return isa<Instruction>(V) && classof(cast<Instruction>(V));
417 }
418
419private:
420 // Shadow Instruction::setInstructionSubclassData with a private forwarding
421 // method so that subclasses cannot accidentally use it.
422 template <typename Bitfield>
423 void setSubclassData(typename Bitfield::Type Value) {
424 Instruction::setSubclassData<Bitfield>(Value);
425 }
426
427 /// The synchronization scope ID of this store instruction. Not quite enough
428 /// room in SubClassData for everything, so synchronization scope ID gets its
429 /// own field.
430 SyncScope::ID SSID;
431};
432
433template <>
434struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
435};
436
437DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)StoreInst::op_iterator StoreInst::op_begin() { return OperandTraits
<StoreInst>::op_begin(this); } StoreInst::const_op_iterator
StoreInst::op_begin() const { return OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this)); } StoreInst
::op_iterator StoreInst::op_end() { return OperandTraits<StoreInst
>::op_end(this); } StoreInst::const_op_iterator StoreInst::
op_end() const { return OperandTraits<StoreInst>::op_end
(const_cast<StoreInst*>(this)); } Value *StoreInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<StoreInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 437, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<StoreInst>::op_begin(const_cast
<StoreInst*>(this))[i_nocapture].get()); } void StoreInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<StoreInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 437, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
StoreInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned StoreInst::getNumOperands() const { return OperandTraits
<StoreInst>::operands(this); } template <int Idx_nocapture
> Use &StoreInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
StoreInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
438
439//===----------------------------------------------------------------------===//
440// FenceInst Class
441//===----------------------------------------------------------------------===//
442
443/// An instruction for ordering other memory operations.
444class FenceInst : public Instruction {
445 using OrderingField = AtomicOrderingBitfieldElementT<0>;
446
447 void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
448
449protected:
450 // Note: Instruction needs to be a friend here to call cloneImpl.
451 friend class Instruction;
452
453 FenceInst *cloneImpl() const;
454
455public:
456 // Ordering may only be Acquire, Release, AcquireRelease, or
457 // SequentiallyConsistent.
458 FenceInst(LLVMContext &C, AtomicOrdering Ordering,
459 SyncScope::ID SSID = SyncScope::System,
460 Instruction *InsertBefore = nullptr);
461 FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
462 BasicBlock *InsertAtEnd);
463
464 // allocate space for exactly zero operands
465 void *operator new(size_t s) {
466 return User::operator new(s, 0);
467 }
468
469 /// Returns the ordering constraint of this fence instruction.
470 AtomicOrdering getOrdering() const {
471 return getSubclassData<OrderingField>();
472 }
473
474 /// Sets the ordering constraint of this fence instruction. May only be
475 /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
476 void setOrdering(AtomicOrdering Ordering) {
477 setSubclassData<OrderingField>(Ordering);
478 }
479
480 /// Returns the synchronization scope ID of this fence instruction.
481 SyncScope::ID getSyncScopeID() const {
482 return SSID;
483 }
484
485 /// Sets the synchronization scope ID of this fence instruction.
486 void setSyncScopeID(SyncScope::ID SSID) {
487 this->SSID = SSID;
488 }
489
490 // Methods for support type inquiry through isa, cast, and dyn_cast:
491 static bool classof(const Instruction *I) {
492 return I->getOpcode() == Instruction::Fence;
493 }
494 static bool classof(const Value *V) {
495 return isa<Instruction>(V) && classof(cast<Instruction>(V));
496 }
497
498private:
499 // Shadow Instruction::setInstructionSubclassData with a private forwarding
500 // method so that subclasses cannot accidentally use it.
501 template <typename Bitfield>
502 void setSubclassData(typename Bitfield::Type Value) {
503 Instruction::setSubclassData<Bitfield>(Value);
504 }
505
506 /// The synchronization scope ID of this fence instruction. Not quite enough
507 /// room in SubClassData for everything, so synchronization scope ID gets its
508 /// own field.
509 SyncScope::ID SSID;
510};
511
512//===----------------------------------------------------------------------===//
513// AtomicCmpXchgInst Class
514//===----------------------------------------------------------------------===//
515
516/// An instruction that atomically checks whether a
517/// specified value is in a memory location, and, if it is, stores a new value
518/// there. The value returned by this instruction is a pair containing the
519/// original value as first element, and an i1 indicating success (true) or
520/// failure (false) as second element.
521///
522class AtomicCmpXchgInst : public Instruction {
523 void Init(Value *Ptr, Value *Cmp, Value *NewVal, Align Align,
524 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
525 SyncScope::ID SSID);
526
527 template <unsigned Offset>
528 using AtomicOrderingBitfieldElement =
529 typename Bitfield::Element<AtomicOrdering, Offset, 3,
530 AtomicOrdering::LAST>;
531
532protected:
533 // Note: Instruction needs to be a friend here to call cloneImpl.
534 friend class Instruction;
535
536 AtomicCmpXchgInst *cloneImpl() const;
537
538public:
539 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
540 AtomicOrdering SuccessOrdering,
541 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
542 Instruction *InsertBefore = nullptr);
543 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
544 AtomicOrdering SuccessOrdering,
545 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
546 BasicBlock *InsertAtEnd);
547
548 // allocate space for exactly three operands
549 void *operator new(size_t s) {
550 return User::operator new(s, 3);
551 }
552
553 using VolatileField = BoolBitfieldElementT<0>;
554 using WeakField = BoolBitfieldElementT<VolatileField::NextBit>;
555 using SuccessOrderingField =
556 AtomicOrderingBitfieldElementT<WeakField::NextBit>;
557 using FailureOrderingField =
558 AtomicOrderingBitfieldElementT<SuccessOrderingField::NextBit>;
559 using AlignmentField =
560 AlignmentBitfieldElementT<FailureOrderingField::NextBit>;
561 static_assert(
562 Bitfield::areContiguous<VolatileField, WeakField, SuccessOrderingField,
563 FailureOrderingField, AlignmentField>(),
564 "Bitfields must be contiguous");
565
566 /// Return the alignment of the memory that is being allocated by the
567 /// instruction.
568 Align getAlign() const {
569 return Align(1ULL << getSubclassData<AlignmentField>());
570 }
571
572 void setAlignment(Align Align) {
573 setSubclassData<AlignmentField>(Log2(Align));
574 }
575
576 /// Return true if this is a cmpxchg from a volatile memory
577 /// location.
578 ///
579 bool isVolatile() const { return getSubclassData<VolatileField>(); }
580
581 /// Specify whether this is a volatile cmpxchg.
582 ///
583 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
584
585 /// Return true if this cmpxchg may spuriously fail.
586 bool isWeak() const { return getSubclassData<WeakField>(); }
587
588 void setWeak(bool IsWeak) { setSubclassData<WeakField>(IsWeak); }
589
590 /// Transparently provide more efficient getOperand methods.
591 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
592
593 static bool isValidSuccessOrdering(AtomicOrdering Ordering) {
594 return Ordering != AtomicOrdering::NotAtomic &&
595 Ordering != AtomicOrdering::Unordered;
596 }
597
598 static bool isValidFailureOrdering(AtomicOrdering Ordering) {
599 return Ordering != AtomicOrdering::NotAtomic &&
600 Ordering != AtomicOrdering::Unordered &&
601 Ordering != AtomicOrdering::AcquireRelease &&
602 Ordering != AtomicOrdering::Release;
603 }
604
605 /// Returns the success ordering constraint of this cmpxchg instruction.
606 AtomicOrdering getSuccessOrdering() const {
607 return getSubclassData<SuccessOrderingField>();
608 }
609
610 /// Sets the success ordering constraint of this cmpxchg instruction.
611 void setSuccessOrdering(AtomicOrdering Ordering) {
612 assert(isValidSuccessOrdering(Ordering) &&(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 613, __extension__ __PRETTY_FUNCTION__))
613 "invalid CmpXchg success ordering")(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 613, __extension__ __PRETTY_FUNCTION__))
;
614 setSubclassData<SuccessOrderingField>(Ordering);
615 }
616
617 /// Returns the failure ordering constraint of this cmpxchg instruction.
618 AtomicOrdering getFailureOrdering() const {
619 return getSubclassData<FailureOrderingField>();
620 }
621
622 /// Sets the failure ordering constraint of this cmpxchg instruction.
623 void setFailureOrdering(AtomicOrdering Ordering) {
624 assert(isValidFailureOrdering(Ordering) &&(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 625, __extension__ __PRETTY_FUNCTION__))
625 "invalid CmpXchg failure ordering")(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 625, __extension__ __PRETTY_FUNCTION__))
;
626 setSubclassData<FailureOrderingField>(Ordering);
627 }
628
629 /// Returns a single ordering which is at least as strong as both the
630 /// success and failure orderings for this cmpxchg.
631 AtomicOrdering getMergedOrdering() const {
632 if (getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
633 return AtomicOrdering::SequentiallyConsistent;
634 if (getFailureOrdering() == AtomicOrdering::Acquire) {
635 if (getSuccessOrdering() == AtomicOrdering::Monotonic)
636 return AtomicOrdering::Acquire;
637 if (getSuccessOrdering() == AtomicOrdering::Release)
638 return AtomicOrdering::AcquireRelease;
639 }
640 return getSuccessOrdering();
641 }
642
643 /// Returns the synchronization scope ID of this cmpxchg instruction.
644 SyncScope::ID getSyncScopeID() const {
645 return SSID;
646 }
647
648 /// Sets the synchronization scope ID of this cmpxchg instruction.
649 void setSyncScopeID(SyncScope::ID SSID) {
650 this->SSID = SSID;
651 }
652
653 Value *getPointerOperand() { return getOperand(0); }
654 const Value *getPointerOperand() const { return getOperand(0); }
655 static unsigned getPointerOperandIndex() { return 0U; }
656
657 Value *getCompareOperand() { return getOperand(1); }
658 const Value *getCompareOperand() const { return getOperand(1); }
659
660 Value *getNewValOperand() { return getOperand(2); }
661 const Value *getNewValOperand() const { return getOperand(2); }
662
663 /// Returns the address space of the pointer operand.
664 unsigned getPointerAddressSpace() const {
665 return getPointerOperand()->getType()->getPointerAddressSpace();
666 }
667
668 /// Returns the strongest permitted ordering on failure, given the
669 /// desired ordering on success.
670 ///
671 /// If the comparison in a cmpxchg operation fails, there is no atomic store
672 /// so release semantics cannot be provided. So this function drops explicit
673 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
674 /// operation would remain SequentiallyConsistent.
675 static AtomicOrdering
676 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
677 switch (SuccessOrdering) {
678 default:
679 llvm_unreachable("invalid cmpxchg success ordering")::llvm::llvm_unreachable_internal("invalid cmpxchg success ordering"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 679)
;
680 case AtomicOrdering::Release:
681 case AtomicOrdering::Monotonic:
682 return AtomicOrdering::Monotonic;
683 case AtomicOrdering::AcquireRelease:
684 case AtomicOrdering::Acquire:
685 return AtomicOrdering::Acquire;
686 case AtomicOrdering::SequentiallyConsistent:
687 return AtomicOrdering::SequentiallyConsistent;
688 }
689 }
690
691 // Methods for support type inquiry through isa, cast, and dyn_cast:
692 static bool classof(const Instruction *I) {
693 return I->getOpcode() == Instruction::AtomicCmpXchg;
694 }
695 static bool classof(const Value *V) {
696 return isa<Instruction>(V) && classof(cast<Instruction>(V));
697 }
698
699private:
700 // Shadow Instruction::setInstructionSubclassData with a private forwarding
701 // method so that subclasses cannot accidentally use it.
702 template <typename Bitfield>
703 void setSubclassData(typename Bitfield::Type Value) {
704 Instruction::setSubclassData<Bitfield>(Value);
705 }
706
707 /// The synchronization scope ID of this cmpxchg instruction. Not quite
708 /// enough room in SubClassData for everything, so synchronization scope ID
709 /// gets its own field.
710 SyncScope::ID SSID;
711};
712
713template <>
714struct OperandTraits<AtomicCmpXchgInst> :
715 public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
716};
717
718DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)AtomicCmpXchgInst::op_iterator AtomicCmpXchgInst::op_begin() {
return OperandTraits<AtomicCmpXchgInst>::op_begin(this
); } AtomicCmpXchgInst::const_op_iterator AtomicCmpXchgInst::
op_begin() const { return OperandTraits<AtomicCmpXchgInst>
::op_begin(const_cast<AtomicCmpXchgInst*>(this)); } AtomicCmpXchgInst
::op_iterator AtomicCmpXchgInst::op_end() { return OperandTraits
<AtomicCmpXchgInst>::op_end(this); } AtomicCmpXchgInst::
const_op_iterator AtomicCmpXchgInst::op_end() const { return OperandTraits
<AtomicCmpXchgInst>::op_end(const_cast<AtomicCmpXchgInst
*>(this)); } Value *AtomicCmpXchgInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 718, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<AtomicCmpXchgInst>::op_begin
(const_cast<AtomicCmpXchgInst*>(this))[i_nocapture].get
()); } void AtomicCmpXchgInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 718, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
AtomicCmpXchgInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned AtomicCmpXchgInst::getNumOperands() const { return
OperandTraits<AtomicCmpXchgInst>::operands(this); } template
<int Idx_nocapture> Use &AtomicCmpXchgInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &AtomicCmpXchgInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
719
720//===----------------------------------------------------------------------===//
721// AtomicRMWInst Class
722//===----------------------------------------------------------------------===//
723
724/// an instruction that atomically reads a memory location,
725/// combines it with another value, and then stores the result back. Returns
726/// the old value.
727///
728class AtomicRMWInst : public Instruction {
729protected:
730 // Note: Instruction needs to be a friend here to call cloneImpl.
731 friend class Instruction;
732
733 AtomicRMWInst *cloneImpl() const;
734
735public:
736 /// This enumeration lists the possible modifications atomicrmw can make. In
737 /// the descriptions, 'p' is the pointer to the instruction's memory location,
738 /// 'old' is the initial value of *p, and 'v' is the other value passed to the
739 /// instruction. These instructions always return 'old'.
740 enum BinOp : unsigned {
741 /// *p = v
742 Xchg,
743 /// *p = old + v
744 Add,
745 /// *p = old - v
746 Sub,
747 /// *p = old & v
748 And,
749 /// *p = ~(old & v)
750 Nand,
751 /// *p = old | v
752 Or,
753 /// *p = old ^ v
754 Xor,
755 /// *p = old >signed v ? old : v
756 Max,
757 /// *p = old <signed v ? old : v
758 Min,
759 /// *p = old >unsigned v ? old : v
760 UMax,
761 /// *p = old <unsigned v ? old : v
762 UMin,
763
764 /// *p = old + v
765 FAdd,
766
767 /// *p = old - v
768 FSub,
769
770 FIRST_BINOP = Xchg,
771 LAST_BINOP = FSub,
772 BAD_BINOP
773 };
774
775private:
776 template <unsigned Offset>
777 using AtomicOrderingBitfieldElement =
778 typename Bitfield::Element<AtomicOrdering, Offset, 3,
779 AtomicOrdering::LAST>;
780
781 template <unsigned Offset>
782 using BinOpBitfieldElement =
783 typename Bitfield::Element<BinOp, Offset, 4, BinOp::LAST_BINOP>;
784
785public:
786 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
787 AtomicOrdering Ordering, SyncScope::ID SSID,
788 Instruction *InsertBefore = nullptr);
789 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
790 AtomicOrdering Ordering, SyncScope::ID SSID,
791 BasicBlock *InsertAtEnd);
792
793 // allocate space for exactly two operands
794 void *operator new(size_t s) {
795 return User::operator new(s, 2);
796 }
797
798 using VolatileField = BoolBitfieldElementT<0>;
799 using AtomicOrderingField =
800 AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
801 using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
802 using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
803 static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
804 OperationField, AlignmentField>(),
805 "Bitfields must be contiguous");
806
807 BinOp getOperation() const { return getSubclassData<OperationField>(); }
808
809 static StringRef getOperationName(BinOp Op);
810
811 static bool isFPOperation(BinOp Op) {
812 switch (Op) {
813 case AtomicRMWInst::FAdd:
814 case AtomicRMWInst::FSub:
815 return true;
816 default:
817 return false;
818 }
819 }
820
821 void setOperation(BinOp Operation) {
822 setSubclassData<OperationField>(Operation);
823 }
824
825 /// Return the alignment of the memory that is being allocated by the
826 /// instruction.
827 Align getAlign() const {
828 return Align(1ULL << getSubclassData<AlignmentField>());
829 }
830
831 void setAlignment(Align Align) {
832 setSubclassData<AlignmentField>(Log2(Align));
833 }
834
835 /// Return true if this is a RMW on a volatile memory location.
836 ///
837 bool isVolatile() const { return getSubclassData<VolatileField>(); }
838
839 /// Specify whether this is a volatile RMW or not.
840 ///
841 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
842
843 /// Transparently provide more efficient getOperand methods.
844 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
845
846 /// Returns the ordering constraint of this rmw instruction.
847 AtomicOrdering getOrdering() const {
848 return getSubclassData<AtomicOrderingField>();
849 }
850
851 /// Sets the ordering constraint of this rmw instruction.
852 void setOrdering(AtomicOrdering Ordering) {
853 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 854, __extension__ __PRETTY_FUNCTION__))
854 "atomicrmw instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 854, __extension__ __PRETTY_FUNCTION__))
;
855 setSubclassData<AtomicOrderingField>(Ordering);
856 }
857
858 /// Returns the synchronization scope ID of this rmw instruction.
859 SyncScope::ID getSyncScopeID() const {
860 return SSID;
861 }
862
863 /// Sets the synchronization scope ID of this rmw instruction.
864 void setSyncScopeID(SyncScope::ID SSID) {
865 this->SSID = SSID;
866 }
867
868 Value *getPointerOperand() { return getOperand(0); }
869 const Value *getPointerOperand() const { return getOperand(0); }
870 static unsigned getPointerOperandIndex() { return 0U; }
871
872 Value *getValOperand() { return getOperand(1); }
873 const Value *getValOperand() const { return getOperand(1); }
874
875 /// Returns the address space of the pointer operand.
876 unsigned getPointerAddressSpace() const {
877 return getPointerOperand()->getType()->getPointerAddressSpace();
878 }
879
880 bool isFloatingPointOperation() const {
881 return isFPOperation(getOperation());
882 }
883
884 // Methods for support type inquiry through isa, cast, and dyn_cast:
885 static bool classof(const Instruction *I) {
886 return I->getOpcode() == Instruction::AtomicRMW;
887 }
888 static bool classof(const Value *V) {
889 return isa<Instruction>(V) && classof(cast<Instruction>(V));
890 }
891
892private:
893 void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
894 AtomicOrdering Ordering, SyncScope::ID SSID);
895
896 // Shadow Instruction::setInstructionSubclassData with a private forwarding
897 // method so that subclasses cannot accidentally use it.
898 template <typename Bitfield>
899 void setSubclassData(typename Bitfield::Type Value) {
900 Instruction::setSubclassData<Bitfield>(Value);
901 }
902
903 /// The synchronization scope ID of this rmw instruction. Not quite enough
904 /// room in SubClassData for everything, so synchronization scope ID gets its
905 /// own field.
906 SyncScope::ID SSID;
907};
908
909template <>
910struct OperandTraits<AtomicRMWInst>
911 : public FixedNumOperandTraits<AtomicRMWInst,2> {
912};
913
914DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)AtomicRMWInst::op_iterator AtomicRMWInst::op_begin() { return
OperandTraits<AtomicRMWInst>::op_begin(this); } AtomicRMWInst
::const_op_iterator AtomicRMWInst::op_begin() const { return OperandTraits
<AtomicRMWInst>::op_begin(const_cast<AtomicRMWInst*>
(this)); } AtomicRMWInst::op_iterator AtomicRMWInst::op_end()
{ return OperandTraits<AtomicRMWInst>::op_end(this); }
AtomicRMWInst::const_op_iterator AtomicRMWInst::op_end() const
{ return OperandTraits<AtomicRMWInst>::op_end(const_cast
<AtomicRMWInst*>(this)); } Value *AtomicRMWInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 914, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<AtomicRMWInst>::op_begin(const_cast
<AtomicRMWInst*>(this))[i_nocapture].get()); } void AtomicRMWInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<AtomicRMWInst
>::operands(this) && "setOperand() out of range!")
? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 914, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
AtomicRMWInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned AtomicRMWInst::getNumOperands() const { return OperandTraits
<AtomicRMWInst>::operands(this); } template <int Idx_nocapture
> Use &AtomicRMWInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &AtomicRMWInst::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
915
916//===----------------------------------------------------------------------===//
917// GetElementPtrInst Class
918//===----------------------------------------------------------------------===//
919
920// checkGEPType - Simple wrapper function to give a better assertion failure
921// message on bad indexes for a gep instruction.
922//
923inline Type *checkGEPType(Type *Ty) {
924 assert(Ty && "Invalid GetElementPtrInst indices for type!")(static_cast <bool> (Ty && "Invalid GetElementPtrInst indices for type!"
) ? void (0) : __assert_fail ("Ty && \"Invalid GetElementPtrInst indices for type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 924, __extension__ __PRETTY_FUNCTION__))
;
925 return Ty;
926}
927
928/// an instruction for type-safe pointer arithmetic to
929/// access elements of arrays and structs
930///
931class GetElementPtrInst : public Instruction {
932 Type *SourceElementType;
933 Type *ResultElementType;
934
935 GetElementPtrInst(const GetElementPtrInst &GEPI);
936
937 /// Constructors - Create a getelementptr instruction with a base pointer an
938 /// list of indices. The first ctor can optionally insert before an existing
939 /// instruction, the second appends the new instruction to the specified
940 /// BasicBlock.
941 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
942 ArrayRef<Value *> IdxList, unsigned Values,
943 const Twine &NameStr, Instruction *InsertBefore);
944 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
945 ArrayRef<Value *> IdxList, unsigned Values,
946 const Twine &NameStr, BasicBlock *InsertAtEnd);
947
948 void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
949
950protected:
951 // Note: Instruction needs to be a friend here to call cloneImpl.
952 friend class Instruction;
953
954 GetElementPtrInst *cloneImpl() const;
955
956public:
957 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
958 ArrayRef<Value *> IdxList,
959 const Twine &NameStr = "",
960 Instruction *InsertBefore = nullptr) {
961 unsigned Values = 1 + unsigned(IdxList.size());
962 if (!PointeeType) {
963 PointeeType =
964 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType();
965 } else {
966 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 967, __extension__ __PRETTY_FUNCTION__))
967 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 967, __extension__ __PRETTY_FUNCTION__))
;
968 }
969 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
970 NameStr, InsertBefore);
971 }
972
973 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
974 ArrayRef<Value *> IdxList,
975 const Twine &NameStr,
976 BasicBlock *InsertAtEnd) {
977 unsigned Values = 1 + unsigned(IdxList.size());
978 if (!PointeeType) {
979 PointeeType =
980 cast<PointerType>(Ptr->getType()->getScalarType())->getElementType();
981 } else {
982 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 983, __extension__ __PRETTY_FUNCTION__))
983 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 983, __extension__ __PRETTY_FUNCTION__))
;
984 }
985 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
986 NameStr, InsertAtEnd);
987 }
988
989 /// Create an "inbounds" getelementptr. See the documentation for the
990 /// "inbounds" flag in LangRef.html for details.
991 static GetElementPtrInst *CreateInBounds(Value *Ptr,
992 ArrayRef<Value *> IdxList,
993 const Twine &NameStr = "",
994 Instruction *InsertBefore = nullptr){
995 return CreateInBounds(nullptr, Ptr, IdxList, NameStr, InsertBefore);
996 }
997
998 static GetElementPtrInst *
999 CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef<Value *> IdxList,
1000 const Twine &NameStr = "",
1001 Instruction *InsertBefore = nullptr) {
1002 GetElementPtrInst *GEP =
1003 Create(PointeeType, Ptr, IdxList, NameStr, InsertBefore);
1004 GEP->setIsInBounds(true);
1005 return GEP;
1006 }
1007
1008 static GetElementPtrInst *CreateInBounds(Value *Ptr,
1009 ArrayRef<Value *> IdxList,
1010 const Twine &NameStr,
1011 BasicBlock *InsertAtEnd) {
1012 return CreateInBounds(nullptr, Ptr, IdxList, NameStr, InsertAtEnd);
1013 }
1014
1015 static GetElementPtrInst *CreateInBounds(Type *PointeeType, Value *Ptr,
1016 ArrayRef<Value *> IdxList,
1017 const Twine &NameStr,
1018 BasicBlock *InsertAtEnd) {
1019 GetElementPtrInst *GEP =
1020 Create(PointeeType, Ptr, IdxList, NameStr, InsertAtEnd);
1021 GEP->setIsInBounds(true);
1022 return GEP;
1023 }
1024
1025 /// Transparently provide more efficient getOperand methods.
1026 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1027
1028 Type *getSourceElementType() const { return SourceElementType; }
1029
1030 void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
1031 void setResultElementType(Type *Ty) { ResultElementType = Ty; }
1032
1033 Type *getResultElementType() const {
1034 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1035, __extension__ __PRETTY_FUNCTION__))
1035 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1035, __extension__ __PRETTY_FUNCTION__))
;
1036 return ResultElementType;
1037 }
1038
1039 /// Returns the address space of this instruction's pointer type.
1040 unsigned getAddressSpace() const {
1041 // Note that this is always the same as the pointer operand's address space
1042 // and that is cheaper to compute, so cheat here.
1043 return getPointerAddressSpace();
1044 }
1045
1046 /// Returns the result type of a getelementptr with the given source
1047 /// element type and indexes.
1048 ///
1049 /// Null is returned if the indices are invalid for the specified
1050 /// source element type.
1051 static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
1052 static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
1053 static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
1054
1055 /// Return the type of the element at the given index of an indexable
1056 /// type. This is equivalent to "getIndexedType(Agg, {Zero, Idx})".
1057 ///
1058 /// Returns null if the type can't be indexed, or the given index is not
1059 /// legal for the given type.
1060 static Type *getTypeAtIndex(Type *Ty, Value *Idx);
1061 static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
1062
1063 inline op_iterator idx_begin() { return op_begin()+1; }
1064 inline const_op_iterator idx_begin() const { return op_begin()+1; }
1065 inline op_iterator idx_end() { return op_end(); }
1066 inline const_op_iterator idx_end() const { return op_end(); }
1067
1068 inline iterator_range<op_iterator> indices() {
1069 return make_range(idx_begin(), idx_end());
1070 }
1071
1072 inline iterator_range<const_op_iterator> indices() const {
1073 return make_range(idx_begin(), idx_end());
1074 }
1075
1076 Value *getPointerOperand() {
1077 return getOperand(0);
1078 }
1079 const Value *getPointerOperand() const {
1080 return getOperand(0);
1081 }
1082 static unsigned getPointerOperandIndex() {
1083 return 0U; // get index for modifying correct operand.
1084 }
1085
1086 /// Method to return the pointer operand as a
1087 /// PointerType.
1088 Type *getPointerOperandType() const {
1089 return getPointerOperand()->getType();
1090 }
1091
1092 /// Returns the address space of the pointer operand.
1093 unsigned getPointerAddressSpace() const {
1094 return getPointerOperandType()->getPointerAddressSpace();
1095 }
1096
1097 /// Returns the pointer type returned by the GEP
1098 /// instruction, which may be a vector of pointers.
1099 static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
1100 ArrayRef<Value *> IdxList) {
1101 Type *PtrTy = PointerType::get(checkGEPType(getIndexedType(ElTy, IdxList)),
1102 Ptr->getType()->getPointerAddressSpace());
1103 // Vector GEP
1104 if (auto *PtrVTy = dyn_cast<VectorType>(Ptr->getType())) {
1105 ElementCount EltCount = PtrVTy->getElementCount();
1106 return VectorType::get(PtrTy, EltCount);
1107 }
1108 for (Value *Index : IdxList)
1109 if (auto *IndexVTy = dyn_cast<VectorType>(Index->getType())) {
1110 ElementCount EltCount = IndexVTy->getElementCount();
1111 return VectorType::get(PtrTy, EltCount);
1112 }
1113 // Scalar GEP
1114 return PtrTy;
1115 }
1116
1117 unsigned getNumIndices() const { // Note: always non-negative
1118 return getNumOperands() - 1;
1119 }
1120
1121 bool hasIndices() const {
1122 return getNumOperands() > 1;
1123 }
1124
1125 /// Return true if all of the indices of this GEP are
1126 /// zeros. If so, the result pointer and the first operand have the same
1127 /// value, just potentially different types.
1128 bool hasAllZeroIndices() const;
1129
1130 /// Return true if all of the indices of this GEP are
1131 /// constant integers. If so, the result pointer and the first operand have
1132 /// a constant offset between them.
1133 bool hasAllConstantIndices() const;
1134
1135 /// Set or clear the inbounds flag on this GEP instruction.
1136 /// See LangRef.html for the meaning of inbounds on a getelementptr.
1137 void setIsInBounds(bool b = true);
1138
1139 /// Determine whether the GEP has the inbounds flag.
1140 bool isInBounds() const;
1141
1142 /// Accumulate the constant address offset of this GEP if possible.
1143 ///
1144 /// This routine accepts an APInt into which it will accumulate the constant
1145 /// offset of this GEP if the GEP is in fact constant. If the GEP is not
1146 /// all-constant, it returns false and the value of the offset APInt is
1147 /// undefined (it is *not* preserved!). The APInt passed into this routine
1148 /// must be at least as wide as the IntPtr type for the address space of
1149 /// the base GEP pointer.
1150 bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
1151
1152 // Methods for support type inquiry through isa, cast, and dyn_cast:
1153 static bool classof(const Instruction *I) {
1154 return (I->getOpcode() == Instruction::GetElementPtr);
1155 }
1156 static bool classof(const Value *V) {
1157 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1158 }
1159};
1160
1161template <>
1162struct OperandTraits<GetElementPtrInst> :
1163 public VariadicOperandTraits<GetElementPtrInst, 1> {
1164};
1165
1166GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1167 ArrayRef<Value *> IdxList, unsigned Values,
1168 const Twine &NameStr,
1169 Instruction *InsertBefore)
1170 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1171 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1172 Values, InsertBefore),
1173 SourceElementType(PointeeType),
1174 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1175 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1176, __extension__ __PRETTY_FUNCTION__))
1176 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1176, __extension__ __PRETTY_FUNCTION__))
;
1177 init(Ptr, IdxList, NameStr);
1178}
1179
1180GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1181 ArrayRef<Value *> IdxList, unsigned Values,
1182 const Twine &NameStr,
1183 BasicBlock *InsertAtEnd)
1184 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1185 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1186 Values, InsertAtEnd),
1187 SourceElementType(PointeeType),
1188 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1189 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1190, __extension__ __PRETTY_FUNCTION__))
1190 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1190, __extension__ __PRETTY_FUNCTION__))
;
1191 init(Ptr, IdxList, NameStr);
1192}
1193
1194DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)GetElementPtrInst::op_iterator GetElementPtrInst::op_begin() {
return OperandTraits<GetElementPtrInst>::op_begin(this
); } GetElementPtrInst::const_op_iterator GetElementPtrInst::
op_begin() const { return OperandTraits<GetElementPtrInst>
::op_begin(const_cast<GetElementPtrInst*>(this)); } GetElementPtrInst
::op_iterator GetElementPtrInst::op_end() { return OperandTraits
<GetElementPtrInst>::op_end(this); } GetElementPtrInst::
const_op_iterator GetElementPtrInst::op_end() const { return OperandTraits
<GetElementPtrInst>::op_end(const_cast<GetElementPtrInst
*>(this)); } Value *GetElementPtrInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<GetElementPtrInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1194, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<GetElementPtrInst>::op_begin
(const_cast<GetElementPtrInst*>(this))[i_nocapture].get
()); } void GetElementPtrInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<GetElementPtrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1194, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
GetElementPtrInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned GetElementPtrInst::getNumOperands() const { return
OperandTraits<GetElementPtrInst>::operands(this); } template
<int Idx_nocapture> Use &GetElementPtrInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &GetElementPtrInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1195
1196//===----------------------------------------------------------------------===//
1197// ICmpInst Class
1198//===----------------------------------------------------------------------===//
1199
1200/// This instruction compares its operands according to the predicate given
1201/// to the constructor. It only operates on integers or pointers. The operands
1202/// must be identical types.
1203/// Represent an integer comparison operator.
1204class ICmpInst: public CmpInst {
1205 void AssertOK() {
1206 assert(isIntPredicate() &&(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1207, __extension__ __PRETTY_FUNCTION__))
1207 "Invalid ICmp predicate value")(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1207, __extension__ __PRETTY_FUNCTION__))
;
1208 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1209, __extension__ __PRETTY_FUNCTION__))
1209 "Both operands to ICmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1209, __extension__ __PRETTY_FUNCTION__))
;
1210 // Check that the operands are the right type
1211 assert((getOperand(0)->getType()->isIntOrIntVectorTy() ||(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1213, __extension__ __PRETTY_FUNCTION__))
1212 getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1213, __extension__ __PRETTY_FUNCTION__))
1213 "Invalid operand types for ICmp instruction")(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1213, __extension__ __PRETTY_FUNCTION__))
;
1214 }
1215
1216protected:
1217 // Note: Instruction needs to be a friend here to call cloneImpl.
1218 friend class Instruction;
1219
1220 /// Clone an identical ICmpInst
1221 ICmpInst *cloneImpl() const;
1222
1223public:
1224 /// Constructor with insert-before-instruction semantics.
1225 ICmpInst(
1226 Instruction *InsertBefore, ///< Where to insert
1227 Predicate pred, ///< The predicate to use for the comparison
1228 Value *LHS, ///< The left-hand-side of the expression
1229 Value *RHS, ///< The right-hand-side of the expression
1230 const Twine &NameStr = "" ///< Name of the instruction
1231 ) : CmpInst(makeCmpResultType(LHS->getType()),
1232 Instruction::ICmp, pred, LHS, RHS, NameStr,
1233 InsertBefore) {
1234#ifndef NDEBUG
1235 AssertOK();
1236#endif
1237 }
1238
1239 /// Constructor with insert-at-end semantics.
1240 ICmpInst(
1241 BasicBlock &InsertAtEnd, ///< Block to insert into.
1242 Predicate pred, ///< The predicate to use for the comparison
1243 Value *LHS, ///< The left-hand-side of the expression
1244 Value *RHS, ///< The right-hand-side of the expression
1245 const Twine &NameStr = "" ///< Name of the instruction
1246 ) : CmpInst(makeCmpResultType(LHS->getType()),
1247 Instruction::ICmp, pred, LHS, RHS, NameStr,
1248 &InsertAtEnd) {
1249#ifndef NDEBUG
1250 AssertOK();
1251#endif
1252 }
1253
1254 /// Constructor with no-insertion semantics
1255 ICmpInst(
1256 Predicate pred, ///< The predicate to use for the comparison
1257 Value *LHS, ///< The left-hand-side of the expression
1258 Value *RHS, ///< The right-hand-side of the expression
1259 const Twine &NameStr = "" ///< Name of the instruction
1260 ) : CmpInst(makeCmpResultType(LHS->getType()),
1261 Instruction::ICmp, pred, LHS, RHS, NameStr) {
1262#ifndef NDEBUG
1263 AssertOK();
1264#endif
1265 }
1266
1267 /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
1268 /// @returns the predicate that would be the result if the operand were
1269 /// regarded as signed.
1270 /// Return the signed version of the predicate
1271 Predicate getSignedPredicate() const {
1272 return getSignedPredicate(getPredicate());
1273 }
1274
1275 /// This is a static version that you can use without an instruction.
1276 /// Return the signed version of the predicate.
1277 static Predicate getSignedPredicate(Predicate pred);
1278
1279 /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
1280 /// @returns the predicate that would be the result if the operand were
1281 /// regarded as unsigned.
1282 /// Return the unsigned version of the predicate
1283 Predicate getUnsignedPredicate() const {
1284 return getUnsignedPredicate(getPredicate());
1285 }
1286
1287 /// This is a static version that you can use without an instruction.
1288 /// Return the unsigned version of the predicate.
1289 static Predicate getUnsignedPredicate(Predicate pred);
1290
1291 /// Return true if this predicate is either EQ or NE. This also
1292 /// tests for commutativity.
1293 static bool isEquality(Predicate P) {
1294 return P == ICMP_EQ || P == ICMP_NE;
1295 }
1296
1297 /// Return true if this predicate is either EQ or NE. This also
1298 /// tests for commutativity.
1299 bool isEquality() const {
1300 return isEquality(getPredicate());
1301 }
1302
1303 /// @returns true if the predicate of this ICmpInst is commutative
1304 /// Determine if this relation is commutative.
1305 bool isCommutative() const { return isEquality(); }
1306
1307 /// Return true if the predicate is relational (not EQ or NE).
1308 ///
1309 bool isRelational() const {
1310 return !isEquality();
1311 }
1312
1313 /// Return true if the predicate is relational (not EQ or NE).
1314 ///
1315 static bool isRelational(Predicate P) {
1316 return !isEquality(P);
1317 }
1318
1319 /// Return true if the predicate is SGT or UGT.
1320 ///
1321 static bool isGT(Predicate P) {
1322 return P == ICMP_SGT || P == ICMP_UGT;
1323 }
1324
1325 /// Return true if the predicate is SLT or ULT.
1326 ///
1327 static bool isLT(Predicate P) {
1328 return P == ICMP_SLT || P == ICMP_ULT;
1329 }
1330
1331 /// Return true if the predicate is SGE or UGE.
1332 ///
1333 static bool isGE(Predicate P) {
1334 return P == ICMP_SGE || P == ICMP_UGE;
1335 }
1336
1337 /// Return true if the predicate is SLE or ULE.
1338 ///
1339 static bool isLE(Predicate P) {
1340 return P == ICMP_SLE || P == ICMP_ULE;
1341 }
1342
1343 /// Exchange the two operands to this instruction in such a way that it does
1344 /// not modify the semantics of the instruction. The predicate value may be
1345 /// changed to retain the same result if the predicate is order dependent
1346 /// (e.g. ult).
1347 /// Swap operands and adjust predicate.
1348 void swapOperands() {
1349 setPredicate(getSwappedPredicate());
1350 Op<0>().swap(Op<1>());
1351 }
1352
1353 // Methods for support type inquiry through isa, cast, and dyn_cast:
1354 static bool classof(const Instruction *I) {
1355 return I->getOpcode() == Instruction::ICmp;
1356 }
1357 static bool classof(const Value *V) {
1358 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1359 }
1360};
1361
1362//===----------------------------------------------------------------------===//
1363// FCmpInst Class
1364//===----------------------------------------------------------------------===//
1365
1366/// This instruction compares its operands according to the predicate given
1367/// to the constructor. It only operates on floating point values or packed
1368/// vectors of floating point values. The operands must be identical types.
1369/// Represents a floating point comparison operator.
1370class FCmpInst: public CmpInst {
1371 void AssertOK() {
1372 assert(isFPPredicate() && "Invalid FCmp predicate value")(static_cast <bool> (isFPPredicate() && "Invalid FCmp predicate value"
) ? void (0) : __assert_fail ("isFPPredicate() && \"Invalid FCmp predicate value\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1372, __extension__ __PRETTY_FUNCTION__))
;
1373 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1374, __extension__ __PRETTY_FUNCTION__))
1374 "Both operands to FCmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1374, __extension__ __PRETTY_FUNCTION__))
;
1375 // Check that the operands are the right type
1376 assert(getOperand(0)->getType()->isFPOrFPVectorTy() &&(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1377, __extension__ __PRETTY_FUNCTION__))
1377 "Invalid operand types for FCmp instruction")(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1377, __extension__ __PRETTY_FUNCTION__))
;
1378 }
1379
1380protected:
1381 // Note: Instruction needs to be a friend here to call cloneImpl.
1382 friend class Instruction;
1383
1384 /// Clone an identical FCmpInst
1385 FCmpInst *cloneImpl() const;
1386
1387public:
1388 /// Constructor with insert-before-instruction semantics.
1389 FCmpInst(
1390 Instruction *InsertBefore, ///< Where to insert
1391 Predicate pred, ///< The predicate to use for the comparison
1392 Value *LHS, ///< The left-hand-side of the expression
1393 Value *RHS, ///< The right-hand-side of the expression
1394 const Twine &NameStr = "" ///< Name of the instruction
1395 ) : CmpInst(makeCmpResultType(LHS->getType()),
1396 Instruction::FCmp, pred, LHS, RHS, NameStr,
1397 InsertBefore) {
1398 AssertOK();
1399 }
1400
1401 /// Constructor with insert-at-end semantics.
1402 FCmpInst(
1403 BasicBlock &InsertAtEnd, ///< Block to insert into.
1404 Predicate pred, ///< The predicate to use for the comparison
1405 Value *LHS, ///< The left-hand-side of the expression
1406 Value *RHS, ///< The right-hand-side of the expression
1407 const Twine &NameStr = "" ///< Name of the instruction
1408 ) : CmpInst(makeCmpResultType(LHS->getType()),
1409 Instruction::FCmp, pred, LHS, RHS, NameStr,
1410 &InsertAtEnd) {
1411 AssertOK();
1412 }
1413
1414 /// Constructor with no-insertion semantics
1415 FCmpInst(
1416 Predicate Pred, ///< The predicate to use for the comparison
1417 Value *LHS, ///< The left-hand-side of the expression
1418 Value *RHS, ///< The right-hand-side of the expression
1419 const Twine &NameStr = "", ///< Name of the instruction
1420 Instruction *FlagsSource = nullptr
1421 ) : CmpInst(makeCmpResultType(LHS->getType()), Instruction::FCmp, Pred, LHS,
1422 RHS, NameStr, nullptr, FlagsSource) {
1423 AssertOK();
1424 }
1425
1426 /// @returns true if the predicate of this instruction is EQ or NE.
1427 /// Determine if this is an equality predicate.
1428 static bool isEquality(Predicate Pred) {
1429 return Pred == FCMP_OEQ || Pred == FCMP_ONE || Pred == FCMP_UEQ ||
1430 Pred == FCMP_UNE;
1431 }
1432
1433 /// @returns true if the predicate of this instruction is EQ or NE.
1434 /// Determine if this is an equality predicate.
1435 bool isEquality() const { return isEquality(getPredicate()); }
1436
1437 /// @returns true if the predicate of this instruction is commutative.
1438 /// Determine if this is a commutative predicate.
1439 bool isCommutative() const {
1440 return isEquality() ||
1441 getPredicate() == FCMP_FALSE ||
1442 getPredicate() == FCMP_TRUE ||
1443 getPredicate() == FCMP_ORD ||
1444 getPredicate() == FCMP_UNO;
1445 }
1446
1447 /// @returns true if the predicate is relational (not EQ or NE).
1448 /// Determine if this a relational predicate.
1449 bool isRelational() const { return !isEquality(); }
1450
1451 /// Exchange the two operands to this instruction in such a way that it does
1452 /// not modify the semantics of the instruction. The predicate value may be
1453 /// changed to retain the same result if the predicate is order dependent
1454 /// (e.g. ult).
1455 /// Swap operands and adjust predicate.
1456 void swapOperands() {
1457 setPredicate(getSwappedPredicate());
1458 Op<0>().swap(Op<1>());
1459 }
1460
1461 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1462 static bool classof(const Instruction *I) {
1463 return I->getOpcode() == Instruction::FCmp;
1464 }
1465 static bool classof(const Value *V) {
1466 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1467 }
1468};
1469
1470//===----------------------------------------------------------------------===//
1471/// This class represents a function call, abstracting a target
1472/// machine's calling convention. This class uses low bit of the SubClassData
1473/// field to indicate whether or not this is a tail call. The rest of the bits
1474/// hold the calling convention of the call.
1475///
1476class CallInst : public CallBase {
1477 CallInst(const CallInst &CI);
1478
1479 /// Construct a CallInst given a range of arguments.
1480 /// Construct a CallInst from a range of arguments
1481 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1482 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1483 Instruction *InsertBefore);
1484
1485 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1486 const Twine &NameStr, Instruction *InsertBefore)
1487 : CallInst(Ty, Func, Args, None, NameStr, InsertBefore) {}
1488
1489 /// Construct a CallInst given a range of arguments.
1490 /// Construct a CallInst from a range of arguments
1491 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1492 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1493 BasicBlock *InsertAtEnd);
1494
1495 explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
1496 Instruction *InsertBefore);
1497
1498 CallInst(FunctionType *ty, Value *F, const Twine &NameStr,
1499 BasicBlock *InsertAtEnd);
1500
1501 void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
1502 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
1503 void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
1504
1505 /// Compute the number of operands to allocate.
1506 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
1507 // We need one operand for the called function, plus the input operand
1508 // counts provided.
1509 return 1 + NumArgs + NumBundleInputs;
1510 }
1511
1512protected:
1513 // Note: Instruction needs to be a friend here to call cloneImpl.
1514 friend class Instruction;
1515
1516 CallInst *cloneImpl() const;
1517
1518public:
1519 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
1520 Instruction *InsertBefore = nullptr) {
1521 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertBefore);
1522 }
1523
1524 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1525 const Twine &NameStr,
1526 Instruction *InsertBefore = nullptr) {
1527 return new (ComputeNumOperands(Args.size()))
1528 CallInst(Ty, Func, Args, None, NameStr, InsertBefore);
1529 }
1530
1531 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1532 ArrayRef<OperandBundleDef> Bundles = None,
1533 const Twine &NameStr = "",
1534 Instruction *InsertBefore = nullptr) {
1535 const int NumOperands =
1536 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1537 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1538
1539 return new (NumOperands, DescriptorBytes)
1540 CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
1541 }
1542
1543 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr,
1544 BasicBlock *InsertAtEnd) {
1545 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertAtEnd);
1546 }
1547
1548 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1549 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1550 return new (ComputeNumOperands(Args.size()))
1551 CallInst(Ty, Func, Args, None, NameStr, InsertAtEnd);
1552 }
1553
1554 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1555 ArrayRef<OperandBundleDef> Bundles,
1556 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1557 const int NumOperands =
1558 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1559 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1560
1561 return new (NumOperands, DescriptorBytes)
1562 CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd);
1563 }
1564
1565 static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "",
1566 Instruction *InsertBefore = nullptr) {
1567 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1568 InsertBefore);
1569 }
1570
1571 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1572 ArrayRef<OperandBundleDef> Bundles = None,
1573 const Twine &NameStr = "",
1574 Instruction *InsertBefore = nullptr) {
1575 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1576 NameStr, InsertBefore);
1577 }
1578
1579 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1580 const Twine &NameStr,
1581 Instruction *InsertBefore = nullptr) {
1582 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1583 InsertBefore);
1584 }
1585
1586 static CallInst *Create(FunctionCallee Func, const Twine &NameStr,
1587 BasicBlock *InsertAtEnd) {
1588 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1589 InsertAtEnd);
1590 }
1591
1592 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1593 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1594 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1595 InsertAtEnd);
1596 }
1597
1598 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1599 ArrayRef<OperandBundleDef> Bundles,
1600 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1601 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1602 NameStr, InsertAtEnd);
1603 }
1604
1605 /// Create a clone of \p CI with a different set of operand bundles and
1606 /// insert it before \p InsertPt.
1607 ///
1608 /// The returned call instruction is identical \p CI in every way except that
1609 /// the operand bundles for the new instruction are set to the operand bundles
1610 /// in \p Bundles.
1611 static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
1612 Instruction *InsertPt = nullptr);
1613
1614 /// Generate the IR for a call to malloc:
1615 /// 1. Compute the malloc call's argument as the specified type's size,
1616 /// possibly multiplied by the array size if the array size is not
1617 /// constant 1.
1618 /// 2. Call malloc with that argument.
1619 /// 3. Bitcast the result of the malloc call to the specified type.
1620 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1621 Type *AllocTy, Value *AllocSize,
1622 Value *ArraySize = nullptr,
1623 Function *MallocF = nullptr,
1624 const Twine &Name = "");
1625 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1626 Type *AllocTy, Value *AllocSize,
1627 Value *ArraySize = nullptr,
1628 Function *MallocF = nullptr,
1629 const Twine &Name = "");
1630 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1631 Type *AllocTy, Value *AllocSize,
1632 Value *ArraySize = nullptr,
1633 ArrayRef<OperandBundleDef> Bundles = None,
1634 Function *MallocF = nullptr,
1635 const Twine &Name = "");
1636 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1637 Type *AllocTy, Value *AllocSize,
1638 Value *ArraySize = nullptr,
1639 ArrayRef<OperandBundleDef> Bundles = None,
1640 Function *MallocF = nullptr,
1641 const Twine &Name = "");
1642 /// Generate the IR for a call to the builtin free function.
1643 static Instruction *CreateFree(Value *Source, Instruction *InsertBefore);
1644 static Instruction *CreateFree(Value *Source, BasicBlock *InsertAtEnd);
1645 static Instruction *CreateFree(Value *Source,
1646 ArrayRef<OperandBundleDef> Bundles,
1647 Instruction *InsertBefore);
1648 static Instruction *CreateFree(Value *Source,
1649 ArrayRef<OperandBundleDef> Bundles,
1650 BasicBlock *InsertAtEnd);
1651
1652 // Note that 'musttail' implies 'tail'.
1653 enum TailCallKind : unsigned {
1654 TCK_None = 0,
1655 TCK_Tail = 1,
1656 TCK_MustTail = 2,
1657 TCK_NoTail = 3,
1658 TCK_LAST = TCK_NoTail
1659 };
1660
1661 using TailCallKindField = Bitfield::Element<TailCallKind, 0, 2, TCK_LAST>;
1662 static_assert(
1663 Bitfield::areContiguous<TailCallKindField, CallBase::CallingConvField>(),
1664 "Bitfields must be contiguous");
1665
1666 TailCallKind getTailCallKind() const {
1667 return getSubclassData<TailCallKindField>();
1668 }
1669
1670 bool isTailCall() const {
1671 TailCallKind Kind = getTailCallKind();
1672 return Kind == TCK_Tail || Kind == TCK_MustTail;
1673 }
1674
1675 bool isMustTailCall() const { return getTailCallKind() == TCK_MustTail; }
1676
1677 bool isNoTailCall() const { return getTailCallKind() == TCK_NoTail; }
1678
1679 void setTailCallKind(TailCallKind TCK) {
1680 setSubclassData<TailCallKindField>(TCK);
1681 }
1682
1683 void setTailCall(bool IsTc = true) {
1684 setTailCallKind(IsTc ? TCK_Tail : TCK_None);
1685 }
1686
1687 /// Return true if the call can return twice
1688 bool canReturnTwice() const { return hasFnAttr(Attribute::ReturnsTwice); }
1689 void setCanReturnTwice() {
1690 addAttribute(AttributeList::FunctionIndex, Attribute::ReturnsTwice);
1691 }
1692
1693 // Methods for support type inquiry through isa, cast, and dyn_cast:
1694 static bool classof(const Instruction *I) {
1695 return I->getOpcode() == Instruction::Call;
1696 }
1697 static bool classof(const Value *V) {
1698 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1699 }
1700
1701 /// Updates profile metadata by scaling it by \p S / \p T.
1702 void updateProfWeight(uint64_t S, uint64_t T);
1703
1704private:
1705 // Shadow Instruction::setInstructionSubclassData with a private forwarding
1706 // method so that subclasses cannot accidentally use it.
1707 template <typename Bitfield>
1708 void setSubclassData(typename Bitfield::Type Value) {
1709 Instruction::setSubclassData<Bitfield>(Value);
1710 }
1711};
1712
1713CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1714 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1715 BasicBlock *InsertAtEnd)
1716 : CallBase(Ty->getReturnType(), Instruction::Call,
1717 OperandTraits<CallBase>::op_end(this) -
1718 (Args.size() + CountBundleInputs(Bundles) + 1),
1719 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1720 InsertAtEnd) {
1721 init(Ty, Func, Args, Bundles, NameStr);
1722}
1723
1724CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1725 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1726 Instruction *InsertBefore)
1727 : CallBase(Ty->getReturnType(), Instruction::Call,
1728 OperandTraits<CallBase>::op_end(this) -
1729 (Args.size() + CountBundleInputs(Bundles) + 1),
1730 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1731 InsertBefore) {
1732 init(Ty, Func, Args, Bundles, NameStr);
1733}
1734
1735//===----------------------------------------------------------------------===//
1736// SelectInst Class
1737//===----------------------------------------------------------------------===//
1738
1739/// This class represents the LLVM 'select' instruction.
1740///
1741class SelectInst : public Instruction {
1742 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1743 Instruction *InsertBefore)
1744 : Instruction(S1->getType(), Instruction::Select,
1745 &Op<0>(), 3, InsertBefore) {
1746 init(C, S1, S2);
1747 setName(NameStr);
1748 }
1749
1750 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1751 BasicBlock *InsertAtEnd)
1752 : Instruction(S1->getType(), Instruction::Select,
1753 &Op<0>(), 3, InsertAtEnd) {
1754 init(C, S1, S2);
1755 setName(NameStr);
1756 }
1757
1758 void init(Value *C, Value *S1, Value *S2) {
1759 assert(!areInvalidOperands(C, S1, S2) && "Invalid operands for select")(static_cast <bool> (!areInvalidOperands(C, S1, S2) &&
"Invalid operands for select") ? void (0) : __assert_fail ("!areInvalidOperands(C, S1, S2) && \"Invalid operands for select\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1759, __extension__ __PRETTY_FUNCTION__))
;
1760 Op<0>() = C;
1761 Op<1>() = S1;
1762 Op<2>() = S2;
1763 }
1764
1765protected:
1766 // Note: Instruction needs to be a friend here to call cloneImpl.
1767 friend class Instruction;
1768
1769 SelectInst *cloneImpl() const;
1770
1771public:
1772 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1773 const Twine &NameStr = "",
1774 Instruction *InsertBefore = nullptr,
1775 Instruction *MDFrom = nullptr) {
1776 SelectInst *Sel = new(3) SelectInst(C, S1, S2, NameStr, InsertBefore);
1777 if (MDFrom)
1778 Sel->copyMetadata(*MDFrom);
1779 return Sel;
1780 }
1781
1782 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1783 const Twine &NameStr,
1784 BasicBlock *InsertAtEnd) {
1785 return new(3) SelectInst(C, S1, S2, NameStr, InsertAtEnd);
1786 }
1787
1788 const Value *getCondition() const { return Op<0>(); }
1789 const Value *getTrueValue() const { return Op<1>(); }
1790 const Value *getFalseValue() const { return Op<2>(); }
1791 Value *getCondition() { return Op<0>(); }
1792 Value *getTrueValue() { return Op<1>(); }
1793 Value *getFalseValue() { return Op<2>(); }
1794
1795 void setCondition(Value *V) { Op<0>() = V; }
1796 void setTrueValue(Value *V) { Op<1>() = V; }
1797 void setFalseValue(Value *V) { Op<2>() = V; }
1798
1799 /// Swap the true and false values of the select instruction.
1800 /// This doesn't swap prof metadata.
1801 void swapValues() { Op<1>().swap(Op<2>()); }
1802
1803 /// Return a string if the specified operands are invalid
1804 /// for a select operation, otherwise return null.
1805 static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
1806
1807 /// Transparently provide more efficient getOperand methods.
1808 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1809
1810 OtherOps getOpcode() const {
1811 return static_cast<OtherOps>(Instruction::getOpcode());
1812 }
1813
1814 // Methods for support type inquiry through isa, cast, and dyn_cast:
1815 static bool classof(const Instruction *I) {
1816 return I->getOpcode() == Instruction::Select;
1817 }
1818 static bool classof(const Value *V) {
1819 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1820 }
1821};
1822
1823template <>
1824struct OperandTraits<SelectInst> : public FixedNumOperandTraits<SelectInst, 3> {
1825};
1826
1827DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)SelectInst::op_iterator SelectInst::op_begin() { return OperandTraits
<SelectInst>::op_begin(this); } SelectInst::const_op_iterator
SelectInst::op_begin() const { return OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this)); } SelectInst
::op_iterator SelectInst::op_end() { return OperandTraits<
SelectInst>::op_end(this); } SelectInst::const_op_iterator
SelectInst::op_end() const { return OperandTraits<SelectInst
>::op_end(const_cast<SelectInst*>(this)); } Value *SelectInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SelectInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1827, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<SelectInst>::op_begin(const_cast
<SelectInst*>(this))[i_nocapture].get()); } void SelectInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<SelectInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1827, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
SelectInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned SelectInst::getNumOperands() const { return OperandTraits
<SelectInst>::operands(this); } template <int Idx_nocapture
> Use &SelectInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
SelectInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
1828
1829//===----------------------------------------------------------------------===//
1830// VAArgInst Class
1831//===----------------------------------------------------------------------===//
1832
1833/// This class represents the va_arg llvm instruction, which returns
1834/// an argument of the specified type given a va_list and increments that list
1835///
1836class VAArgInst : public UnaryInstruction {
1837protected:
1838 // Note: Instruction needs to be a friend here to call cloneImpl.
1839 friend class Instruction;
1840
1841 VAArgInst *cloneImpl() const;
1842
1843public:
1844 VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
1845 Instruction *InsertBefore = nullptr)
1846 : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
1847 setName(NameStr);
1848 }
1849
1850 VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
1851 BasicBlock *InsertAtEnd)
1852 : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
1853 setName(NameStr);
1854 }
1855
1856 Value *getPointerOperand() { return getOperand(0); }
1857 const Value *getPointerOperand() const { return getOperand(0); }
1858 static unsigned getPointerOperandIndex() { return 0U; }
1859
1860 // Methods for support type inquiry through isa, cast, and dyn_cast:
1861 static bool classof(const Instruction *I) {
1862 return I->getOpcode() == VAArg;
1863 }
1864 static bool classof(const Value *V) {
1865 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1866 }
1867};
1868
1869//===----------------------------------------------------------------------===//
1870// ExtractElementInst Class
1871//===----------------------------------------------------------------------===//
1872
1873/// This instruction extracts a single (scalar)
1874/// element from a VectorType value
1875///
1876class ExtractElementInst : public Instruction {
1877 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
1878 Instruction *InsertBefore = nullptr);
1879 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
1880 BasicBlock *InsertAtEnd);
1881
1882protected:
1883 // Note: Instruction needs to be a friend here to call cloneImpl.
1884 friend class Instruction;
1885
1886 ExtractElementInst *cloneImpl() const;
1887
1888public:
1889 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1890 const Twine &NameStr = "",
1891 Instruction *InsertBefore = nullptr) {
1892 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertBefore);
1893 }
1894
1895 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1896 const Twine &NameStr,
1897 BasicBlock *InsertAtEnd) {
1898 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertAtEnd);
1899 }
1900
1901 /// Return true if an extractelement instruction can be
1902 /// formed with the specified operands.
1903 static bool isValidOperands(const Value *Vec, const Value *Idx);
1904
1905 Value *getVectorOperand() { return Op<0>(); }
1906 Value *getIndexOperand() { return Op<1>(); }
1907 const Value *getVectorOperand() const { return Op<0>(); }
1908 const Value *getIndexOperand() const { return Op<1>(); }
1909
1910 VectorType *getVectorOperandType() const {
1911 return cast<VectorType>(getVectorOperand()->getType());
1912 }
1913
1914 /// Transparently provide more efficient getOperand methods.
1915 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1916
1917 // Methods for support type inquiry through isa, cast, and dyn_cast:
1918 static bool classof(const Instruction *I) {
1919 return I->getOpcode() == Instruction::ExtractElement;
1920 }
1921 static bool classof(const Value *V) {
1922 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1923 }
1924};
1925
1926template <>
1927struct OperandTraits<ExtractElementInst> :
1928 public FixedNumOperandTraits<ExtractElementInst, 2> {
1929};
1930
1931DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)ExtractElementInst::op_iterator ExtractElementInst::op_begin(
) { return OperandTraits<ExtractElementInst>::op_begin(
this); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_begin() const { return OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this)); }
ExtractElementInst::op_iterator ExtractElementInst::op_end()
{ return OperandTraits<ExtractElementInst>::op_end(this
); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_end() const { return OperandTraits<ExtractElementInst
>::op_end(const_cast<ExtractElementInst*>(this)); } Value
*ExtractElementInst::getOperand(unsigned i_nocapture) const {
(static_cast <bool> (i_nocapture < OperandTraits<
ExtractElementInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1931, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ExtractElementInst>::op_begin
(const_cast<ExtractElementInst*>(this))[i_nocapture].get
()); } void ExtractElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ExtractElementInst>::operands(this)
&& "setOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1931, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ExtractElementInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned ExtractElementInst::getNumOperands() const { return
OperandTraits<ExtractElementInst>::operands(this); } template
<int Idx_nocapture> Use &ExtractElementInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ExtractElementInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1932
1933//===----------------------------------------------------------------------===//
1934// InsertElementInst Class
1935//===----------------------------------------------------------------------===//
1936
1937/// This instruction inserts a single (scalar)
1938/// element into a VectorType value
1939///
1940class InsertElementInst : public Instruction {
1941 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
1942 const Twine &NameStr = "",
1943 Instruction *InsertBefore = nullptr);
1944 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr,
1945 BasicBlock *InsertAtEnd);
1946
1947protected:
1948 // Note: Instruction needs to be a friend here to call cloneImpl.
1949 friend class Instruction;
1950
1951 InsertElementInst *cloneImpl() const;
1952
1953public:
1954 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1955 const Twine &NameStr = "",
1956 Instruction *InsertBefore = nullptr) {
1957 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertBefore);
1958 }
1959
1960 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1961 const Twine &NameStr,
1962 BasicBlock *InsertAtEnd) {
1963 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertAtEnd);
1964 }
1965
1966 /// Return true if an insertelement instruction can be
1967 /// formed with the specified operands.
1968 static bool isValidOperands(const Value *Vec, const Value *NewElt,
1969 const Value *Idx);
1970
1971 /// Overload to return most specific vector type.
1972 ///
1973 VectorType *getType() const {
1974 return cast<VectorType>(Instruction::getType());
1975 }
1976
1977 /// Transparently provide more efficient getOperand methods.
1978 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1979
1980 // Methods for support type inquiry through isa, cast, and dyn_cast:
1981 static bool classof(const Instruction *I) {
1982 return I->getOpcode() == Instruction::InsertElement;
1983 }
1984 static bool classof(const Value *V) {
1985 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1986 }
1987};
1988
1989template <>
1990struct OperandTraits<InsertElementInst> :
1991 public FixedNumOperandTraits<InsertElementInst, 3> {
1992};
1993
1994DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)InsertElementInst::op_iterator InsertElementInst::op_begin() {
return OperandTraits<InsertElementInst>::op_begin(this
); } InsertElementInst::const_op_iterator InsertElementInst::
op_begin() const { return OperandTraits<InsertElementInst>
::op_begin(const_cast<InsertElementInst*>(this)); } InsertElementInst
::op_iterator InsertElementInst::op_end() { return OperandTraits
<InsertElementInst>::op_end(this); } InsertElementInst::
const_op_iterator InsertElementInst::op_end() const { return OperandTraits
<InsertElementInst>::op_end(const_cast<InsertElementInst
*>(this)); } Value *InsertElementInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<InsertElementInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1994, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InsertElementInst>::op_begin
(const_cast<InsertElementInst*>(this))[i_nocapture].get
()); } void InsertElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertElementInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 1994, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InsertElementInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned InsertElementInst::getNumOperands() const { return
OperandTraits<InsertElementInst>::operands(this); } template
<int Idx_nocapture> Use &InsertElementInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &InsertElementInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1995
1996//===----------------------------------------------------------------------===//
1997// ShuffleVectorInst Class
1998//===----------------------------------------------------------------------===//
1999
2000constexpr int UndefMaskElem = -1;
2001
2002/// This instruction constructs a fixed permutation of two
2003/// input vectors.
2004///
2005/// For each element of the result vector, the shuffle mask selects an element
2006/// from one of the input vectors to copy to the result. Non-negative elements
2007/// in the mask represent an index into the concatenated pair of input vectors.
2008/// UndefMaskElem (-1) specifies that the result element is undefined.
2009///
2010/// For scalable vectors, all the elements of the mask must be 0 or -1. This
2011/// requirement may be relaxed in the future.
2012class ShuffleVectorInst : public Instruction {
2013 SmallVector<int, 4> ShuffleMask;
2014 Constant *ShuffleMaskForBitcode;
2015
2016protected:
2017 // Note: Instruction needs to be a friend here to call cloneImpl.
2018 friend class Instruction;
2019
2020 ShuffleVectorInst *cloneImpl() const;
2021
2022public:
2023 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2024 const Twine &NameStr = "",
2025 Instruction *InsertBefor = nullptr);
2026 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2027 const Twine &NameStr, BasicBlock *InsertAtEnd);
2028 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2029 const Twine &NameStr = "",
2030 Instruction *InsertBefor = nullptr);
2031 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2032 const Twine &NameStr, BasicBlock *InsertAtEnd);
2033
2034 void *operator new(size_t s) { return User::operator new(s, 2); }
2035
2036 /// Swap the operands and adjust the mask to preserve the semantics
2037 /// of the instruction.
2038 void commute();
2039
2040 /// Return true if a shufflevector instruction can be
2041 /// formed with the specified operands.
2042 static bool isValidOperands(const Value *V1, const Value *V2,
2043 const Value *Mask);
2044 static bool isValidOperands(const Value *V1, const Value *V2,
2045 ArrayRef<int> Mask);
2046
2047 /// Overload to return most specific vector type.
2048 ///
2049 VectorType *getType() const {
2050 return cast<VectorType>(Instruction::getType());
2051 }
2052
2053 /// Transparently provide more efficient getOperand methods.
2054 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2055
2056 /// Return the shuffle mask value of this instruction for the given element
2057 /// index. Return UndefMaskElem if the element is undef.
2058 int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
2059
2060 /// Convert the input shuffle mask operand to a vector of integers. Undefined
2061 /// elements of the mask are returned as UndefMaskElem.
2062 static void getShuffleMask(const Constant *Mask,
2063 SmallVectorImpl<int> &Result);
2064
2065 /// Return the mask for this instruction as a vector of integers. Undefined
2066 /// elements of the mask are returned as UndefMaskElem.
2067 void getShuffleMask(SmallVectorImpl<int> &Result) const {
2068 Result.assign(ShuffleMask.begin(), ShuffleMask.end());
2069 }
2070
2071 /// Return the mask for this instruction, for use in bitcode.
2072 ///
2073 /// TODO: This is temporary until we decide a new bitcode encoding for
2074 /// shufflevector.
2075 Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
2076
2077 static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
2078 Type *ResultTy);
2079
2080 void setShuffleMask(ArrayRef<int> Mask);
2081
2082 ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
2083
2084 /// Return true if this shuffle returns a vector with a different number of
2085 /// elements than its source vectors.
2086 /// Examples: shufflevector <4 x n> A, <4 x n> B, <1,2,3>
2087 /// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
2088 bool changesLength() const {
2089 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2090 ->getElementCount()
2091 .getKnownMinValue();
2092 unsigned NumMaskElts = ShuffleMask.size();
2093 return NumSourceElts != NumMaskElts;
2094 }
2095
2096 /// Return true if this shuffle returns a vector with a greater number of
2097 /// elements than its source vectors.
2098 /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
2099 bool increasesLength() const {
2100 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2101 ->getElementCount()
2102 .getKnownMinValue();
2103 unsigned NumMaskElts = ShuffleMask.size();
2104 return NumSourceElts < NumMaskElts;
2105 }
2106
2107 /// Return true if this shuffle mask chooses elements from exactly one source
2108 /// vector.
2109 /// Example: <7,5,undef,7>
2110 /// This assumes that vector operands are the same length as the mask.
2111 static bool isSingleSourceMask(ArrayRef<int> Mask);
2112 static bool isSingleSourceMask(const Constant *Mask) {
2113 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2113, __extension__ __PRETTY_FUNCTION__))
;
2114 SmallVector<int, 16> MaskAsInts;
2115 getShuffleMask(Mask, MaskAsInts);
2116 return isSingleSourceMask(MaskAsInts);
2117 }
2118
2119 /// Return true if this shuffle chooses elements from exactly one source
2120 /// vector without changing the length of that vector.
2121 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
2122 /// TODO: Optionally allow length-changing shuffles.
2123 bool isSingleSource() const {
2124 return !changesLength() && isSingleSourceMask(ShuffleMask);
2125 }
2126
2127 /// Return true if this shuffle mask chooses elements from exactly one source
2128 /// vector without lane crossings. A shuffle using this mask is not
2129 /// necessarily a no-op because it may change the number of elements from its
2130 /// input vectors or it may provide demanded bits knowledge via undef lanes.
2131 /// Example: <undef,undef,2,3>
2132 static bool isIdentityMask(ArrayRef<int> Mask);
2133 static bool isIdentityMask(const Constant *Mask) {
2134 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2134, __extension__ __PRETTY_FUNCTION__))
;
2135 SmallVector<int, 16> MaskAsInts;
2136 getShuffleMask(Mask, MaskAsInts);
2137 return isIdentityMask(MaskAsInts);
2138 }
2139
2140 /// Return true if this shuffle chooses elements from exactly one source
2141 /// vector without lane crossings and does not change the number of elements
2142 /// from its input vectors.
2143 /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
2144 bool isIdentity() const {
2145 return !changesLength() && isIdentityMask(ShuffleMask);
2146 }
2147
2148 /// Return true if this shuffle lengthens exactly one source vector with
2149 /// undefs in the high elements.
2150 bool isIdentityWithPadding() const;
2151
2152 /// Return true if this shuffle extracts the first N elements of exactly one
2153 /// source vector.
2154 bool isIdentityWithExtract() const;
2155
2156 /// Return true if this shuffle concatenates its 2 source vectors. This
2157 /// returns false if either input is undefined. In that case, the shuffle is
2158 /// is better classified as an identity with padding operation.
2159 bool isConcat() const;
2160
2161 /// Return true if this shuffle mask chooses elements from its source vectors
2162 /// without lane crossings. A shuffle using this mask would be
2163 /// equivalent to a vector select with a constant condition operand.
2164 /// Example: <4,1,6,undef>
2165 /// This returns false if the mask does not choose from both input vectors.
2166 /// In that case, the shuffle is better classified as an identity shuffle.
2167 /// This assumes that vector operands are the same length as the mask
2168 /// (a length-changing shuffle can never be equivalent to a vector select).
2169 static bool isSelectMask(ArrayRef<int> Mask);
2170 static bool isSelectMask(const Constant *Mask) {
2171 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2171, __extension__ __PRETTY_FUNCTION__))
;
2172 SmallVector<int, 16> MaskAsInts;
2173 getShuffleMask(Mask, MaskAsInts);
2174 return isSelectMask(MaskAsInts);
2175 }
2176
2177 /// Return true if this shuffle chooses elements from its source vectors
2178 /// without lane crossings and all operands have the same number of elements.
2179 /// In other words, this shuffle is equivalent to a vector select with a
2180 /// constant condition operand.
2181 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,1,6,3>
2182 /// This returns false if the mask does not choose from both input vectors.
2183 /// In that case, the shuffle is better classified as an identity shuffle.
2184 /// TODO: Optionally allow length-changing shuffles.
2185 bool isSelect() const {
2186 return !changesLength() && isSelectMask(ShuffleMask);
2187 }
2188
2189 /// Return true if this shuffle mask swaps the order of elements from exactly
2190 /// one source vector.
2191 /// Example: <7,6,undef,4>
2192 /// This assumes that vector operands are the same length as the mask.
2193 static bool isReverseMask(ArrayRef<int> Mask);
2194 static bool isReverseMask(const Constant *Mask) {
2195 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2195, __extension__ __PRETTY_FUNCTION__))
;
2196 SmallVector<int, 16> MaskAsInts;
2197 getShuffleMask(Mask, MaskAsInts);
2198 return isReverseMask(MaskAsInts);
2199 }
2200
2201 /// Return true if this shuffle swaps the order of elements from exactly
2202 /// one source vector.
2203 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
2204 /// TODO: Optionally allow length-changing shuffles.
2205 bool isReverse() const {
2206 return !changesLength() && isReverseMask(ShuffleMask);
2207 }
2208
2209 /// Return true if this shuffle mask chooses all elements with the same value
2210 /// as the first element of exactly one source vector.
2211 /// Example: <4,undef,undef,4>
2212 /// This assumes that vector operands are the same length as the mask.
2213 static bool isZeroEltSplatMask(ArrayRef<int> Mask);
2214 static bool isZeroEltSplatMask(const Constant *Mask) {
2215 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2215, __extension__ __PRETTY_FUNCTION__))
;
2216 SmallVector<int, 16> MaskAsInts;
2217 getShuffleMask(Mask, MaskAsInts);
2218 return isZeroEltSplatMask(MaskAsInts);
2219 }
2220
2221 /// Return true if all elements of this shuffle are the same value as the
2222 /// first element of exactly one source vector without changing the length
2223 /// of that vector.
2224 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,0,undef,0>
2225 /// TODO: Optionally allow length-changing shuffles.
2226 /// TODO: Optionally allow splats from other elements.
2227 bool isZeroEltSplat() const {
2228 return !changesLength() && isZeroEltSplatMask(ShuffleMask);
2229 }
2230
2231 /// Return true if this shuffle mask is a transpose mask.
2232 /// Transpose vector masks transpose a 2xn matrix. They read corresponding
2233 /// even- or odd-numbered vector elements from two n-dimensional source
2234 /// vectors and write each result into consecutive elements of an
2235 /// n-dimensional destination vector. Two shuffles are necessary to complete
2236 /// the transpose, one for the even elements and another for the odd elements.
2237 /// This description closely follows how the TRN1 and TRN2 AArch64
2238 /// instructions operate.
2239 ///
2240 /// For example, a simple 2x2 matrix can be transposed with:
2241 ///
2242 /// ; Original matrix
2243 /// m0 = < a, b >
2244 /// m1 = < c, d >
2245 ///
2246 /// ; Transposed matrix
2247 /// t0 = < a, c > = shufflevector m0, m1, < 0, 2 >
2248 /// t1 = < b, d > = shufflevector m0, m1, < 1, 3 >
2249 ///
2250 /// For matrices having greater than n columns, the resulting nx2 transposed
2251 /// matrix is stored in two result vectors such that one vector contains
2252 /// interleaved elements from all the even-numbered rows and the other vector
2253 /// contains interleaved elements from all the odd-numbered rows. For example,
2254 /// a 2x4 matrix can be transposed with:
2255 ///
2256 /// ; Original matrix
2257 /// m0 = < a, b, c, d >
2258 /// m1 = < e, f, g, h >
2259 ///
2260 /// ; Transposed matrix
2261 /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
2262 /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
2263 static bool isTransposeMask(ArrayRef<int> Mask);
2264 static bool isTransposeMask(const Constant *Mask) {
2265 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2265, __extension__ __PRETTY_FUNCTION__))
;
2266 SmallVector<int, 16> MaskAsInts;
2267 getShuffleMask(Mask, MaskAsInts);
2268 return isTransposeMask(MaskAsInts);
2269 }
2270
2271 /// Return true if this shuffle transposes the elements of its inputs without
2272 /// changing the length of the vectors. This operation may also be known as a
2273 /// merge or interleave. See the description for isTransposeMask() for the
2274 /// exact specification.
2275 /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
2276 bool isTranspose() const {
2277 return !changesLength() && isTransposeMask(ShuffleMask);
2278 }
2279
2280 /// Return true if this shuffle mask is an extract subvector mask.
2281 /// A valid extract subvector mask returns a smaller vector from a single
2282 /// source operand. The base extraction index is returned as well.
2283 static bool isExtractSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2284 int &Index);
2285 static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
2286 int &Index) {
2287 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2287, __extension__ __PRETTY_FUNCTION__))
;
2288 // Not possible to express a shuffle mask for a scalable vector for this
2289 // case.
2290 if (isa<ScalableVectorType>(Mask->getType()))
2291 return false;
2292 SmallVector<int, 16> MaskAsInts;
2293 getShuffleMask(Mask, MaskAsInts);
2294 return isExtractSubvectorMask(MaskAsInts, NumSrcElts, Index);
2295 }
2296
2297 /// Return true if this shuffle mask is an extract subvector mask.
2298 bool isExtractSubvectorMask(int &Index) const {
2299 // Not possible to express a shuffle mask for a scalable vector for this
2300 // case.
2301 if (isa<ScalableVectorType>(getType()))
2302 return false;
2303
2304 int NumSrcElts =
2305 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2306 return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
2307 }
2308
2309 /// Change values in a shuffle permute mask assuming the two vector operands
2310 /// of length InVecNumElts have swapped position.
2311 static void commuteShuffleMask(MutableArrayRef<int> Mask,
2312 unsigned InVecNumElts) {
2313 for (int &Idx : Mask) {
2314 if (Idx == -1)
2315 continue;
2316 Idx = Idx < (int)InVecNumElts ? Idx + InVecNumElts : Idx - InVecNumElts;
2317 assert(Idx >= 0 && Idx < (int)InVecNumElts * 2 &&(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2318, __extension__ __PRETTY_FUNCTION__))
2318 "shufflevector mask index out of range")(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2318, __extension__ __PRETTY_FUNCTION__))
;
2319 }
2320 }
2321
2322 // Methods for support type inquiry through isa, cast, and dyn_cast:
2323 static bool classof(const Instruction *I) {
2324 return I->getOpcode() == Instruction::ShuffleVector;
2325 }
2326 static bool classof(const Value *V) {
2327 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2328 }
2329};
2330
2331template <>
2332struct OperandTraits<ShuffleVectorInst>
2333 : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
2334
2335DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)ShuffleVectorInst::op_iterator ShuffleVectorInst::op_begin() {
return OperandTraits<ShuffleVectorInst>::op_begin(this
); } ShuffleVectorInst::const_op_iterator ShuffleVectorInst::
op_begin() const { return OperandTraits<ShuffleVectorInst>
::op_begin(const_cast<ShuffleVectorInst*>(this)); } ShuffleVectorInst
::op_iterator ShuffleVectorInst::op_end() { return OperandTraits
<ShuffleVectorInst>::op_end(this); } ShuffleVectorInst::
const_op_iterator ShuffleVectorInst::op_end() const { return OperandTraits
<ShuffleVectorInst>::op_end(const_cast<ShuffleVectorInst
*>(this)); } Value *ShuffleVectorInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<ShuffleVectorInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2335, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ShuffleVectorInst>::op_begin
(const_cast<ShuffleVectorInst*>(this))[i_nocapture].get
()); } void ShuffleVectorInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ShuffleVectorInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2335, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ShuffleVectorInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned ShuffleVectorInst::getNumOperands() const { return
OperandTraits<ShuffleVectorInst>::operands(this); } template
<int Idx_nocapture> Use &ShuffleVectorInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ShuffleVectorInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2336
2337//===----------------------------------------------------------------------===//
2338// ExtractValueInst Class
2339//===----------------------------------------------------------------------===//
2340
2341/// This instruction extracts a struct member or array
2342/// element value from an aggregate value.
2343///
2344class ExtractValueInst : public UnaryInstruction {
2345 SmallVector<unsigned, 4> Indices;
2346
2347 ExtractValueInst(const ExtractValueInst &EVI);
2348
2349 /// Constructors - Create a extractvalue instruction with a base aggregate
2350 /// value and a list of indices. The first ctor can optionally insert before
2351 /// an existing instruction, the second appends the new instruction to the
2352 /// specified BasicBlock.
2353 inline ExtractValueInst(Value *Agg,
2354 ArrayRef<unsigned> Idxs,
2355 const Twine &NameStr,
2356 Instruction *InsertBefore);
2357 inline ExtractValueInst(Value *Agg,
2358 ArrayRef<unsigned> Idxs,
2359 const Twine &NameStr, BasicBlock *InsertAtEnd);
2360
2361 void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
2362
2363protected:
2364 // Note: Instruction needs to be a friend here to call cloneImpl.
2365 friend class Instruction;
2366
2367 ExtractValueInst *cloneImpl() const;
2368
2369public:
2370 static ExtractValueInst *Create(Value *Agg,
2371 ArrayRef<unsigned> Idxs,
2372 const Twine &NameStr = "",
2373 Instruction *InsertBefore = nullptr) {
2374 return new
2375 ExtractValueInst(Agg, Idxs, NameStr, InsertBefore);
2376 }
2377
2378 static ExtractValueInst *Create(Value *Agg,
2379 ArrayRef<unsigned> Idxs,
2380 const Twine &NameStr,
2381 BasicBlock *InsertAtEnd) {
2382 return new ExtractValueInst(Agg, Idxs, NameStr, InsertAtEnd);
2383 }
2384
2385 /// Returns the type of the element that would be extracted
2386 /// with an extractvalue instruction with the specified parameters.
2387 ///
2388 /// Null is returned if the indices are invalid for the specified type.
2389 static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
2390
2391 using idx_iterator = const unsigned*;
2392
2393 inline idx_iterator idx_begin() const { return Indices.begin(); }
2394 inline idx_iterator idx_end() const { return Indices.end(); }
2395 inline iterator_range<idx_iterator> indices() const {
2396 return make_range(idx_begin(), idx_end());
2397 }
2398
2399 Value *getAggregateOperand() {
2400 return getOperand(0);
2401 }
2402 const Value *getAggregateOperand() const {
2403 return getOperand(0);
2404 }
2405 static unsigned getAggregateOperandIndex() {
2406 return 0U; // get index for modifying correct operand
2407 }
2408
2409 ArrayRef<unsigned> getIndices() const {
2410 return Indices;
2411 }
2412
2413 unsigned getNumIndices() const {
2414 return (unsigned)Indices.size();
2415 }
2416
2417 bool hasIndices() const {
2418 return true;
2419 }
2420
2421 // Methods for support type inquiry through isa, cast, and dyn_cast:
2422 static bool classof(const Instruction *I) {
2423 return I->getOpcode() == Instruction::ExtractValue;
2424 }
2425 static bool classof(const Value *V) {
2426 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2427 }
2428};
2429
2430ExtractValueInst::ExtractValueInst(Value *Agg,
2431 ArrayRef<unsigned> Idxs,
2432 const Twine &NameStr,
2433 Instruction *InsertBefore)
2434 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2435 ExtractValue, Agg, InsertBefore) {
2436 init(Idxs, NameStr);
2437}
2438
2439ExtractValueInst::ExtractValueInst(Value *Agg,
2440 ArrayRef<unsigned> Idxs,
2441 const Twine &NameStr,
2442 BasicBlock *InsertAtEnd)
2443 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2444 ExtractValue, Agg, InsertAtEnd) {
2445 init(Idxs, NameStr);
2446}
2447
2448//===----------------------------------------------------------------------===//
2449// InsertValueInst Class
2450//===----------------------------------------------------------------------===//
2451
2452/// This instruction inserts a struct field of array element
2453/// value into an aggregate value.
2454///
2455class InsertValueInst : public Instruction {
2456 SmallVector<unsigned, 4> Indices;
2457
2458 InsertValueInst(const InsertValueInst &IVI);
2459
2460 /// Constructors - Create a insertvalue instruction with a base aggregate
2461 /// value, a value to insert, and a list of indices. The first ctor can
2462 /// optionally insert before an existing instruction, the second appends
2463 /// the new instruction to the specified BasicBlock.
2464 inline InsertValueInst(Value *Agg, Value *Val,
2465 ArrayRef<unsigned> Idxs,
2466 const Twine &NameStr,
2467 Instruction *InsertBefore);
2468 inline InsertValueInst(Value *Agg, Value *Val,
2469 ArrayRef<unsigned> Idxs,
2470 const Twine &NameStr, BasicBlock *InsertAtEnd);
2471
2472 /// Constructors - These two constructors are convenience methods because one
2473 /// and two index insertvalue instructions are so common.
2474 InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
2475 const Twine &NameStr = "",
2476 Instruction *InsertBefore = nullptr);
2477 InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr,
2478 BasicBlock *InsertAtEnd);
2479
2480 void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2481 const Twine &NameStr);
2482
2483protected:
2484 // Note: Instruction needs to be a friend here to call cloneImpl.
2485 friend class Instruction;
2486
2487 InsertValueInst *cloneImpl() const;
2488
2489public:
2490 // allocate space for exactly two operands
2491 void *operator new(size_t s) {
2492 return User::operator new(s, 2);
2493 }
2494
2495 static InsertValueInst *Create(Value *Agg, Value *Val,
2496 ArrayRef<unsigned> Idxs,
2497 const Twine &NameStr = "",
2498 Instruction *InsertBefore = nullptr) {
2499 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertBefore);
2500 }
2501
2502 static InsertValueInst *Create(Value *Agg, Value *Val,
2503 ArrayRef<unsigned> Idxs,
2504 const Twine &NameStr,
2505 BasicBlock *InsertAtEnd) {
2506 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertAtEnd);
2507 }
2508
2509 /// Transparently provide more efficient getOperand methods.
2510 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2511
2512 using idx_iterator = const unsigned*;
2513
2514 inline idx_iterator idx_begin() const { return Indices.begin(); }
2515 inline idx_iterator idx_end() const { return Indices.end(); }
2516 inline iterator_range<idx_iterator> indices() const {
2517 return make_range(idx_begin(), idx_end());
2518 }
2519
2520 Value *getAggregateOperand() {
2521 return getOperand(0);
2522 }
2523 const Value *getAggregateOperand() const {
2524 return getOperand(0);
2525 }
2526 static unsigned getAggregateOperandIndex() {
2527 return 0U; // get index for modifying correct operand
2528 }
2529
2530 Value *getInsertedValueOperand() {
2531 return getOperand(1);
2532 }
2533 const Value *getInsertedValueOperand() const {
2534 return getOperand(1);
2535 }
2536 static unsigned getInsertedValueOperandIndex() {
2537 return 1U; // get index for modifying correct operand
2538 }
2539
2540 ArrayRef<unsigned> getIndices() const {
2541 return Indices;
2542 }
2543
2544 unsigned getNumIndices() const {
2545 return (unsigned)Indices.size();
2546 }
2547
2548 bool hasIndices() const {
2549 return true;
2550 }
2551
2552 // Methods for support type inquiry through isa, cast, and dyn_cast:
2553 static bool classof(const Instruction *I) {
2554 return I->getOpcode() == Instruction::InsertValue;
2555 }
2556 static bool classof(const Value *V) {
2557 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2558 }
2559};
2560
2561template <>
2562struct OperandTraits<InsertValueInst> :
2563 public FixedNumOperandTraits<InsertValueInst, 2> {
2564};
2565
2566InsertValueInst::InsertValueInst(Value *Agg,
2567 Value *Val,
2568 ArrayRef<unsigned> Idxs,
2569 const Twine &NameStr,
2570 Instruction *InsertBefore)
2571 : Instruction(Agg->getType(), InsertValue,
2572 OperandTraits<InsertValueInst>::op_begin(this),
2573 2, InsertBefore) {
2574 init(Agg, Val, Idxs, NameStr);
2575}
2576
2577InsertValueInst::InsertValueInst(Value *Agg,
2578 Value *Val,
2579 ArrayRef<unsigned> Idxs,
2580 const Twine &NameStr,
2581 BasicBlock *InsertAtEnd)
2582 : Instruction(Agg->getType(), InsertValue,
2583 OperandTraits<InsertValueInst>::op_begin(this),
2584 2, InsertAtEnd) {
2585 init(Agg, Val, Idxs, NameStr);
2586}
2587
2588DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)InsertValueInst::op_iterator InsertValueInst::op_begin() { return
OperandTraits<InsertValueInst>::op_begin(this); } InsertValueInst
::const_op_iterator InsertValueInst::op_begin() const { return
OperandTraits<InsertValueInst>::op_begin(const_cast<
InsertValueInst*>(this)); } InsertValueInst::op_iterator InsertValueInst
::op_end() { return OperandTraits<InsertValueInst>::op_end
(this); } InsertValueInst::const_op_iterator InsertValueInst::
op_end() const { return OperandTraits<InsertValueInst>::
op_end(const_cast<InsertValueInst*>(this)); } Value *InsertValueInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<InsertValueInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2588, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InsertValueInst>::op_begin
(const_cast<InsertValueInst*>(this))[i_nocapture].get()
); } void InsertValueInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<InsertValueInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2588, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InsertValueInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned InsertValueInst::getNumOperands() const { return
OperandTraits<InsertValueInst>::operands(this); } template
<int Idx_nocapture> Use &InsertValueInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &InsertValueInst::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
2589
2590//===----------------------------------------------------------------------===//
2591// PHINode Class
2592//===----------------------------------------------------------------------===//
2593
2594// PHINode - The PHINode class is used to represent the magical mystical PHI
2595// node, that can not exist in nature, but can be synthesized in a computer
2596// scientist's overactive imagination.
2597//
2598class PHINode : public Instruction {
2599 /// The number of operands actually allocated. NumOperands is
2600 /// the number actually in use.
2601 unsigned ReservedSpace;
2602
2603 PHINode(const PHINode &PN);
2604
2605 explicit PHINode(Type *Ty, unsigned NumReservedValues,
2606 const Twine &NameStr = "",
2607 Instruction *InsertBefore = nullptr)
2608 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
2609 ReservedSpace(NumReservedValues) {
2610 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2610, __extension__ __PRETTY_FUNCTION__))
;
2611 setName(NameStr);
2612 allocHungoffUses(ReservedSpace);
2613 }
2614
2615 PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
2616 BasicBlock *InsertAtEnd)
2617 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
2618 ReservedSpace(NumReservedValues) {
2619 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2619, __extension__ __PRETTY_FUNCTION__))
;
2620 setName(NameStr);
2621 allocHungoffUses(ReservedSpace);
2622 }
2623
2624protected:
2625 // Note: Instruction needs to be a friend here to call cloneImpl.
2626 friend class Instruction;
2627
2628 PHINode *cloneImpl() const;
2629
2630 // allocHungoffUses - this is more complicated than the generic
2631 // User::allocHungoffUses, because we have to allocate Uses for the incoming
2632 // values and pointers to the incoming blocks, all in one allocation.
2633 void allocHungoffUses(unsigned N) {
2634 User::allocHungoffUses(N, /* IsPhi */ true);
2635 }
2636
2637public:
2638 /// Constructors - NumReservedValues is a hint for the number of incoming
2639 /// edges that this phi node will have (use 0 if you really have no idea).
2640 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2641 const Twine &NameStr = "",
2642 Instruction *InsertBefore = nullptr) {
2643 return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
2644 }
2645
2646 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2647 const Twine &NameStr, BasicBlock *InsertAtEnd) {
2648 return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
2649 }
2650
2651 /// Provide fast operand accessors
2652 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2653
2654 // Block iterator interface. This provides access to the list of incoming
2655 // basic blocks, which parallels the list of incoming values.
2656
2657 using block_iterator = BasicBlock **;
2658 using const_block_iterator = BasicBlock * const *;
2659
2660 block_iterator block_begin() {
2661 return reinterpret_cast<block_iterator>(op_begin() + ReservedSpace);
2662 }
2663
2664 const_block_iterator block_begin() const {
2665 return reinterpret_cast<const_block_iterator>(op_begin() + ReservedSpace);
2666 }
2667
2668 block_iterator block_end() {
2669 return block_begin() + getNumOperands();
2670 }
2671
2672 const_block_iterator block_end() const {
2673 return block_begin() + getNumOperands();
2674 }
2675
2676 iterator_range<block_iterator> blocks() {
2677 return make_range(block_begin(), block_end());
2678 }
2679
2680 iterator_range<const_block_iterator> blocks() const {
2681 return make_range(block_begin(), block_end());
2682 }
2683
2684 op_range incoming_values() { return operands(); }
2685
2686 const_op_range incoming_values() const { return operands(); }
2687
2688 /// Return the number of incoming edges
2689 ///
2690 unsigned getNumIncomingValues() const { return getNumOperands(); }
2691
2692 /// Return incoming value number x
2693 ///
2694 Value *getIncomingValue(unsigned i) const {
2695 return getOperand(i);
2696 }
2697 void setIncomingValue(unsigned i, Value *V) {
2698 assert(V && "PHI node got a null value!")(static_cast <bool> (V && "PHI node got a null value!"
) ? void (0) : __assert_fail ("V && \"PHI node got a null value!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2698, __extension__ __PRETTY_FUNCTION__))
;
2699 assert(getType() == V->getType() &&(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2700, __extension__ __PRETTY_FUNCTION__))
2700 "All operands to PHI node must be the same type as the PHI node!")(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2700, __extension__ __PRETTY_FUNCTION__))
;
2701 setOperand(i, V);
2702 }
2703
2704 static unsigned getOperandNumForIncomingValue(unsigned i) {
2705 return i;
2706 }
2707
2708 static unsigned getIncomingValueNumForOperand(unsigned i) {
2709 return i;
2710 }
2711
2712 /// Return incoming basic block number @p i.
2713 ///
2714 BasicBlock *getIncomingBlock(unsigned i) const {
2715 return block_begin()[i];
2716 }
2717
2718 /// Return incoming basic block corresponding
2719 /// to an operand of the PHI.
2720 ///
2721 BasicBlock *getIncomingBlock(const Use &U) const {
2722 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?")(static_cast <bool> (this == U.getUser() && "Iterator doesn't point to PHI's Uses?"
) ? void (0) : __assert_fail ("this == U.getUser() && \"Iterator doesn't point to PHI's Uses?\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2722, __extension__ __PRETTY_FUNCTION__))
;
2723 return getIncomingBlock(unsigned(&U - op_begin()));
2724 }
2725
2726 /// Return incoming basic block corresponding
2727 /// to value use iterator.
2728 ///
2729 BasicBlock *getIncomingBlock(Value::const_user_iterator I) const {
2730 return getIncomingBlock(I.getUse());
2731 }
2732
2733 void setIncomingBlock(unsigned i, BasicBlock *BB) {
2734 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2734, __extension__ __PRETTY_FUNCTION__))
;
2735 block_begin()[i] = BB;
2736 }
2737
2738 /// Replace every incoming basic block \p Old to basic block \p New.
2739 void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
2740 assert(New && Old && "PHI node got a null basic block!")(static_cast <bool> (New && Old && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("New && Old && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2740, __extension__ __PRETTY_FUNCTION__))
;
2741 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2742 if (getIncomingBlock(Op) == Old)
2743 setIncomingBlock(Op, New);
2744 }
2745
2746 /// Add an incoming value to the end of the PHI list
2747 ///
2748 void addIncoming(Value *V, BasicBlock *BB) {
2749 if (getNumOperands() == ReservedSpace)
2750 growOperands(); // Get more space!
2751 // Initialize some new operands.
2752 setNumHungOffUseOperands(getNumOperands() + 1);
2753 setIncomingValue(getNumOperands() - 1, V);
2754 setIncomingBlock(getNumOperands() - 1, BB);
2755 }
2756
2757 /// Remove an incoming value. This is useful if a
2758 /// predecessor basic block is deleted. The value removed is returned.
2759 ///
2760 /// If the last incoming value for a PHI node is removed (and DeletePHIIfEmpty
2761 /// is true), the PHI node is destroyed and any uses of it are replaced with
2762 /// dummy values. The only time there should be zero incoming values to a PHI
2763 /// node is when the block is dead, so this strategy is sound.
2764 ///
2765 Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
2766
2767 Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
2768 int Idx = getBasicBlockIndex(BB);
2769 assert(Idx >= 0 && "Invalid basic block argument to remove!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument to remove!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument to remove!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2769, __extension__ __PRETTY_FUNCTION__))
;
2770 return removeIncomingValue(Idx, DeletePHIIfEmpty);
2771 }
2772
2773 /// Return the first index of the specified basic
2774 /// block in the value list for this PHI. Returns -1 if no instance.
2775 ///
2776 int getBasicBlockIndex(const BasicBlock *BB) const {
2777 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
2778 if (block_begin()[i] == BB)
2779 return i;
2780 return -1;
2781 }
2782
2783 Value *getIncomingValueForBlock(const BasicBlock *BB) const {
2784 int Idx = getBasicBlockIndex(BB);
2785 assert(Idx >= 0 && "Invalid basic block argument!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2785, __extension__ __PRETTY_FUNCTION__))
;
2786 return getIncomingValue(Idx);
2787 }
2788
2789 /// Set every incoming value(s) for block \p BB to \p V.
2790 void setIncomingValueForBlock(const BasicBlock *BB, Value *V) {
2791 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2791, __extension__ __PRETTY_FUNCTION__))
;
2792 bool Found = false;
2793 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2794 if (getIncomingBlock(Op) == BB) {
2795 Found = true;
2796 setIncomingValue(Op, V);
2797 }
2798 (void)Found;
2799 assert(Found && "Invalid basic block argument to set!")(static_cast <bool> (Found && "Invalid basic block argument to set!"
) ? void (0) : __assert_fail ("Found && \"Invalid basic block argument to set!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2799, __extension__ __PRETTY_FUNCTION__))
;
2800 }
2801
2802 /// If the specified PHI node always merges together the
2803 /// same value, return the value, otherwise return null.
2804 Value *hasConstantValue() const;
2805
2806 /// Whether the specified PHI node always merges
2807 /// together the same value, assuming undefs are equal to a unique
2808 /// non-undef value.
2809 bool hasConstantOrUndefValue() const;
2810
2811 /// If the PHI node is complete which means all of its parent's predecessors
2812 /// have incoming value in this PHI, return true, otherwise return false.
2813 bool isComplete() const {
2814 return llvm::all_of(predecessors(getParent()),
2815 [this](const BasicBlock *Pred) {
2816 return getBasicBlockIndex(Pred) >= 0;
2817 });
2818 }
2819
2820 /// Methods for support type inquiry through isa, cast, and dyn_cast:
2821 static bool classof(const Instruction *I) {
2822 return I->getOpcode() == Instruction::PHI;
2823 }
2824 static bool classof(const Value *V) {
2825 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2826 }
2827
2828private:
2829 void growOperands();
2830};
2831
2832template <>
2833struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
2834};
2835
2836DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)PHINode::op_iterator PHINode::op_begin() { return OperandTraits
<PHINode>::op_begin(this); } PHINode::const_op_iterator
PHINode::op_begin() const { return OperandTraits<PHINode>
::op_begin(const_cast<PHINode*>(this)); } PHINode::op_iterator
PHINode::op_end() { return OperandTraits<PHINode>::op_end
(this); } PHINode::const_op_iterator PHINode::op_end() const {
return OperandTraits<PHINode>::op_end(const_cast<PHINode
*>(this)); } Value *PHINode::getOperand(unsigned i_nocapture
) const { (static_cast <bool> (i_nocapture < OperandTraits
<PHINode>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2836, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<PHINode>::op_begin(const_cast
<PHINode*>(this))[i_nocapture].get()); } void PHINode::
setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<PHINode>::
operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2836, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
PHINode>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
PHINode::getNumOperands() const { return OperandTraits<PHINode
>::operands(this); } template <int Idx_nocapture> Use
&PHINode::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
PHINode::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
2837
2838//===----------------------------------------------------------------------===//
2839// LandingPadInst Class
2840//===----------------------------------------------------------------------===//
2841
2842//===---------------------------------------------------------------------------
2843/// The landingpad instruction holds all of the information
2844/// necessary to generate correct exception handling. The landingpad instruction
2845/// cannot be moved from the top of a landing pad block, which itself is
2846/// accessible only from the 'unwind' edge of an invoke. This uses the
2847/// SubclassData field in Value to store whether or not the landingpad is a
2848/// cleanup.
2849///
2850class LandingPadInst : public Instruction {
2851 using CleanupField = BoolBitfieldElementT<0>;
2852
2853 /// The number of operands actually allocated. NumOperands is
2854 /// the number actually in use.
2855 unsigned ReservedSpace;
2856
2857 LandingPadInst(const LandingPadInst &LP);
2858
2859public:
2860 enum ClauseType { Catch, Filter };
2861
2862private:
2863 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2864 const Twine &NameStr, Instruction *InsertBefore);
2865 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2866 const Twine &NameStr, BasicBlock *InsertAtEnd);
2867
2868 // Allocate space for exactly zero operands.
2869 void *operator new(size_t s) {
2870 return User::operator new(s);
2871 }
2872
2873 void growOperands(unsigned Size);
2874 void init(unsigned NumReservedValues, const Twine &NameStr);
2875
2876protected:
2877 // Note: Instruction needs to be a friend here to call cloneImpl.
2878 friend class Instruction;
2879
2880 LandingPadInst *cloneImpl() const;
2881
2882public:
2883 /// Constructors - NumReservedClauses is a hint for the number of incoming
2884 /// clauses that this landingpad will have (use 0 if you really have no idea).
2885 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2886 const Twine &NameStr = "",
2887 Instruction *InsertBefore = nullptr);
2888 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2889 const Twine &NameStr, BasicBlock *InsertAtEnd);
2890
2891 /// Provide fast operand accessors
2892 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2893
2894 /// Return 'true' if this landingpad instruction is a
2895 /// cleanup. I.e., it should be run when unwinding even if its landing pad
2896 /// doesn't catch the exception.
2897 bool isCleanup() const { return getSubclassData<CleanupField>(); }
2898
2899 /// Indicate that this landingpad instruction is a cleanup.
2900 void setCleanup(bool V) { setSubclassData<CleanupField>(V); }
2901
2902 /// Add a catch or filter clause to the landing pad.
2903 void addClause(Constant *ClauseVal);
2904
2905 /// Get the value of the clause at index Idx. Use isCatch/isFilter to
2906 /// determine what type of clause this is.
2907 Constant *getClause(unsigned Idx) const {
2908 return cast<Constant>(getOperandList()[Idx]);
2909 }
2910
2911 /// Return 'true' if the clause and index Idx is a catch clause.
2912 bool isCatch(unsigned Idx) const {
2913 return !isa<ArrayType>(getOperandList()[Idx]->getType());
2914 }
2915
2916 /// Return 'true' if the clause and index Idx is a filter clause.
2917 bool isFilter(unsigned Idx) const {
2918 return isa<ArrayType>(getOperandList()[Idx]->getType());
2919 }
2920
2921 /// Get the number of clauses for this landing pad.
2922 unsigned getNumClauses() const { return getNumOperands(); }
2923
2924 /// Grow the size of the operand list to accommodate the new
2925 /// number of clauses.
2926 void reserveClauses(unsigned Size) { growOperands(Size); }
2927
2928 // Methods for support type inquiry through isa, cast, and dyn_cast:
2929 static bool classof(const Instruction *I) {
2930 return I->getOpcode() == Instruction::LandingPad;
2931 }
2932 static bool classof(const Value *V) {
2933 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2934 }
2935};
2936
2937template <>
2938struct OperandTraits<LandingPadInst> : public HungoffOperandTraits<1> {
2939};
2940
2941DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LandingPadInst, Value)LandingPadInst::op_iterator LandingPadInst::op_begin() { return
OperandTraits<LandingPadInst>::op_begin(this); } LandingPadInst
::const_op_iterator LandingPadInst::op_begin() const { return
OperandTraits<LandingPadInst>::op_begin(const_cast<
LandingPadInst*>(this)); } LandingPadInst::op_iterator LandingPadInst
::op_end() { return OperandTraits<LandingPadInst>::op_end
(this); } LandingPadInst::const_op_iterator LandingPadInst::op_end
() const { return OperandTraits<LandingPadInst>::op_end
(const_cast<LandingPadInst*>(this)); } Value *LandingPadInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<LandingPadInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<LandingPadInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2941, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<LandingPadInst>::op_begin(
const_cast<LandingPadInst*>(this))[i_nocapture].get());
} void LandingPadInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<LandingPadInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<LandingPadInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210621111111+acefe0eaaf82/llvm/include/llvm/IR/Instructions.h"
, 2941, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
LandingPadInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned LandingPadInst::getNumOperands() const { return OperandTraits
<LandingPadInst>::operands(this); } template <int Idx_nocapture
> Use &LandingPadInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &LandingPadInst::Op() const { return this->
OpFrom<Idx_nocapture>(this); }
2942
2943//===----------------------------------------------------------------------===//
2944// ReturnInst Class
2945//===----------------------------------------------------------------------===//
2946
2947//===---------------------------------------------------------------------------
2948/// Return a value (possibly void), from a function. Execution
2949