Bug Summary

File:llvm/lib/Transforms/Scalar/LICM.cpp
Warning:line 1196, column 33
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LICM.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar -I include -I /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/build-llvm -ferror-limit 19 -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-26-234817-15343-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp

/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp

1//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs loop invariant code motion, attempting to remove as much
10// code from the body of a loop as possible. It does this by either hoisting
11// code into the preheader block, or by sinking code to the exit blocks if it is
12// safe. This pass also promotes must-aliased memory locations in the loop to
13// live in registers, thus hoisting and sinking "invariant" loads and stores.
14//
15// Hoisting operations out of loops is a canonicalization transform. It
16// enables and simplifies subsequent optimizations in the middle-end.
17// Rematerialization of hoisted instructions to reduce register pressure is the
18// responsibility of the back-end, which has more accurate information about
19// register pressure and also handles other optimizations than LICM that
20// increase live-ranges.
21//
22// This pass uses alias analysis for two purposes:
23//
24// 1. Moving loop invariant loads and calls out of loops. If we can determine
25// that a load or call inside of a loop never aliases anything stored to,
26// we can hoist it or sink it like any other instruction.
27// 2. Scalar Promotion of Memory - If there is a store instruction inside of
28// the loop, we try to move the store to happen AFTER the loop instead of
29// inside of the loop. This can only happen if a few conditions are true:
30// A. The pointer stored through is loop invariant
31// B. There are no stores or loads in the loop which _may_ alias the
32// pointer. There are no calls in the loop which mod/ref the pointer.
33// If these conditions are true, we can promote the loads and stores in the
34// loop of the pointer to use a temporary alloca'd variable. We then use
35// the SSAUpdater to construct the appropriate SSA form for the value.
36//
37//===----------------------------------------------------------------------===//
38
39#include "llvm/Transforms/Scalar/LICM.h"
40#include "llvm/ADT/SetOperations.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/Analysis/AliasAnalysis.h"
43#include "llvm/Analysis/AliasSetTracker.h"
44#include "llvm/Analysis/BasicAliasAnalysis.h"
45#include "llvm/Analysis/BlockFrequencyInfo.h"
46#include "llvm/Analysis/CaptureTracking.h"
47#include "llvm/Analysis/ConstantFolding.h"
48#include "llvm/Analysis/GlobalsModRef.h"
49#include "llvm/Analysis/GuardUtils.h"
50#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
51#include "llvm/Analysis/Loads.h"
52#include "llvm/Analysis/LoopInfo.h"
53#include "llvm/Analysis/LoopIterator.h"
54#include "llvm/Analysis/LoopPass.h"
55#include "llvm/Analysis/MemoryBuiltins.h"
56#include "llvm/Analysis/MemorySSA.h"
57#include "llvm/Analysis/MemorySSAUpdater.h"
58#include "llvm/Analysis/MustExecute.h"
59#include "llvm/Analysis/OptimizationRemarkEmitter.h"
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
62#include "llvm/Analysis/TargetLibraryInfo.h"
63#include "llvm/Analysis/ValueTracking.h"
64#include "llvm/IR/CFG.h"
65#include "llvm/IR/Constants.h"
66#include "llvm/IR/DataLayout.h"
67#include "llvm/IR/DebugInfoMetadata.h"
68#include "llvm/IR/DerivedTypes.h"
69#include "llvm/IR/Dominators.h"
70#include "llvm/IR/Instructions.h"
71#include "llvm/IR/IntrinsicInst.h"
72#include "llvm/IR/LLVMContext.h"
73#include "llvm/IR/Metadata.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/PredIteratorCache.h"
76#include "llvm/InitializePasses.h"
77#include "llvm/Support/CommandLine.h"
78#include "llvm/Support/Debug.h"
79#include "llvm/Support/raw_ostream.h"
80#include "llvm/Transforms/Scalar.h"
81#include "llvm/Transforms/Scalar/LoopPassManager.h"
82#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
83#include "llvm/Transforms/Utils/BasicBlockUtils.h"
84#include "llvm/Transforms/Utils/Local.h"
85#include "llvm/Transforms/Utils/LoopUtils.h"
86#include "llvm/Transforms/Utils/SSAUpdater.h"
87#include <algorithm>
88#include <utility>
89using namespace llvm;
90
91#define DEBUG_TYPE"licm" "licm"
92
93STATISTIC(NumCreatedBlocks, "Number of blocks created")static llvm::Statistic NumCreatedBlocks = {"licm", "NumCreatedBlocks"
, "Number of blocks created"}
;
94STATISTIC(NumClonedBranches, "Number of branches cloned")static llvm::Statistic NumClonedBranches = {"licm", "NumClonedBranches"
, "Number of branches cloned"}
;
95STATISTIC(NumSunk, "Number of instructions sunk out of loop")static llvm::Statistic NumSunk = {"licm", "NumSunk", "Number of instructions sunk out of loop"
}
;
96STATISTIC(NumHoisted, "Number of instructions hoisted out of loop")static llvm::Statistic NumHoisted = {"licm", "NumHoisted", "Number of instructions hoisted out of loop"
}
;
97STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk")static llvm::Statistic NumMovedLoads = {"licm", "NumMovedLoads"
, "Number of load insts hoisted or sunk"}
;
98STATISTIC(NumMovedCalls, "Number of call insts hoisted or sunk")static llvm::Statistic NumMovedCalls = {"licm", "NumMovedCalls"
, "Number of call insts hoisted or sunk"}
;
99STATISTIC(NumPromoted, "Number of memory locations promoted to registers")static llvm::Statistic NumPromoted = {"licm", "NumPromoted", "Number of memory locations promoted to registers"
}
;
100
101/// Memory promotion is enabled by default.
102static cl::opt<bool>
103 DisablePromotion("disable-licm-promotion", cl::Hidden, cl::init(false),
104 cl::desc("Disable memory promotion in LICM pass"));
105
106static cl::opt<bool> ControlFlowHoisting(
107 "licm-control-flow-hoisting", cl::Hidden, cl::init(false),
108 cl::desc("Enable control flow (and PHI) hoisting in LICM"));
109
110static cl::opt<unsigned> HoistSinkColdnessThreshold(
111 "licm-coldness-threshold", cl::Hidden, cl::init(4),
112 cl::desc("Relative coldness Threshold of hoisting/sinking destination "
113 "block for LICM to be considered beneficial"));
114
115static cl::opt<uint32_t> MaxNumUsesTraversed(
116 "licm-max-num-uses-traversed", cl::Hidden, cl::init(8),
117 cl::desc("Max num uses visited for identifying load "
118 "invariance in loop using invariant start (default = 8)"));
119
120// Experimental option to allow imprecision in LICM in pathological cases, in
121// exchange for faster compile. This is to be removed if MemorySSA starts to
122// address the same issue. This flag applies only when LICM uses MemorySSA
123// instead on AliasSetTracker. LICM calls MemorySSAWalker's
124// getClobberingMemoryAccess, up to the value of the Cap, getting perfect
125// accuracy. Afterwards, LICM will call into MemorySSA's getDefiningAccess,
126// which may not be precise, since optimizeUses is capped. The result is
127// correct, but we may not get as "far up" as possible to get which access is
128// clobbering the one queried.
129cl::opt<unsigned> llvm::SetLicmMssaOptCap(
130 "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
131 cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
132 "for faster compile. Caps the MemorySSA clobbering calls."));
133
134// Experimentally, memory promotion carries less importance than sinking and
135// hoisting. Limit when we do promotion when using MemorySSA, in order to save
136// compile time.
137cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
138 "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
139 cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
140 "effect. When MSSA in LICM is enabled, then this is the maximum "
141 "number of accesses allowed to be present in a loop in order to "
142 "enable memory promotion."));
143
144static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
145static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
146 const LoopSafetyInfo *SafetyInfo,
147 TargetTransformInfo *TTI, bool &FreeInLoop,
148 bool LoopNestMode);
149static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
150 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
151 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
152 OptimizationRemarkEmitter *ORE);
153static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
154 BlockFrequencyInfo *BFI, const Loop *CurLoop,
155 ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
156 OptimizationRemarkEmitter *ORE);
157static bool isSafeToExecuteUnconditionally(Instruction &Inst,
158 const DominatorTree *DT,
159 const TargetLibraryInfo *TLI,
160 const Loop *CurLoop,
161 const LoopSafetyInfo *SafetyInfo,
162 OptimizationRemarkEmitter *ORE,
163 const Instruction *CtxI = nullptr);
164static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
165 AliasSetTracker *CurAST, Loop *CurLoop,
166 AAResults *AA);
167static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
168 Loop *CurLoop, Instruction &I,
169 SinkAndHoistLICMFlags &Flags);
170static bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
171 MemoryUse &MU);
172static Instruction *cloneInstructionInExitBlock(
173 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
174 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
175
176static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
177 MemorySSAUpdater *MSSAU);
178
179static void moveInstructionBefore(Instruction &I, Instruction &Dest,
180 ICFLoopSafetyInfo &SafetyInfo,
181 MemorySSAUpdater *MSSAU, ScalarEvolution *SE);
182
183static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
184 function_ref<void(Instruction *)> Fn);
185static SmallVector<SmallSetVector<Value *, 8>, 0>
186collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L);
187
188namespace {
189struct LoopInvariantCodeMotion {
190 bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
191 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
192 TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
193 OptimizationRemarkEmitter *ORE, bool LoopNestMode = false);
194
195 LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
196 unsigned LicmMssaNoAccForPromotionCap)
197 : LicmMssaOptCap(LicmMssaOptCap),
198 LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}
199
200private:
201 unsigned LicmMssaOptCap;
202 unsigned LicmMssaNoAccForPromotionCap;
203};
204
205struct LegacyLICMPass : public LoopPass {
206 static char ID; // Pass identification, replacement for typeid
207 LegacyLICMPass(
208 unsigned LicmMssaOptCap = SetLicmMssaOptCap,
209 unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
210 : LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
211 initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
212 }
213
214 bool runOnLoop(Loop *L, LPPassManager &LPM) override {
215 if (skipLoop(L))
216 return false;
217
218 LLVM_DEBUG(dbgs() << "Perform LICM on Loop with header at block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
<< L->getHeader()->getNameOrAsOperand() <<
"\n"; } } while (false)
219 << L->getHeader()->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "Perform LICM on Loop with header at block "
<< L->getHeader()->getNameOrAsOperand() <<
"\n"; } } while (false)
;
220
221 auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
222 MemorySSA *MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
223 bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
224 BlockFrequencyInfo *BFI =
225 hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
226 : nullptr;
227 // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
228 // pass. Function analyses need to be preserved across loop transformations
229 // but ORE cannot be preserved (see comment before the pass definition).
230 OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
231 return LICM.runOnLoop(
232 L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
233 &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
234 &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
235 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
236 *L->getHeader()->getParent()),
237 &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
238 *L->getHeader()->getParent()),
239 SE ? &SE->getSE() : nullptr, MSSA, &ORE);
240 }
241
242 /// This transformation requires natural loop information & requires that
243 /// loop preheaders be inserted into the CFG...
244 ///
245 void getAnalysisUsage(AnalysisUsage &AU) const override {
246 AU.addPreserved<DominatorTreeWrapperPass>();
247 AU.addPreserved<LoopInfoWrapperPass>();
248 AU.addRequired<TargetLibraryInfoWrapperPass>();
249 AU.addRequired<MemorySSAWrapperPass>();
250 AU.addPreserved<MemorySSAWrapperPass>();
251 AU.addRequired<TargetTransformInfoWrapperPass>();
252 getLoopAnalysisUsage(AU);
253 LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
254 AU.addPreserved<LazyBlockFrequencyInfoPass>();
255 AU.addPreserved<LazyBranchProbabilityInfoPass>();
256 }
257
258private:
259 LoopInvariantCodeMotion LICM;
260};
261} // namespace
262
263PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
264 LoopStandardAnalysisResults &AR, LPMUpdater &) {
265 if (!AR.MSSA)
266 report_fatal_error("LICM requires MemorySSA (loop-mssa)");
267
268 // For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
269 // pass. Function analyses need to be preserved across loop transformations
270 // but ORE cannot be preserved (see comment before the pass definition).
271 OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
272
273 LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
274 if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
275 &AR.SE, AR.MSSA, &ORE))
276 return PreservedAnalyses::all();
277
278 auto PA = getLoopPassPreservedAnalyses();
279
280 PA.preserve<DominatorTreeAnalysis>();
281 PA.preserve<LoopAnalysis>();
282 PA.preserve<MemorySSAAnalysis>();
283
284 return PA;
285}
286
287PreservedAnalyses LNICMPass::run(LoopNest &LN, LoopAnalysisManager &AM,
288 LoopStandardAnalysisResults &AR,
289 LPMUpdater &) {
290 if (!AR.MSSA)
291 report_fatal_error("LNICM requires MemorySSA (loop-mssa)");
292
293 // For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
294 // pass. Function analyses need to be preserved across loop transformations
295 // but ORE cannot be preserved (see comment before the pass definition).
296 OptimizationRemarkEmitter ORE(LN.getParent());
297
298 LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
299
300 Loop &OutermostLoop = LN.getOutermostLoop();
301 bool Changed = LICM.runOnLoop(&OutermostLoop, &AR.AA, &AR.LI, &AR.DT, AR.BFI,
302 &AR.TLI, &AR.TTI, &AR.SE, AR.MSSA, &ORE, true);
303
304 if (!Changed)
305 return PreservedAnalyses::all();
306
307 auto PA = getLoopPassPreservedAnalyses();
308
309 PA.preserve<DominatorTreeAnalysis>();
310 PA.preserve<LoopAnalysis>();
311 PA.preserve<MemorySSAAnalysis>();
312
313 return PA;
314}
315
316char LegacyLICMPass::ID = 0;
317INITIALIZE_PASS_BEGIN(LegacyLICMPass, "licm", "Loop Invariant Code Motion",static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
318 false, false)static void *initializeLegacyLICMPassPassOnce(PassRegistry &
Registry) {
319INITIALIZE_PASS_DEPENDENCY(LoopPass)initializeLoopPassPass(Registry);
320INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
321INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
322INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)initializeMemorySSAWrapperPassPass(Registry);
323INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)initializeLazyBFIPassPass(Registry);
324INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
325 false)PassInfo *PI = new PassInfo( "Loop Invariant Code Motion", "licm"
, &LegacyLICMPass::ID, PassInfo::NormalCtor_t(callDefaultCtor
<LegacyLICMPass>), false, false); Registry.registerPass
(*PI, true); return PI; } static llvm::once_flag InitializeLegacyLICMPassPassFlag
; void llvm::initializeLegacyLICMPassPass(PassRegistry &Registry
) { llvm::call_once(InitializeLegacyLICMPassPassFlag, initializeLegacyLICMPassPassOnce
, std::ref(Registry)); }
326
327Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
328Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
329 unsigned LicmMssaNoAccForPromotionCap) {
330 return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
331}
332
333llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(bool IsSink, Loop *L,
334 MemorySSA *MSSA)
335 : SinkAndHoistLICMFlags(SetLicmMssaOptCap, SetLicmMssaNoAccForPromotionCap,
336 IsSink, L, MSSA) {}
337
338llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(
339 unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap, bool IsSink,
340 Loop *L, MemorySSA *MSSA)
341 : LicmMssaOptCap(LicmMssaOptCap),
342 LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap),
343 IsSink(IsSink) {
344 assert(((L != nullptr) == (MSSA != nullptr)) &&(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 345, __extension__ __PRETTY_FUNCTION__))
345 "Unexpected values for SinkAndHoistLICMFlags")(static_cast <bool> (((L != nullptr) == (MSSA != nullptr
)) && "Unexpected values for SinkAndHoistLICMFlags") ?
void (0) : __assert_fail ("((L != nullptr) == (MSSA != nullptr)) && \"Unexpected values for SinkAndHoistLICMFlags\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 345, __extension__ __PRETTY_FUNCTION__))
;
346 if (!MSSA)
347 return;
348
349 unsigned AccessCapCount = 0;
350 for (auto *BB : L->getBlocks())
351 if (const auto *Accesses = MSSA->getBlockAccesses(BB))
352 for (const auto &MA : *Accesses) {
353 (void)MA;
354 ++AccessCapCount;
355 if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
356 NoOfMemAccTooLarge = true;
357 return;
358 }
359 }
360}
361
362/// Hoist expressions out of the specified loop. Note, alias info for inner
363/// loop is not preserved so it is not a good idea to run LICM multiple
364/// times on one loop.
365bool LoopInvariantCodeMotion::runOnLoop(
366 Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
367 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
368 ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE,
369 bool LoopNestMode) {
370 bool Changed = false;
371
372 assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop is not in LCSSA form."
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop is not in LCSSA form.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 372, __extension__ __PRETTY_FUNCTION__))
;
373
374 // If this loop has metadata indicating that LICM is not to be performed then
375 // just exit.
376 if (hasDisableLICMTransformsHint(L)) {
377 return false;
378 }
379
380 // Don't sink stores from loops with coroutine suspend instructions.
381 // LICM would sink instructions into the default destination of
382 // the coroutine switch. The default destination of the switch is to
383 // handle the case where the coroutine is suspended, by which point the
384 // coroutine frame may have been destroyed. No instruction can be sunk there.
385 // FIXME: This would unfortunately hurt the performance of coroutines, however
386 // there is currently no general solution for this. Similar issues could also
387 // potentially happen in other passes where instructions are being moved
388 // across that edge.
389 bool HasCoroSuspendInst = llvm::any_of(L->getBlocks(), [](BasicBlock *BB) {
390 return llvm::any_of(*BB, [](Instruction &I) {
391 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
392 return II && II->getIntrinsicID() == Intrinsic::coro_suspend;
393 });
394 });
395
396 MemorySSAUpdater MSSAU(MSSA);
397 SinkAndHoistLICMFlags Flags(LicmMssaOptCap, LicmMssaNoAccForPromotionCap,
398 /*IsSink=*/true, L, MSSA);
399
400 // Get the preheader block to move instructions into...
401 BasicBlock *Preheader = L->getLoopPreheader();
402
403 // Compute loop safety information.
404 ICFLoopSafetyInfo SafetyInfo;
405 SafetyInfo.computeLoopSafetyInfo(L);
406
407 // We want to visit all of the instructions in this loop... that are not parts
408 // of our subloops (they have already had their invariants hoisted out of
409 // their loop, into this loop, so there is no need to process the BODIES of
410 // the subloops).
411 //
412 // Traverse the body of the loop in depth first order on the dominator tree so
413 // that we are guaranteed to see definitions before we see uses. This allows
414 // us to sink instructions in one pass, without iteration. After sinking
415 // instructions, we perform another pass to hoist them out of the loop.
416 if (L->hasDedicatedExits())
417 Changed |= LoopNestMode
418 ? sinkRegionForLoopNest(DT->getNode(L->getHeader()), AA, LI,
419 DT, BFI, TLI, TTI, L, &MSSAU,
420 &SafetyInfo, Flags, ORE)
421 : sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI,
422 TLI, TTI, L, &MSSAU, &SafetyInfo, Flags, ORE);
423 Flags.setIsSink(false);
424 if (Preheader)
425 Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, L,
426 &MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode);
427
428 // Now that all loop invariants have been removed from the loop, promote any
429 // memory references to scalars that we can.
430 // Don't sink stores from loops without dedicated block exits. Exits
431 // containing indirect branches are not transformed by loop simplify,
432 // make sure we catch that. An additional load may be generated in the
433 // preheader for SSA updater, so also avoid sinking when no preheader
434 // is available.
435 if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
436 !Flags.tooManyMemoryAccesses() && !HasCoroSuspendInst) {
437 // Figure out the loop exits and their insertion points
438 SmallVector<BasicBlock *, 8> ExitBlocks;
439 L->getUniqueExitBlocks(ExitBlocks);
440
441 // We can't insert into a catchswitch.
442 bool HasCatchSwitch = llvm::any_of(ExitBlocks, [](BasicBlock *Exit) {
443 return isa<CatchSwitchInst>(Exit->getTerminator());
444 });
445
446 if (!HasCatchSwitch) {
447 SmallVector<Instruction *, 8> InsertPts;
448 SmallVector<MemoryAccess *, 8> MSSAInsertPts;
449 InsertPts.reserve(ExitBlocks.size());
450 MSSAInsertPts.reserve(ExitBlocks.size());
451 for (BasicBlock *ExitBlock : ExitBlocks) {
452 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
453 MSSAInsertPts.push_back(nullptr);
454 }
455
456 PredIteratorCache PIC;
457
458 // Promoting one set of accesses may make the pointers for another set
459 // loop invariant, so run this in a loop (with the MaybePromotable set
460 // decreasing in size over time).
461 bool Promoted = false;
462 bool LocalPromoted;
463 do {
464 LocalPromoted = false;
465 for (const SmallSetVector<Value *, 8> &PointerMustAliases :
466 collectPromotionCandidates(MSSA, AA, L)) {
467 LocalPromoted |= promoteLoopAccessesToScalars(
468 PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC,
469 LI, DT, TLI, L, &MSSAU, &SafetyInfo, ORE);
470 }
471 Promoted |= LocalPromoted;
472 } while (LocalPromoted);
473
474 // Once we have promoted values across the loop body we have to
475 // recursively reform LCSSA as any nested loop may now have values defined
476 // within the loop used in the outer loop.
477 // FIXME: This is really heavy handed. It would be a bit better to use an
478 // SSAUpdater strategy during promotion that was LCSSA aware and reformed
479 // it as it went.
480 if (Promoted)
481 formLCSSARecursively(*L, *DT, LI, SE);
482
483 Changed |= Promoted;
484 }
485 }
486
487 // Check that neither this loop nor its parent have had LCSSA broken. LICM is
488 // specifically moving instructions across the loop boundary and so it is
489 // especially in need of sanity checking here.
490 assert(L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!")(static_cast <bool> (L->isLCSSAForm(*DT) && "Loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("L->isLCSSAForm(*DT) && \"Loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 490, __extension__ __PRETTY_FUNCTION__))
;
491 assert((L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) &&(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 492, __extension__ __PRETTY_FUNCTION__))
492 "Parent loop not left in LCSSA form after LICM!")(static_cast <bool> ((L->isOutermost() || L->getParentLoop
()->isLCSSAForm(*DT)) && "Parent loop not left in LCSSA form after LICM!"
) ? void (0) : __assert_fail ("(L->isOutermost() || L->getParentLoop()->isLCSSAForm(*DT)) && \"Parent loop not left in LCSSA form after LICM!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 492, __extension__ __PRETTY_FUNCTION__))
;
493
494 if (VerifyMemorySSA)
495 MSSA->verifyMemorySSA();
496
497 if (Changed && SE)
498 SE->forgetLoopDispositions(L);
499 return Changed;
500}
501
502/// Walk the specified region of the CFG (defined by all blocks dominated by
503/// the specified block, and that are in the current loop) in reverse depth
504/// first order w.r.t the DominatorTree. This allows us to visit uses before
505/// definitions, allowing us to sink a loop body in one pass without iteration.
506///
507bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
508 DominatorTree *DT, BlockFrequencyInfo *BFI,
509 TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
510 Loop *CurLoop, MemorySSAUpdater *MSSAU,
511 ICFLoopSafetyInfo *SafetyInfo,
512 SinkAndHoistLICMFlags &Flags,
513 OptimizationRemarkEmitter *ORE, Loop *OutermostLoop) {
514
515 // Verify inputs.
516 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to sinkRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 518, __extension__ __PRETTY_FUNCTION__))
517 CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to sinkRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 518, __extension__ __PRETTY_FUNCTION__))
518 "Unexpected input to sinkRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to sinkRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to sinkRegion.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 518, __extension__ __PRETTY_FUNCTION__))
;
519
520 // We want to visit children before parents. We will enque all the parents
521 // before their children in the worklist and process the worklist in reverse
522 // order.
523 SmallVector<DomTreeNode *, 16> Worklist = collectChildrenInLoop(N, CurLoop);
524
525 bool Changed = false;
526 for (DomTreeNode *DTN : reverse(Worklist)) {
527 BasicBlock *BB = DTN->getBlock();
528 // Only need to process the contents of this block if it is not part of a
529 // subloop (which would already have been processed).
530 if (inSubLoop(BB, CurLoop, LI))
531 continue;
532
533 for (BasicBlock::iterator II = BB->end(); II != BB->begin();) {
534 Instruction &I = *--II;
535
536 // The instruction is not used in the loop if it is dead. In this case,
537 // we just delete it instead of sinking it.
538 if (isInstructionTriviallyDead(&I, TLI)) {
539 LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM deleting dead inst: " <<
I << '\n'; } } while (false)
;
540 salvageKnowledge(&I);
541 salvageDebugInfo(I);
542 ++II;
543 eraseInstruction(I, *SafetyInfo, MSSAU);
544 Changed = true;
545 continue;
546 }
547
548 // Check to see if we can sink this instruction to the exit blocks
549 // of the loop. We can do this if the all users of the instruction are
550 // outside of the loop. In this case, it doesn't even matter if the
551 // operands of the instruction are loop invariant.
552 //
553 bool FreeInLoop = false;
554 bool LoopNestMode = OutermostLoop != nullptr;
555 if (!I.mayHaveSideEffects() &&
556 isNotUsedOrFreeInLoop(I, LoopNestMode ? OutermostLoop : CurLoop,
557 SafetyInfo, TTI, FreeInLoop, LoopNestMode) &&
558 canSinkOrHoistInst(I, AA, DT, CurLoop, /*CurAST*/nullptr, MSSAU, true,
559 &Flags, ORE)) {
560 if (sink(I, LI, DT, BFI, CurLoop, SafetyInfo, MSSAU, ORE)) {
561 if (!FreeInLoop) {
562 ++II;
563 salvageDebugInfo(I);
564 eraseInstruction(I, *SafetyInfo, MSSAU);
565 }
566 Changed = true;
567 }
568 }
569 }
570 }
571 if (VerifyMemorySSA)
572 MSSAU->getMemorySSA()->verifyMemorySSA();
573 return Changed;
574}
575
576bool llvm::sinkRegionForLoopNest(
577 DomTreeNode *N, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
578 BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
579 Loop *CurLoop, MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
580 SinkAndHoistLICMFlags &Flags, OptimizationRemarkEmitter *ORE) {
581
582 bool Changed = false;
583 SmallPriorityWorklist<Loop *, 4> Worklist;
584 Worklist.insert(CurLoop);
585 appendLoopsToWorklist(*CurLoop, Worklist);
586 while (!Worklist.empty()) {
587 Loop *L = Worklist.pop_back_val();
588 Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI,
589 TTI, L, MSSAU, SafetyInfo, Flags, ORE, CurLoop);
590 }
591 return Changed;
592}
593
594namespace {
595// This is a helper class for hoistRegion to make it able to hoist control flow
596// in order to be able to hoist phis. The way this works is that we initially
597// start hoisting to the loop preheader, and when we see a loop invariant branch
598// we make note of this. When we then come to hoist an instruction that's
599// conditional on such a branch we duplicate the branch and the relevant control
600// flow, then hoist the instruction into the block corresponding to its original
601// block in the duplicated control flow.
602class ControlFlowHoister {
603private:
604 // Information about the loop we are hoisting from
605 LoopInfo *LI;
606 DominatorTree *DT;
607 Loop *CurLoop;
608 MemorySSAUpdater *MSSAU;
609
610 // A map of blocks in the loop to the block their instructions will be hoisted
611 // to.
612 DenseMap<BasicBlock *, BasicBlock *> HoistDestinationMap;
613
614 // The branches that we can hoist, mapped to the block that marks a
615 // convergence point of their control flow.
616 DenseMap<BranchInst *, BasicBlock *> HoistableBranches;
617
618public:
619 ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
620 MemorySSAUpdater *MSSAU)
621 : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}
622
623 void registerPossiblyHoistableBranch(BranchInst *BI) {
624 // We can only hoist conditional branches with loop invariant operands.
625 if (!ControlFlowHoisting || !BI->isConditional() ||
626 !CurLoop->hasLoopInvariantOperands(BI))
627 return;
628
629 // The branch destinations need to be in the loop, and we don't gain
630 // anything by duplicating conditional branches with duplicate successors,
631 // as it's essentially the same as an unconditional branch.
632 BasicBlock *TrueDest = BI->getSuccessor(0);
633 BasicBlock *FalseDest = BI->getSuccessor(1);
634 if (!CurLoop->contains(TrueDest) || !CurLoop->contains(FalseDest) ||
635 TrueDest == FalseDest)
636 return;
637
638 // We can hoist BI if one branch destination is the successor of the other,
639 // or both have common successor which we check by seeing if the
640 // intersection of their successors is non-empty.
641 // TODO: This could be expanded to allowing branches where both ends
642 // eventually converge to a single block.
643 SmallPtrSet<BasicBlock *, 4> TrueDestSucc, FalseDestSucc;
644 TrueDestSucc.insert(succ_begin(TrueDest), succ_end(TrueDest));
645 FalseDestSucc.insert(succ_begin(FalseDest), succ_end(FalseDest));
646 BasicBlock *CommonSucc = nullptr;
647 if (TrueDestSucc.count(FalseDest)) {
648 CommonSucc = FalseDest;
649 } else if (FalseDestSucc.count(TrueDest)) {
650 CommonSucc = TrueDest;
651 } else {
652 set_intersect(TrueDestSucc, FalseDestSucc);
653 // If there's one common successor use that.
654 if (TrueDestSucc.size() == 1)
655 CommonSucc = *TrueDestSucc.begin();
656 // If there's more than one pick whichever appears first in the block list
657 // (we can't use the value returned by TrueDestSucc.begin() as it's
658 // unpredicatable which element gets returned).
659 else if (!TrueDestSucc.empty()) {
660 Function *F = TrueDest->getParent();
661 auto IsSucc = [&](BasicBlock &BB) { return TrueDestSucc.count(&BB); };
662 auto It = llvm::find_if(*F, IsSucc);
663 assert(It != F->end() && "Could not find successor in function")(static_cast <bool> (It != F->end() && "Could not find successor in function"
) ? void (0) : __assert_fail ("It != F->end() && \"Could not find successor in function\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 663, __extension__ __PRETTY_FUNCTION__))
;
664 CommonSucc = &*It;
665 }
666 }
667 // The common successor has to be dominated by the branch, as otherwise
668 // there will be some other path to the successor that will not be
669 // controlled by this branch so any phi we hoist would be controlled by the
670 // wrong condition. This also takes care of avoiding hoisting of loop back
671 // edges.
672 // TODO: In some cases this could be relaxed if the successor is dominated
673 // by another block that's been hoisted and we can guarantee that the
674 // control flow has been replicated exactly.
675 if (CommonSucc && DT->dominates(BI, CommonSucc))
676 HoistableBranches[BI] = CommonSucc;
677 }
678
679 bool canHoistPHI(PHINode *PN) {
680 // The phi must have loop invariant operands.
681 if (!ControlFlowHoisting || !CurLoop->hasLoopInvariantOperands(PN))
682 return false;
683 // We can hoist phis if the block they are in is the target of hoistable
684 // branches which cover all of the predecessors of the block.
685 SmallPtrSet<BasicBlock *, 8> PredecessorBlocks;
686 BasicBlock *BB = PN->getParent();
687 for (BasicBlock *PredBB : predecessors(BB))
688 PredecessorBlocks.insert(PredBB);
689 // If we have less predecessor blocks than predecessors then the phi will
690 // have more than one incoming value for the same block which we can't
691 // handle.
692 // TODO: This could be handled be erasing some of the duplicate incoming
693 // values.
694 if (PredecessorBlocks.size() != pred_size(BB))
695 return false;
696 for (auto &Pair : HoistableBranches) {
697 if (Pair.second == BB) {
698 // Which blocks are predecessors via this branch depends on if the
699 // branch is triangle-like or diamond-like.
700 if (Pair.first->getSuccessor(0) == BB) {
701 PredecessorBlocks.erase(Pair.first->getParent());
702 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
703 } else if (Pair.first->getSuccessor(1) == BB) {
704 PredecessorBlocks.erase(Pair.first->getParent());
705 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
706 } else {
707 PredecessorBlocks.erase(Pair.first->getSuccessor(0));
708 PredecessorBlocks.erase(Pair.first->getSuccessor(1));
709 }
710 }
711 }
712 // PredecessorBlocks will now be empty if for every predecessor of BB we
713 // found a hoistable branch source.
714 return PredecessorBlocks.empty();
715 }
716
717 BasicBlock *getOrCreateHoistedBlock(BasicBlock *BB) {
718 if (!ControlFlowHoisting)
719 return CurLoop->getLoopPreheader();
720 // If BB has already been hoisted, return that
721 if (HoistDestinationMap.count(BB))
722 return HoistDestinationMap[BB];
723
724 // Check if this block is conditional based on a pending branch
725 auto HasBBAsSuccessor =
726 [&](DenseMap<BranchInst *, BasicBlock *>::value_type &Pair) {
727 return BB != Pair.second && (Pair.first->getSuccessor(0) == BB ||
728 Pair.first->getSuccessor(1) == BB);
729 };
730 auto It = llvm::find_if(HoistableBranches, HasBBAsSuccessor);
731
732 // If not involved in a pending branch, hoist to preheader
733 BasicBlock *InitialPreheader = CurLoop->getLoopPreheader();
734 if (It == HoistableBranches.end()) {
735 LLVM_DEBUG(dbgs() << "LICM using "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
736 << InitialPreheader->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
737 << " as hoist destination for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
738 << BB->getNameOrAsOperand() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM using " << InitialPreheader
->getNameOrAsOperand() << " as hoist destination for "
<< BB->getNameOrAsOperand() << "\n"; } } while
(false)
;
739 HoistDestinationMap[BB] = InitialPreheader;
740 return InitialPreheader;
741 }
742 BranchInst *BI = It->first;
743 assert(std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) ==(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 745, __extension__ __PRETTY_FUNCTION__))
744 HoistableBranches.end() &&(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 745, __extension__ __PRETTY_FUNCTION__))
745 "BB is expected to be the target of at most one branch")(static_cast <bool> (std::find_if(++It, HoistableBranches
.end(), HasBBAsSuccessor) == HoistableBranches.end() &&
"BB is expected to be the target of at most one branch") ? void
(0) : __assert_fail ("std::find_if(++It, HoistableBranches.end(), HasBBAsSuccessor) == HoistableBranches.end() && \"BB is expected to be the target of at most one branch\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 745, __extension__ __PRETTY_FUNCTION__))
;
746
747 LLVMContext &C = BB->getContext();
748 BasicBlock *TrueDest = BI->getSuccessor(0);
749 BasicBlock *FalseDest = BI->getSuccessor(1);
750 BasicBlock *CommonSucc = HoistableBranches[BI];
751 BasicBlock *HoistTarget = getOrCreateHoistedBlock(BI->getParent());
752
753 // Create hoisted versions of blocks that currently don't have them
754 auto CreateHoistedBlock = [&](BasicBlock *Orig) {
755 if (HoistDestinationMap.count(Orig))
756 return HoistDestinationMap[Orig];
757 BasicBlock *New =
758 BasicBlock::Create(C, Orig->getName() + ".licm", Orig->getParent());
759 HoistDestinationMap[Orig] = New;
760 DT->addNewBlock(New, HoistTarget);
761 if (CurLoop->getParentLoop())
762 CurLoop->getParentLoop()->addBasicBlockToLoop(New, *LI);
763 ++NumCreatedBlocks;
764 LLVM_DEBUG(dbgs() << "LICM created " << New->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
765 << " as hoist destination for " << Orig->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
766 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM created " << New->
getName() << " as hoist destination for " << Orig
->getName() << "\n"; } } while (false)
;
767 return New;
768 };
769 BasicBlock *HoistTrueDest = CreateHoistedBlock(TrueDest);
770 BasicBlock *HoistFalseDest = CreateHoistedBlock(FalseDest);
771 BasicBlock *HoistCommonSucc = CreateHoistedBlock(CommonSucc);
772
773 // Link up these blocks with branches.
774 if (!HoistCommonSucc->getTerminator()) {
775 // The new common successor we've generated will branch to whatever that
776 // hoist target branched to.
777 BasicBlock *TargetSucc = HoistTarget->getSingleSuccessor();
778 assert(TargetSucc && "Expected hoist target to have a single successor")(static_cast <bool> (TargetSucc && "Expected hoist target to have a single successor"
) ? void (0) : __assert_fail ("TargetSucc && \"Expected hoist target to have a single successor\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 778, __extension__ __PRETTY_FUNCTION__))
;
779 HoistCommonSucc->moveBefore(TargetSucc);
780 BranchInst::Create(TargetSucc, HoistCommonSucc);
781 }
782 if (!HoistTrueDest->getTerminator()) {
783 HoistTrueDest->moveBefore(HoistCommonSucc);
784 BranchInst::Create(HoistCommonSucc, HoistTrueDest);
785 }
786 if (!HoistFalseDest->getTerminator()) {
787 HoistFalseDest->moveBefore(HoistCommonSucc);
788 BranchInst::Create(HoistCommonSucc, HoistFalseDest);
789 }
790
791 // If BI is being cloned to what was originally the preheader then
792 // HoistCommonSucc will now be the new preheader.
793 if (HoistTarget == InitialPreheader) {
794 // Phis in the loop header now need to use the new preheader.
795 InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
796 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
797 HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
798 // The new preheader dominates the loop header.
799 DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
800 DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
801 DT->changeImmediateDominator(HeaderNode, PreheaderNode);
802 // The preheader hoist destination is now the new preheader, with the
803 // exception of the hoist destination of this branch.
804 for (auto &Pair : HoistDestinationMap)
805 if (Pair.second == InitialPreheader && Pair.first != BI->getParent())
806 Pair.second = HoistCommonSucc;
807 }
808
809 // Now finally clone BI.
810 ReplaceInstWithInst(
811 HoistTarget->getTerminator(),
812 BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
813 ++NumClonedBranches;
814
815 assert(CurLoop->getLoopPreheader() &&(static_cast <bool> (CurLoop->getLoopPreheader() &&
"Hoisting blocks should not have destroyed preheader") ? void
(0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 816, __extension__ __PRETTY_FUNCTION__))
816 "Hoisting blocks should not have destroyed preheader")(static_cast <bool> (CurLoop->getLoopPreheader() &&
"Hoisting blocks should not have destroyed preheader") ? void
(0) : __assert_fail ("CurLoop->getLoopPreheader() && \"Hoisting blocks should not have destroyed preheader\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 816, __extension__ __PRETTY_FUNCTION__))
;
817 return HoistDestinationMap[BB];
818 }
819};
820} // namespace
821
822// Hoisting/sinking instruction out of a loop isn't always beneficial. It's only
823// only worthwhile if the destination block is actually colder than current
824// block.
825static bool worthSinkOrHoistInst(Instruction &I, BasicBlock *DstBlock,
826 OptimizationRemarkEmitter *ORE,
827 BlockFrequencyInfo *BFI) {
828 // Check block frequency only when runtime profile is available
829 // to avoid pathological cases. With static profile, lean towards
830 // hosting because it helps canonicalize the loop for vectorizer.
831 if (!DstBlock->getParent()->hasProfileData())
832 return true;
833
834 if (!HoistSinkColdnessThreshold || !BFI)
835 return true;
836
837 BasicBlock *SrcBlock = I.getParent();
838 if (BFI->getBlockFreq(DstBlock).getFrequency() / HoistSinkColdnessThreshold >
839 BFI->getBlockFreq(SrcBlock).getFrequency()) {
840 ORE->emit([&]() {
841 return OptimizationRemarkMissed(DEBUG_TYPE"licm", "SinkHoistInst", &I)
842 << "failed to sink or hoist instruction because containing block "
843 "has lower frequency than destination block";
844 });
845 return false;
846 }
847
848 return true;
849}
850
851/// Walk the specified region of the CFG (defined by all blocks dominated by
852/// the specified block, and that are in the current loop) in depth first
853/// order w.r.t the DominatorTree. This allows us to visit definitions before
854/// uses, allowing us to hoist a loop body in one pass without iteration.
855///
856bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
857 DominatorTree *DT, BlockFrequencyInfo *BFI,
858 TargetLibraryInfo *TLI, Loop *CurLoop,
859 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
860 ICFLoopSafetyInfo *SafetyInfo,
861 SinkAndHoistLICMFlags &Flags,
862 OptimizationRemarkEmitter *ORE, bool LoopNestMode) {
863 // Verify inputs.
864 assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to hoistRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 866, __extension__ __PRETTY_FUNCTION__))
865 CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr &&(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to hoistRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 866, __extension__ __PRETTY_FUNCTION__))
866 "Unexpected input to hoistRegion.")(static_cast <bool> (N != nullptr && AA != nullptr
&& LI != nullptr && DT != nullptr &&
CurLoop != nullptr && MSSAU != nullptr && SafetyInfo
!= nullptr && "Unexpected input to hoistRegion.") ? void
(0) : __assert_fail ("N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr && CurLoop != nullptr && MSSAU != nullptr && SafetyInfo != nullptr && \"Unexpected input to hoistRegion.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 866, __extension__ __PRETTY_FUNCTION__))
;
867
868 ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);
869
870 // Keep track of instructions that have been hoisted, as they may need to be
871 // re-hoisted if they end up not dominating all of their uses.
872 SmallVector<Instruction *, 16> HoistedInstructions;
873
874 // For PHI hoisting to work we need to hoist blocks before their successors.
875 // We can do this by iterating through the blocks in the loop in reverse
876 // post-order.
877 LoopBlocksRPO Worklist(CurLoop);
878 Worklist.perform(LI);
879 bool Changed = false;
880 for (BasicBlock *BB : Worklist) {
881 // Only need to process the contents of this block if it is not part of a
882 // subloop (which would already have been processed).
883 if (!LoopNestMode && inSubLoop(BB, CurLoop, LI))
884 continue;
885
886 for (Instruction &I : llvm::make_early_inc_range(*BB)) {
887 // Try constant folding this instruction. If all the operands are
888 // constants, it is technically hoistable, but it would be better to
889 // just fold it.
890 if (Constant *C = ConstantFoldInstruction(
891 &I, I.getModule()->getDataLayout(), TLI)) {
892 LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << " --> " << *Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
" --> " << *C << '\n'; } } while (false)
893 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM folding inst: " << I <<
" --> " << *C << '\n'; } } while (false)
;
894 // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
895 I.replaceAllUsesWith(C);
896 if (isInstructionTriviallyDead(&I, TLI))
897 eraseInstruction(I, *SafetyInfo, MSSAU);
898 Changed = true;
899 continue;
900 }
901
902 // Try hoisting the instruction out to the preheader. We can only do
903 // this if all of the operands of the instruction are loop invariant and
904 // if it is safe to hoist the instruction. We also check block frequency
905 // to make sure instruction only gets hoisted into colder blocks.
906 // TODO: It may be safe to hoist if we are hoisting to a conditional block
907 // and we have accurately duplicated the control flow from the loop header
908 // to that block.
909 if (CurLoop->hasLoopInvariantOperands(&I) &&
910 canSinkOrHoistInst(I, AA, DT, CurLoop, /*CurAST*/ nullptr, MSSAU,
911 true, &Flags, ORE) &&
912 worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
913 isSafeToExecuteUnconditionally(
914 I, DT, TLI, CurLoop, SafetyInfo, ORE,
915 CurLoop->getLoopPreheader()->getTerminator())) {
916 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
917 MSSAU, SE, ORE);
918 HoistedInstructions.push_back(&I);
919 Changed = true;
920 continue;
921 }
922
923 // Attempt to remove floating point division out of the loop by
924 // converting it to a reciprocal multiplication.
925 if (I.getOpcode() == Instruction::FDiv && I.hasAllowReciprocal() &&
926 CurLoop->isLoopInvariant(I.getOperand(1))) {
927 auto Divisor = I.getOperand(1);
928 auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
929 auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
930 ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
931 SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
932 ReciprocalDivisor->insertBefore(&I);
933
934 auto Product =
935 BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
936 Product->setFastMathFlags(I.getFastMathFlags());
937 SafetyInfo->insertInstructionTo(Product, I.getParent());
938 Product->insertAfter(&I);
939 I.replaceAllUsesWith(Product);
940 eraseInstruction(I, *SafetyInfo, MSSAU);
941
942 hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
943 SafetyInfo, MSSAU, SE, ORE);
944 HoistedInstructions.push_back(ReciprocalDivisor);
945 Changed = true;
946 continue;
947 }
948
949 auto IsInvariantStart = [&](Instruction &I) {
950 using namespace PatternMatch;
951 return I.use_empty() &&
952 match(&I, m_Intrinsic<Intrinsic::invariant_start>());
953 };
954 auto MustExecuteWithoutWritesBefore = [&](Instruction &I) {
955 return SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
956 SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop);
957 };
958 if ((IsInvariantStart(I) || isGuard(&I)) &&
959 CurLoop->hasLoopInvariantOperands(&I) &&
960 MustExecuteWithoutWritesBefore(I)) {
961 hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
962 MSSAU, SE, ORE);
963 HoistedInstructions.push_back(&I);
964 Changed = true;
965 continue;
966 }
967
968 if (PHINode *PN = dyn_cast<PHINode>(&I)) {
969 if (CFH.canHoistPHI(PN)) {
970 // Redirect incoming blocks first to ensure that we create hoisted
971 // versions of those blocks before we hoist the phi.
972 for (unsigned int i = 0; i < PN->getNumIncomingValues(); ++i)
973 PN->setIncomingBlock(
974 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
975 hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
976 MSSAU, SE, ORE);
977 assert(DT->dominates(PN, BB) && "Conditional PHIs not expected")(static_cast <bool> (DT->dominates(PN, BB) &&
"Conditional PHIs not expected") ? void (0) : __assert_fail (
"DT->dominates(PN, BB) && \"Conditional PHIs not expected\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 977, __extension__ __PRETTY_FUNCTION__))
;
978 Changed = true;
979 continue;
980 }
981 }
982
983 // Remember possibly hoistable branches so we can actually hoist them
984 // later if needed.
985 if (BranchInst *BI = dyn_cast<BranchInst>(&I))
986 CFH.registerPossiblyHoistableBranch(BI);
987 }
988 }
989
990 // If we hoisted instructions to a conditional block they may not dominate
991 // their uses that weren't hoisted (such as phis where some operands are not
992 // loop invariant). If so make them unconditional by moving them to their
993 // immediate dominator. We iterate through the instructions in reverse order
994 // which ensures that when we rehoist an instruction we rehoist its operands,
995 // and also keep track of where in the block we are rehoisting to to make sure
996 // that we rehoist instructions before the instructions that use them.
997 Instruction *HoistPoint = nullptr;
998 if (ControlFlowHoisting) {
999 for (Instruction *I : reverse(HoistedInstructions)) {
1000 if (!llvm::all_of(I->uses(),
1001 [&](Use &U) { return DT->dominates(I, U); })) {
1002 BasicBlock *Dominator =
1003 DT->getNode(I->getParent())->getIDom()->getBlock();
1004 if (!HoistPoint || !DT->dominates(HoistPoint->getParent(), Dominator)) {
1005 if (HoistPoint)
1006 assert(DT->dominates(Dominator, HoistPoint->getParent()) &&(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1007, __extension__ __PRETTY_FUNCTION__))
1007 "New hoist point expected to dominate old hoist point")(static_cast <bool> (DT->dominates(Dominator, HoistPoint
->getParent()) && "New hoist point expected to dominate old hoist point"
) ? void (0) : __assert_fail ("DT->dominates(Dominator, HoistPoint->getParent()) && \"New hoist point expected to dominate old hoist point\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1007, __extension__ __PRETTY_FUNCTION__))
;
1008 HoistPoint = Dominator->getTerminator();
1009 }
1010 LLVM_DEBUG(dbgs() << "LICM rehoisting to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
1011 << HoistPoint->getParent()->getNameOrAsOperand()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
1012 << ": " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM rehoisting to " << HoistPoint
->getParent()->getNameOrAsOperand() << ": " <<
*I << "\n"; } } while (false)
;
1013 moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
1014 HoistPoint = I;
1015 Changed = true;
1016 }
1017 }
1018 }
1019 if (VerifyMemorySSA)
1020 MSSAU->getMemorySSA()->verifyMemorySSA();
1021
1022 // Now that we've finished hoisting make sure that LI and DT are still
1023 // valid.
1024#ifdef EXPENSIVE_CHECKS
1025 if (Changed) {
1026 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
(0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1027, __extension__ __PRETTY_FUNCTION__))
1027 "Dominator tree verification failed")(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast) && "Dominator tree verification failed") ? void
(0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast) && \"Dominator tree verification failed\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1027, __extension__ __PRETTY_FUNCTION__))
;
1028 LI->verify(*DT);
1029 }
1030#endif
1031
1032 return Changed;
1033}
1034
1035// Return true if LI is invariant within scope of the loop. LI is invariant if
1036// CurLoop is dominated by an invariant.start representing the same memory
1037// location and size as the memory location LI loads from, and also the
1038// invariant.start has no uses.
1039static bool isLoadInvariantInLoop(LoadInst *LI, DominatorTree *DT,
1040 Loop *CurLoop) {
1041 Value *Addr = LI->getOperand(0);
1042 const DataLayout &DL = LI->getModule()->getDataLayout();
1043 const TypeSize LocSizeInBits = DL.getTypeSizeInBits(LI->getType());
1044
1045 // It is not currently possible for clang to generate an invariant.start
1046 // intrinsic with scalable vector types because we don't support thread local
1047 // sizeless types and we don't permit sizeless types in structs or classes.
1048 // Furthermore, even if support is added for this in future the intrinsic
1049 // itself is defined to have a size of -1 for variable sized objects. This
1050 // makes it impossible to verify if the intrinsic envelops our region of
1051 // interest. For example, both <vscale x 32 x i8> and <vscale x 16 x i8>
1052 // types would have a -1 parameter, but the former is clearly double the size
1053 // of the latter.
1054 if (LocSizeInBits.isScalable())
29
Calling 'LinearPolySize::isScalable'
32
Returning from 'LinearPolySize::isScalable'
33
Taking true branch
1055 return false;
34
Returning zero, which participates in a condition later
1056
1057 // if the type is i8 addrspace(x)*, we know this is the type of
1058 // llvm.invariant.start operand
1059 auto *PtrInt8Ty = PointerType::get(Type::getInt8Ty(LI->getContext()),
1060 LI->getPointerAddressSpace());
1061 unsigned BitcastsVisited = 0;
1062 // Look through bitcasts until we reach the i8* type (this is invariant.start
1063 // operand type).
1064 while (Addr->getType() != PtrInt8Ty) {
1065 auto *BC = dyn_cast<BitCastInst>(Addr);
1066 // Avoid traversing high number of bitcast uses.
1067 if (++BitcastsVisited > MaxNumUsesTraversed || !BC)
1068 return false;
1069 Addr = BC->getOperand(0);
1070 }
1071
1072 unsigned UsesVisited = 0;
1073 // Traverse all uses of the load operand value, to see if invariant.start is
1074 // one of the uses, and whether it dominates the load instruction.
1075 for (auto *U : Addr->users()) {
1076 // Avoid traversing for Load operand with high number of users.
1077 if (++UsesVisited > MaxNumUsesTraversed)
1078 return false;
1079 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
1080 // If there are escaping uses of invariant.start instruction, the load maybe
1081 // non-invariant.
1082 if (!II || II->getIntrinsicID() != Intrinsic::invariant_start ||
1083 !II->use_empty())
1084 continue;
1085 ConstantInt *InvariantSize = cast<ConstantInt>(II->getArgOperand(0));
1086 // The intrinsic supports having a -1 argument for variable sized objects
1087 // so we should check for that here.
1088 if (InvariantSize->isNegative())
1089 continue;
1090 uint64_t InvariantSizeInBits = InvariantSize->getSExtValue() * 8;
1091 // Confirm the invariant.start location size contains the load operand size
1092 // in bits. Also, the invariant.start should dominate the load, and we
1093 // should not hoist the load out of a loop that contains this dominating
1094 // invariant.start.
1095 if (LocSizeInBits.getFixedSize() <= InvariantSizeInBits &&
1096 DT->properlyDominates(II->getParent(), CurLoop->getHeader()))
1097 return true;
1098 }
1099
1100 return false;
1101}
1102
1103namespace {
1104/// Return true if-and-only-if we know how to (mechanically) both hoist and
1105/// sink a given instruction out of a loop. Does not address legality
1106/// concerns such as aliasing or speculation safety.
1107bool isHoistableAndSinkableInst(Instruction &I) {
1108 // Only these instructions are hoistable/sinkable.
1109 return (isa<LoadInst>(I) || isa<StoreInst>(I) || isa<CallInst>(I) ||
5
Assuming 'I' is a 'LoadInst'
6
Returning the value 1, which participates in a condition later
1110 isa<FenceInst>(I) || isa<CastInst>(I) || isa<UnaryOperator>(I) ||
1111 isa<BinaryOperator>(I) || isa<SelectInst>(I) ||
1112 isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
1113 isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
1114 isa<ShuffleVectorInst>(I) || isa<ExtractValueInst>(I) ||
1115 isa<InsertValueInst>(I) || isa<FreezeInst>(I));
1116}
1117/// Return true if all of the alias sets within this AST are known not to
1118/// contain a Mod, or if MSSA knows there are no MemoryDefs in the loop.
1119bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
1120 const Loop *L) {
1121 if (CurAST) {
1122 for (AliasSet &AS : *CurAST) {
1123 if (!AS.isForwardingAliasSet() && AS.isMod()) {
1124 return false;
1125 }
1126 }
1127 return true;
1128 } else { /*MSSAU*/
1129 for (auto *BB : L->getBlocks())
1130 if (MSSAU->getMemorySSA()->getBlockDefs(BB))
1131 return false;
1132 return true;
1133 }
1134}
1135
1136/// Return true if I is the only Instruction with a MemoryAccess in L.
1137bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
1138 const MemorySSAUpdater *MSSAU) {
1139 for (auto *BB : L->getBlocks())
1140 if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
1141 int NotAPhi = 0;
1142 for (const auto &Acc : *Accs) {
1143 if (isa<MemoryPhi>(&Acc))
1144 continue;
1145 const auto *MUD = cast<MemoryUseOrDef>(&Acc);
1146 if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
1147 return false;
1148 }
1149 }
1150 return true;
1151}
1152}
1153
1154bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
1155 Loop *CurLoop, AliasSetTracker *CurAST,
1156 MemorySSAUpdater *MSSAU,
1157 bool TargetExecutesOncePerLoop,
1158 SinkAndHoistLICMFlags *Flags,
1159 OptimizationRemarkEmitter *ORE) {
1160 assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1161, __extension__ __PRETTY_FUNCTION__))
1
Assuming pointer value is null
2
Assuming the condition is true
3
'?' condition is true
1161 "Either AliasSetTracker or MemorySSA should be initialized.")(static_cast <bool> (((CurAST != nullptr) ^ (MSSAU != nullptr
)) && "Either AliasSetTracker or MemorySSA should be initialized."
) ? void (0) : __assert_fail ("((CurAST != nullptr) ^ (MSSAU != nullptr)) && \"Either AliasSetTracker or MemorySSA should be initialized.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1161, __extension__ __PRETTY_FUNCTION__))
;
1162
1163 // If we don't understand the instruction, bail early.
1164 if (!isHoistableAndSinkableInst(I))
4
Calling 'isHoistableAndSinkableInst'
7
Returning from 'isHoistableAndSinkableInst'
1165 return false;
1166
1167 MemorySSA *MSSA = MSSAU
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
8.1
'MSSAU' is non-null
? MSSAU->getMemorySSA() : nullptr;
8
Taking false branch
9
'?' condition is true
10
'MSSA' initialized here
1168 if (MSSA)
11
Assuming 'MSSA' is null
12
Taking false branch
1169 assert(Flags != nullptr && "Flags cannot be null.")(static_cast <bool> (Flags != nullptr && "Flags cannot be null."
) ? void (0) : __assert_fail ("Flags != nullptr && \"Flags cannot be null.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1169, __extension__ __PRETTY_FUNCTION__))
;
1170
1171 // Loads have extra constraints we have to verify before we can hoist them.
1172 if (LoadInst *LI
13.1
'LI' is non-null
13.1
'LI' is non-null
13.1
'LI' is non-null
13.1
'LI' is non-null
= dyn_cast<LoadInst>(&I)) {
13
Assuming the object is a 'LoadInst'
14
Taking true branch
1173 if (!LI->isUnordered())
15
Calling 'LoadInst::isUnordered'
19
Returning from 'LoadInst::isUnordered'
20
Taking false branch
1174 return false; // Don't sink/hoist volatile or ordered atomic loads!
1175
1176 // Loads from constant memory are always safe to move, even if they end up
1177 // in the same alias set as something that ends up being modified.
1178 if (AA->pointsToConstantMemory(LI->getOperand(0)))
21
Assuming the condition is false
22
Taking false branch
1179 return true;
1180 if (LI->hasMetadata(LLVMContext::MD_invariant_load))
23
Calling 'Instruction::hasMetadata'
26
Returning from 'Instruction::hasMetadata'
1181 return true;
1182
1183 if (LI->isAtomic() && !TargetExecutesOncePerLoop)
27
Assuming the condition is false
1184 return false; // Don't risk duplicating unordered loads
1185
1186 // This checks for an invariant.start dominating the load.
1187 if (isLoadInvariantInLoop(LI, DT, CurLoop))
28
Calling 'isLoadInvariantInLoop'
35
Returning from 'isLoadInvariantInLoop'
36
Taking false branch
1188 return true;
1189
1190 bool Invalidated;
1191 if (CurAST
36.1
'CurAST' is null
36.1
'CurAST' is null
36.1
'CurAST' is null
36.1
'CurAST' is null
)
37
Taking false branch
1192 Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
1193 CurLoop, AA);
1194 else
1195 Invalidated = pointerInvalidatedByLoopWithMSSA(
1196 MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, I, *Flags);
38
Called C++ object pointer is null
1197 // Check loop-invariant address because this may also be a sinkable load
1198 // whose address is not necessarily loop-invariant.
1199 if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
1200 ORE->emit([&]() {
1201 return OptimizationRemarkMissed(
1202 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressInvalidated", LI)
1203 << "failed to move load with loop-invariant address "
1204 "because the loop may invalidate its value";
1205 });
1206
1207 return !Invalidated;
1208 } else if (CallInst *CI = dyn_cast<CallInst>(&I)) {
1209 // Don't sink or hoist dbg info; it's legal, but not useful.
1210 if (isa<DbgInfoIntrinsic>(I))
1211 return false;
1212
1213 // Don't sink calls which can throw.
1214 if (CI->mayThrow())
1215 return false;
1216
1217 // Convergent attribute has been used on operations that involve
1218 // inter-thread communication which results are implicitly affected by the
1219 // enclosing control flows. It is not safe to hoist or sink such operations
1220 // across control flow.
1221 if (CI->isConvergent())
1222 return false;
1223
1224 using namespace PatternMatch;
1225 if (match(CI, m_Intrinsic<Intrinsic::assume>()))
1226 // Assumes don't actually alias anything or throw
1227 return true;
1228
1229 if (match(CI, m_Intrinsic<Intrinsic::experimental_widenable_condition>()))
1230 // Widenable conditions don't actually alias anything or throw
1231 return true;
1232
1233 // Handle simple cases by querying alias analysis.
1234 FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
1235 if (Behavior == FMRB_DoesNotAccessMemory)
1236 return true;
1237 if (AAResults::onlyReadsMemory(Behavior)) {
1238 // A readonly argmemonly function only reads from memory pointed to by
1239 // it's arguments with arbitrary offsets. If we can prove there are no
1240 // writes to this memory in the loop, we can hoist or sink.
1241 if (AAResults::onlyAccessesArgPointees(Behavior)) {
1242 // TODO: expand to writeable arguments
1243 for (Value *Op : CI->arg_operands())
1244 if (Op->getType()->isPointerTy()) {
1245 bool Invalidated;
1246 if (CurAST)
1247 Invalidated = pointerInvalidatedByLoop(
1248 MemoryLocation::getBeforeOrAfter(Op), CurAST, CurLoop, AA);
1249 else
1250 Invalidated = pointerInvalidatedByLoopWithMSSA(
1251 MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I,
1252 *Flags);
1253 if (Invalidated)
1254 return false;
1255 }
1256 return true;
1257 }
1258
1259 // If this call only reads from memory and there are no writes to memory
1260 // in the loop, we can hoist or sink the call as appropriate.
1261 if (isReadOnly(CurAST, MSSAU, CurLoop))
1262 return true;
1263 }
1264
1265 // FIXME: This should use mod/ref information to see if we can hoist or
1266 // sink the call.
1267
1268 return false;
1269 } else if (auto *FI = dyn_cast<FenceInst>(&I)) {
1270 // Fences alias (most) everything to provide ordering. For the moment,
1271 // just give up if there are any other memory operations in the loop.
1272 if (CurAST) {
1273 auto Begin = CurAST->begin();
1274 assert(Begin != CurAST->end() && "must contain FI")(static_cast <bool> (Begin != CurAST->end() &&
"must contain FI") ? void (0) : __assert_fail ("Begin != CurAST->end() && \"must contain FI\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1274, __extension__ __PRETTY_FUNCTION__))
;
1275 if (std::next(Begin) != CurAST->end())
1276 // constant memory for instance, TODO: handle better
1277 return false;
1278 auto *UniqueI = Begin->getUniqueInstruction();
1279 if (!UniqueI)
1280 // other memory op, give up
1281 return false;
1282 (void)FI; // suppress unused variable warning
1283 assert(UniqueI == FI && "AS must contain FI")(static_cast <bool> (UniqueI == FI && "AS must contain FI"
) ? void (0) : __assert_fail ("UniqueI == FI && \"AS must contain FI\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1283, __extension__ __PRETTY_FUNCTION__))
;
1284 return true;
1285 } else // MSSAU
1286 return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
1287 } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
1288 if (!SI->isUnordered())
1289 return false; // Don't sink/hoist volatile or ordered atomic store!
1290
1291 // We can only hoist a store that we can prove writes a value which is not
1292 // read or overwritten within the loop. For those cases, we fallback to
1293 // load store promotion instead. TODO: We can extend this to cases where
1294 // there is exactly one write to the location and that write dominates an
1295 // arbitrary number of reads in the loop.
1296 if (CurAST) {
1297 auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
1298
1299 if (AS.isRef() || !AS.isMustAlias())
1300 // Quick exit test, handled by the full path below as well.
1301 return false;
1302 auto *UniqueI = AS.getUniqueInstruction();
1303 if (!UniqueI)
1304 // other memory op, give up
1305 return false;
1306 assert(UniqueI == SI && "AS must contain SI")(static_cast <bool> (UniqueI == SI && "AS must contain SI"
) ? void (0) : __assert_fail ("UniqueI == SI && \"AS must contain SI\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1306, __extension__ __PRETTY_FUNCTION__))
;
1307 return true;
1308 } else { // MSSAU
1309 if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
1310 return true;
1311 // If there are more accesses than the Promotion cap or no "quota" to
1312 // check clobber, then give up as we're not walking a list that long.
1313 if (Flags->tooManyMemoryAccesses() || Flags->tooManyClobberingCalls())
1314 return false;
1315 // If there are interfering Uses (i.e. their defining access is in the
1316 // loop), or ordered loads (stored as Defs!), don't move this store.
1317 // Could do better here, but this is conservatively correct.
1318 // TODO: Cache set of Uses on the first walk in runOnLoop, update when
1319 // moving accesses. Can also extend to dominating uses.
1320 auto *SIMD = MSSA->getMemoryAccess(SI);
1321 for (auto *BB : CurLoop->getBlocks())
1322 if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
1323 for (const auto &MA : *Accesses)
1324 if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
1325 auto *MD = MU->getDefiningAccess();
1326 if (!MSSA->isLiveOnEntryDef(MD) &&
1327 CurLoop->contains(MD->getBlock()))
1328 return false;
1329 // Disable hoisting past potentially interfering loads. Optimized
1330 // Uses may point to an access outside the loop, as getClobbering
1331 // checks the previous iteration when walking the backedge.
1332 // FIXME: More precise: no Uses that alias SI.
1333 if (!Flags->getIsSink() && !MSSA->dominates(SIMD, MU))
1334 return false;
1335 } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
1336 if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
1337 (void)LI; // Silence warning.
1338 assert(!LI->isUnordered() && "Expected unordered load")(static_cast <bool> (!LI->isUnordered() && "Expected unordered load"
) ? void (0) : __assert_fail ("!LI->isUnordered() && \"Expected unordered load\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1338, __extension__ __PRETTY_FUNCTION__))
;
1339 return false;
1340 }
1341 // Any call, while it may not be clobbering SI, it may be a use.
1342 if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
1343 // Check if the call may read from the memory location written
1344 // to by SI. Check CI's attributes and arguments; the number of
1345 // such checks performed is limited above by NoOfMemAccTooLarge.
1346 ModRefInfo MRI = AA->getModRefInfo(CI, MemoryLocation::get(SI));
1347 if (isModOrRefSet(MRI))
1348 return false;
1349 }
1350 }
1351 }
1352 auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
1353 Flags->incrementClobberingCalls();
1354 // If there are no clobbering Defs in the loop, store is safe to hoist.
1355 return MSSA->isLiveOnEntryDef(Source) ||
1356 !CurLoop->contains(Source->getBlock());
1357 }
1358 }
1359
1360 assert(!I.mayReadOrWriteMemory() && "unhandled aliasing")(static_cast <bool> (!I.mayReadOrWriteMemory() &&
"unhandled aliasing") ? void (0) : __assert_fail ("!I.mayReadOrWriteMemory() && \"unhandled aliasing\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1360, __extension__ __PRETTY_FUNCTION__))
;
1361
1362 // We've established mechanical ability and aliasing, it's up to the caller
1363 // to check fault safety
1364 return true;
1365}
1366
1367/// Returns true if a PHINode is a trivially replaceable with an
1368/// Instruction.
1369/// This is true when all incoming values are that instruction.
1370/// This pattern occurs most often with LCSSA PHI nodes.
1371///
1372static bool isTriviallyReplaceablePHI(const PHINode &PN, const Instruction &I) {
1373 for (const Value *IncValue : PN.incoming_values())
1374 if (IncValue != &I)
1375 return false;
1376
1377 return true;
1378}
1379
1380/// Return true if the instruction is free in the loop.
1381static bool isFreeInLoop(const Instruction &I, const Loop *CurLoop,
1382 const TargetTransformInfo *TTI) {
1383
1384 if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I)) {
1385 if (TTI->getUserCost(GEP, TargetTransformInfo::TCK_SizeAndLatency) !=
1386 TargetTransformInfo::TCC_Free)
1387 return false;
1388 // For a GEP, we cannot simply use getUserCost because currently it
1389 // optimistically assume that a GEP will fold into addressing mode
1390 // regardless of its users.
1391 const BasicBlock *BB = GEP->getParent();
1392 for (const User *U : GEP->users()) {
1393 const Instruction *UI = cast<Instruction>(U);
1394 if (CurLoop->contains(UI) &&
1395 (BB != UI->getParent() ||
1396 (!isa<StoreInst>(UI) && !isa<LoadInst>(UI))))
1397 return false;
1398 }
1399 return true;
1400 } else
1401 return TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency) ==
1402 TargetTransformInfo::TCC_Free;
1403}
1404
1405/// Return true if the only users of this instruction are outside of
1406/// the loop. If this is true, we can sink the instruction to the exit
1407/// blocks of the loop.
1408///
1409/// We also return true if the instruction could be folded away in lowering.
1410/// (e.g., a GEP can be folded into a load as an addressing mode in the loop).
1411static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
1412 const LoopSafetyInfo *SafetyInfo,
1413 TargetTransformInfo *TTI, bool &FreeInLoop,
1414 bool LoopNestMode) {
1415 const auto &BlockColors = SafetyInfo->getBlockColors();
1416 bool IsFree = isFreeInLoop(I, CurLoop, TTI);
1417 for (const User *U : I.users()) {
1418 const Instruction *UI = cast<Instruction>(U);
1419 if (const PHINode *PN = dyn_cast<PHINode>(UI)) {
1420 const BasicBlock *BB = PN->getParent();
1421 // We cannot sink uses in catchswitches.
1422 if (isa<CatchSwitchInst>(BB->getTerminator()))
1423 return false;
1424
1425 // We need to sink a callsite to a unique funclet. Avoid sinking if the
1426 // phi use is too muddled.
1427 if (isa<CallInst>(I))
1428 if (!BlockColors.empty() &&
1429 BlockColors.find(const_cast<BasicBlock *>(BB))->second.size() != 1)
1430 return false;
1431
1432 if (LoopNestMode) {
1433 while (isa<PHINode>(UI) && UI->hasOneUser() &&
1434 UI->getNumOperands() == 1) {
1435 if (!CurLoop->contains(UI))
1436 break;
1437 UI = cast<Instruction>(UI->user_back());
1438 }
1439 }
1440 }
1441
1442 if (CurLoop->contains(UI)) {
1443 if (IsFree) {
1444 FreeInLoop = true;
1445 continue;
1446 }
1447 return false;
1448 }
1449 }
1450 return true;
1451}
1452
1453static Instruction *cloneInstructionInExitBlock(
1454 Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
1455 const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
1456 Instruction *New;
1457 if (auto *CI = dyn_cast<CallInst>(&I)) {
1458 const auto &BlockColors = SafetyInfo->getBlockColors();
1459
1460 // Sinking call-sites need to be handled differently from other
1461 // instructions. The cloned call-site needs a funclet bundle operand
1462 // appropriate for its location in the CFG.
1463 SmallVector<OperandBundleDef, 1> OpBundles;
1464 for (unsigned BundleIdx = 0, BundleEnd = CI->getNumOperandBundles();
1465 BundleIdx != BundleEnd; ++BundleIdx) {
1466 OperandBundleUse Bundle = CI->getOperandBundleAt(BundleIdx);
1467 if (Bundle.getTagID() == LLVMContext::OB_funclet)
1468 continue;
1469
1470 OpBundles.emplace_back(Bundle);
1471 }
1472
1473 if (!BlockColors.empty()) {
1474 const ColorVector &CV = BlockColors.find(&ExitBlock)->second;
1475 assert(CV.size() == 1 && "non-unique color for exit block!")(static_cast <bool> (CV.size() == 1 && "non-unique color for exit block!"
) ? void (0) : __assert_fail ("CV.size() == 1 && \"non-unique color for exit block!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1475, __extension__ __PRETTY_FUNCTION__))
;
1476 BasicBlock *BBColor = CV.front();
1477 Instruction *EHPad = BBColor->getFirstNonPHI();
1478 if (EHPad->isEHPad())
1479 OpBundles.emplace_back("funclet", EHPad);
1480 }
1481
1482 New = CallInst::Create(CI, OpBundles);
1483 } else {
1484 New = I.clone();
1485 }
1486
1487 ExitBlock.getInstList().insert(ExitBlock.getFirstInsertionPt(), New);
1488 if (!I.getName().empty())
1489 New->setName(I.getName() + ".le");
1490
1491 if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
1492 // Create a new MemoryAccess and let MemorySSA set its defining access.
1493 MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
1494 New, nullptr, New->getParent(), MemorySSA::Beginning);
1495 if (NewMemAcc) {
1496 if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
1497 MSSAU->insertDef(MemDef, /*RenameUses=*/true);
1498 else {
1499 auto *MemUse = cast<MemoryUse>(NewMemAcc);
1500 MSSAU->insertUse(MemUse, /*RenameUses=*/true);
1501 }
1502 }
1503 }
1504
1505 // Build LCSSA PHI nodes for any in-loop operands (if legal). Note that
1506 // this is particularly cheap because we can rip off the PHI node that we're
1507 // replacing for the number and blocks of the predecessors.
1508 // OPT: If this shows up in a profile, we can instead finish sinking all
1509 // invariant instructions, and then walk their operands to re-establish
1510 // LCSSA. That will eliminate creating PHI nodes just to nuke them when
1511 // sinking bottom-up.
1512 for (Use &Op : New->operands())
1513 if (LI->wouldBeOutOfLoopUseRequiringLCSSA(Op.get(), PN.getParent())) {
1514 auto *OInst = cast<Instruction>(Op.get());
1515 PHINode *OpPN =
1516 PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
1517 OInst->getName() + ".lcssa", &ExitBlock.front());
1518 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1519 OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
1520 Op = OpPN;
1521 }
1522 return New;
1523}
1524
1525static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
1526 MemorySSAUpdater *MSSAU) {
1527 if (MSSAU)
1528 MSSAU->removeMemoryAccess(&I);
1529 SafetyInfo.removeInstruction(&I);
1530 I.eraseFromParent();
1531}
1532
1533static void moveInstructionBefore(Instruction &I, Instruction &Dest,
1534 ICFLoopSafetyInfo &SafetyInfo,
1535 MemorySSAUpdater *MSSAU,
1536 ScalarEvolution *SE) {
1537 SafetyInfo.removeInstruction(&I);
1538 SafetyInfo.insertInstructionTo(&I, Dest.getParent());
1539 I.moveBefore(&Dest);
1540 if (MSSAU)
1541 if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
1542 MSSAU->getMemorySSA()->getMemoryAccess(&I)))
1543 MSSAU->moveToPlace(OldMemAcc, Dest.getParent(),
1544 MemorySSA::BeforeTerminator);
1545 if (SE)
1546 SE->forgetValue(&I);
1547}
1548
1549static Instruction *sinkThroughTriviallyReplaceablePHI(
1550 PHINode *TPN, Instruction *I, LoopInfo *LI,
1551 SmallDenseMap<BasicBlock *, Instruction *, 32> &SunkCopies,
1552 const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
1553 MemorySSAUpdater *MSSAU) {
1554 assert(isTriviallyReplaceablePHI(*TPN, *I) &&(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1555, __extension__ __PRETTY_FUNCTION__))
1555 "Expect only trivially replaceable PHI")(static_cast <bool> (isTriviallyReplaceablePHI(*TPN, *I
) && "Expect only trivially replaceable PHI") ? void (
0) : __assert_fail ("isTriviallyReplaceablePHI(*TPN, *I) && \"Expect only trivially replaceable PHI\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1555, __extension__ __PRETTY_FUNCTION__))
;
1556 BasicBlock *ExitBlock = TPN->getParent();
1557 Instruction *New;
1558 auto It = SunkCopies.find(ExitBlock);
1559 if (It != SunkCopies.end())
1560 New = It->second;
1561 else
1562 New = SunkCopies[ExitBlock] = cloneInstructionInExitBlock(
1563 *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
1564 return New;
1565}
1566
1567static bool canSplitPredecessors(PHINode *PN, LoopSafetyInfo *SafetyInfo) {
1568 BasicBlock *BB = PN->getParent();
1569 if (!BB->canSplitPredecessors())
1570 return false;
1571 // It's not impossible to split EHPad blocks, but if BlockColors already exist
1572 // it require updating BlockColors for all offspring blocks accordingly. By
1573 // skipping such corner case, we can make updating BlockColors after splitting
1574 // predecessor fairly simple.
1575 if (!SafetyInfo->getBlockColors().empty() && BB->getFirstNonPHI()->isEHPad())
1576 return false;
1577 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
1578 BasicBlock *BBPred = *PI;
1579 if (isa<IndirectBrInst>(BBPred->getTerminator()) ||
1580 isa<CallBrInst>(BBPred->getTerminator()))
1581 return false;
1582 }
1583 return true;
1584}
1585
1586static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
1587 LoopInfo *LI, const Loop *CurLoop,
1588 LoopSafetyInfo *SafetyInfo,
1589 MemorySSAUpdater *MSSAU) {
1590#ifndef NDEBUG
1591 SmallVector<BasicBlock *, 32> ExitBlocks;
1592 CurLoop->getUniqueExitBlocks(ExitBlocks);
1593 SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
1594 ExitBlocks.end());
1595#endif
1596 BasicBlock *ExitBB = PN->getParent();
1597 assert(ExitBlockSet.count(ExitBB) && "Expect the PHI is in an exit block.")(static_cast <bool> (ExitBlockSet.count(ExitBB) &&
"Expect the PHI is in an exit block.") ? void (0) : __assert_fail
("ExitBlockSet.count(ExitBB) && \"Expect the PHI is in an exit block.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1597, __extension__ __PRETTY_FUNCTION__))
;
1598
1599 // Split predecessors of the loop exit to make instructions in the loop are
1600 // exposed to exit blocks through trivially replaceable PHIs while keeping the
1601 // loop in the canonical form where each predecessor of each exit block should
1602 // be contained within the loop. For example, this will convert the loop below
1603 // from
1604 //
1605 // LB1:
1606 // %v1 =
1607 // br %LE, %LB2
1608 // LB2:
1609 // %v2 =
1610 // br %LE, %LB1
1611 // LE:
1612 // %p = phi [%v1, %LB1], [%v2, %LB2] <-- non-trivially replaceable
1613 //
1614 // to
1615 //
1616 // LB1:
1617 // %v1 =
1618 // br %LE.split, %LB2
1619 // LB2:
1620 // %v2 =
1621 // br %LE.split2, %LB1
1622 // LE.split:
1623 // %p1 = phi [%v1, %LB1] <-- trivially replaceable
1624 // br %LE
1625 // LE.split2:
1626 // %p2 = phi [%v2, %LB2] <-- trivially replaceable
1627 // br %LE
1628 // LE:
1629 // %p = phi [%p1, %LE.split], [%p2, %LE.split2]
1630 //
1631 const auto &BlockColors = SafetyInfo->getBlockColors();
1632 SmallSetVector<BasicBlock *, 8> PredBBs(pred_begin(ExitBB), pred_end(ExitBB));
1633 while (!PredBBs.empty()) {
1634 BasicBlock *PredBB = *PredBBs.begin();
1635 assert(CurLoop->contains(PredBB) &&(static_cast <bool> (CurLoop->contains(PredBB) &&
"Expect all predecessors are in the loop") ? void (0) : __assert_fail
("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1636, __extension__ __PRETTY_FUNCTION__))
1636 "Expect all predecessors are in the loop")(static_cast <bool> (CurLoop->contains(PredBB) &&
"Expect all predecessors are in the loop") ? void (0) : __assert_fail
("CurLoop->contains(PredBB) && \"Expect all predecessors are in the loop\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1636, __extension__ __PRETTY_FUNCTION__))
;
1637 if (PN->getBasicBlockIndex(PredBB) >= 0) {
1638 BasicBlock *NewPred = SplitBlockPredecessors(
1639 ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
1640 // Since we do not allow splitting EH-block with BlockColors in
1641 // canSplitPredecessors(), we can simply assign predecessor's color to
1642 // the new block.
1643 if (!BlockColors.empty())
1644 // Grab a reference to the ColorVector to be inserted before getting the
1645 // reference to the vector we are copying because inserting the new
1646 // element in BlockColors might cause the map to be reallocated.
1647 SafetyInfo->copyColors(NewPred, PredBB);
1648 }
1649 PredBBs.remove(PredBB);
1650 }
1651}
1652
1653/// When an instruction is found to only be used outside of the loop, this
1654/// function moves it to the exit blocks and patches up SSA form as needed.
1655/// This method is guaranteed to remove the original instruction from its
1656/// position, and may either delete it or move it to outside of the loop.
1657///
1658static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
1659 BlockFrequencyInfo *BFI, const Loop *CurLoop,
1660 ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
1661 OptimizationRemarkEmitter *ORE) {
1662 bool Changed = false;
1663 LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM sinking instruction: " <<
I << "\n"; } } while (false)
;
1664
1665 // Iterate over users to be ready for actual sinking. Replace users via
1666 // unreachable blocks with undef and make all user PHIs trivially replaceable.
1667 SmallPtrSet<Instruction *, 8> VisitedUsers;
1668 for (Value::user_iterator UI = I.user_begin(), UE = I.user_end(); UI != UE;) {
1669 auto *User = cast<Instruction>(*UI);
1670 Use &U = UI.getUse();
1671 ++UI;
1672
1673 if (VisitedUsers.count(User) || CurLoop->contains(User))
1674 continue;
1675
1676 if (!DT->isReachableFromEntry(User->getParent())) {
1677 U = UndefValue::get(I.getType());
1678 Changed = true;
1679 continue;
1680 }
1681
1682 // The user must be a PHI node.
1683 PHINode *PN = cast<PHINode>(User);
1684
1685 // Surprisingly, instructions can be used outside of loops without any
1686 // exits. This can only happen in PHI nodes if the incoming block is
1687 // unreachable.
1688 BasicBlock *BB = PN->getIncomingBlock(U);
1689 if (!DT->isReachableFromEntry(BB)) {
1690 U = UndefValue::get(I.getType());
1691 Changed = true;
1692 continue;
1693 }
1694
1695 VisitedUsers.insert(PN);
1696 if (isTriviallyReplaceablePHI(*PN, I))
1697 continue;
1698
1699 if (!canSplitPredecessors(PN, SafetyInfo))
1700 return Changed;
1701
1702 // Split predecessors of the PHI so that we can make users trivially
1703 // replaceable.
1704 splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);
1705
1706 // Should rebuild the iterators, as they may be invalidated by
1707 // splitPredecessorsOfLoopExit().
1708 UI = I.user_begin();
1709 UE = I.user_end();
1710 }
1711
1712 if (VisitedUsers.empty())
1713 return Changed;
1714
1715 ORE->emit([&]() {
1716 return OptimizationRemark(DEBUG_TYPE"licm", "InstSunk", &I)
1717 << "sinking " << ore::NV("Inst", &I);
1718 });
1719 if (isa<LoadInst>(I))
1720 ++NumMovedLoads;
1721 else if (isa<CallInst>(I))
1722 ++NumMovedCalls;
1723 ++NumSunk;
1724
1725#ifndef NDEBUG
1726 SmallVector<BasicBlock *, 32> ExitBlocks;
1727 CurLoop->getUniqueExitBlocks(ExitBlocks);
1728 SmallPtrSet<BasicBlock *, 32> ExitBlockSet(ExitBlocks.begin(),
1729 ExitBlocks.end());
1730#endif
1731
1732 // Clones of this instruction. Don't create more than one per exit block!
1733 SmallDenseMap<BasicBlock *, Instruction *, 32> SunkCopies;
1734
1735 // If this instruction is only used outside of the loop, then all users are
1736 // PHI nodes in exit blocks due to LCSSA form. Just RAUW them with clones of
1737 // the instruction.
1738 // First check if I is worth sinking for all uses. Sink only when it is worth
1739 // across all uses.
1740 SmallSetVector<User*, 8> Users(I.user_begin(), I.user_end());
1741 SmallVector<PHINode *, 8> ExitPNs;
1742 for (auto *UI : Users) {
1743 auto *User = cast<Instruction>(UI);
1744
1745 if (CurLoop->contains(User))
1746 continue;
1747
1748 PHINode *PN = cast<PHINode>(User);
1749 assert(ExitBlockSet.count(PN->getParent()) &&(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
(0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1750, __extension__ __PRETTY_FUNCTION__))
1750 "The LCSSA PHI is not in an exit block!")(static_cast <bool> (ExitBlockSet.count(PN->getParent
()) && "The LCSSA PHI is not in an exit block!") ? void
(0) : __assert_fail ("ExitBlockSet.count(PN->getParent()) && \"The LCSSA PHI is not in an exit block!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1750, __extension__ __PRETTY_FUNCTION__))
;
1751 if (!worthSinkOrHoistInst(I, PN->getParent(), ORE, BFI)) {
1752 return Changed;
1753 }
1754
1755 ExitPNs.push_back(PN);
1756 }
1757
1758 for (auto *PN : ExitPNs) {
1759
1760 // The PHI must be trivially replaceable.
1761 Instruction *New = sinkThroughTriviallyReplaceablePHI(
1762 PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
1763 PN->replaceAllUsesWith(New);
1764 eraseInstruction(*PN, *SafetyInfo, nullptr);
1765 Changed = true;
1766 }
1767 return Changed;
1768}
1769
1770/// When an instruction is found to only use loop invariant operands that
1771/// is safe to hoist, this instruction is called to do the dirty work.
1772///
1773static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
1774 BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
1775 MemorySSAUpdater *MSSAU, ScalarEvolution *SE,
1776 OptimizationRemarkEmitter *ORE) {
1777 LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getNameOrAsOperand() << ": "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
1778 << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM hoisting to " << Dest
->getNameOrAsOperand() << ": " << I << "\n"
; } } while (false)
;
1779 ORE->emit([&]() {
1780 return OptimizationRemark(DEBUG_TYPE"licm", "Hoisted", &I) << "hoisting "
1781 << ore::NV("Inst", &I);
1782 });
1783
1784 // Metadata can be dependent on conditions we are hoisting above.
1785 // Conservatively strip all metadata on the instruction unless we were
1786 // guaranteed to execute I if we entered the loop, in which case the metadata
1787 // is valid in the loop preheader.
1788 // Similarly, If I is a call and it is not guaranteed to execute in the loop,
1789 // then moving to the preheader means we should strip attributes on the call
1790 // that can cause UB since we may be hoisting above conditions that allowed
1791 // inferring those attributes. They may not be valid at the preheader.
1792 if ((I.hasMetadataOtherThanDebugLoc() || isa<CallInst>(I)) &&
1793 // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
1794 // time in isGuaranteedToExecute if we don't actually have anything to
1795 // drop. It is a compile time optimization, not required for correctness.
1796 !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
1797 I.dropUndefImplyingAttrsAndUnknownMetadata();
1798
1799 if (isa<PHINode>(I))
1800 // Move the new node to the end of the phi list in the destination block.
1801 moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
1802 else
1803 // Move the new node to the destination block, before its terminator.
1804 moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);
1805
1806 I.updateLocationAfterHoist();
1807
1808 if (isa<LoadInst>(I))
1809 ++NumMovedLoads;
1810 else if (isa<CallInst>(I))
1811 ++NumMovedCalls;
1812 ++NumHoisted;
1813}
1814
1815/// Only sink or hoist an instruction if it is not a trapping instruction,
1816/// or if the instruction is known not to trap when moved to the preheader.
1817/// or if it is a trapping instruction and is guaranteed to execute.
1818static bool isSafeToExecuteUnconditionally(Instruction &Inst,
1819 const DominatorTree *DT,
1820 const TargetLibraryInfo *TLI,
1821 const Loop *CurLoop,
1822 const LoopSafetyInfo *SafetyInfo,
1823 OptimizationRemarkEmitter *ORE,
1824 const Instruction *CtxI) {
1825 if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
1826 return true;
1827
1828 bool GuaranteedToExecute =
1829 SafetyInfo->isGuaranteedToExecute(Inst, DT, CurLoop);
1830
1831 if (!GuaranteedToExecute) {
1832 auto *LI = dyn_cast<LoadInst>(&Inst);
1833 if (LI && CurLoop->isLoopInvariant(LI->getPointerOperand()))
1834 ORE->emit([&]() {
1835 return OptimizationRemarkMissed(
1836 DEBUG_TYPE"licm", "LoadWithLoopInvariantAddressCondExecuted", LI)
1837 << "failed to hoist load with loop-invariant address "
1838 "because load is conditionally executed";
1839 });
1840 }
1841
1842 return GuaranteedToExecute;
1843}
1844
1845namespace {
1846class LoopPromoter : public LoadAndStorePromoter {
1847 Value *SomePtr; // Designated pointer to store to.
1848 const SmallSetVector<Value *, 8> &PointerMustAliases;
1849 SmallVectorImpl<BasicBlock *> &LoopExitBlocks;
1850 SmallVectorImpl<Instruction *> &LoopInsertPts;
1851 SmallVectorImpl<MemoryAccess *> &MSSAInsertPts;
1852 PredIteratorCache &PredCache;
1853 MemorySSAUpdater *MSSAU;
1854 LoopInfo &LI;
1855 DebugLoc DL;
1856 int Alignment;
1857 bool UnorderedAtomic;
1858 AAMDNodes AATags;
1859 ICFLoopSafetyInfo &SafetyInfo;
1860
1861 // We're about to add a use of V in a loop exit block. Insert an LCSSA phi
1862 // (if legal) if doing so would add an out-of-loop use to an instruction
1863 // defined in-loop.
1864 Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
1865 if (!LI.wouldBeOutOfLoopUseRequiringLCSSA(V, BB))
1866 return V;
1867
1868 Instruction *I = cast<Instruction>(V);
1869 // We need to create an LCSSA PHI node for the incoming value and
1870 // store that.
1871 PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
1872 I->getName() + ".lcssa", &BB->front());
1873 for (BasicBlock *Pred : PredCache.get(BB))
1874 PN->addIncoming(I, Pred);
1875 return PN;
1876 }
1877
1878public:
1879 LoopPromoter(Value *SP, ArrayRef<const Instruction *> Insts, SSAUpdater &S,
1880 const SmallSetVector<Value *, 8> &PMA,
1881 SmallVectorImpl<BasicBlock *> &LEB,
1882 SmallVectorImpl<Instruction *> &LIP,
1883 SmallVectorImpl<MemoryAccess *> &MSSAIP, PredIteratorCache &PIC,
1884 MemorySSAUpdater *MSSAU, LoopInfo &li, DebugLoc dl,
1885 int alignment, bool UnorderedAtomic, const AAMDNodes &AATags,
1886 ICFLoopSafetyInfo &SafetyInfo)
1887 : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
1888 LoopExitBlocks(LEB), LoopInsertPts(LIP), MSSAInsertPts(MSSAIP),
1889 PredCache(PIC), MSSAU(MSSAU), LI(li), DL(std::move(dl)),
1890 Alignment(alignment), UnorderedAtomic(UnorderedAtomic), AATags(AATags),
1891 SafetyInfo(SafetyInfo) {}
1892
1893 bool isInstInList(Instruction *I,
1894 const SmallVectorImpl<Instruction *> &) const override {
1895 Value *Ptr;
1896 if (LoadInst *LI = dyn_cast<LoadInst>(I))
1897 Ptr = LI->getOperand(0);
1898 else
1899 Ptr = cast<StoreInst>(I)->getPointerOperand();
1900 return PointerMustAliases.count(Ptr);
1901 }
1902
1903 void doExtraRewritesBeforeFinalDeletion() override {
1904 // Insert stores after in the loop exit blocks. Each exit block gets a
1905 // store of the live-out values that feed them. Since we've already told
1906 // the SSA updater about the defs in the loop and the preheader
1907 // definition, it is all set and we can start using it.
1908 for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) {
1909 BasicBlock *ExitBlock = LoopExitBlocks[i];
1910 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
1911 LiveInValue = maybeInsertLCSSAPHI(LiveInValue, ExitBlock);
1912 Value *Ptr = maybeInsertLCSSAPHI(SomePtr, ExitBlock);
1913 Instruction *InsertPos = LoopInsertPts[i];
1914 StoreInst *NewSI = new StoreInst(LiveInValue, Ptr, InsertPos);
1915 if (UnorderedAtomic)
1916 NewSI->setOrdering(AtomicOrdering::Unordered);
1917 NewSI->setAlignment(Align(Alignment));
1918 NewSI->setDebugLoc(DL);
1919 if (AATags)
1920 NewSI->setAAMetadata(AATags);
1921
1922 MemoryAccess *MSSAInsertPoint = MSSAInsertPts[i];
1923 MemoryAccess *NewMemAcc;
1924 if (!MSSAInsertPoint) {
1925 NewMemAcc = MSSAU->createMemoryAccessInBB(
1926 NewSI, nullptr, NewSI->getParent(), MemorySSA::Beginning);
1927 } else {
1928 NewMemAcc =
1929 MSSAU->createMemoryAccessAfter(NewSI, nullptr, MSSAInsertPoint);
1930 }
1931 MSSAInsertPts[i] = NewMemAcc;
1932 MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
1933 // FIXME: true for safety, false may still be correct.
1934 }
1935 }
1936
1937 void instructionDeleted(Instruction *I) const override {
1938 SafetyInfo.removeInstruction(I);
1939 MSSAU->removeMemoryAccess(I);
1940 }
1941};
1942
1943bool isNotCapturedBeforeOrInLoop(const Value *V, const Loop *L,
1944 DominatorTree *DT) {
1945 // We can perform the captured-before check against any instruction in the
1946 // loop header, as the loop header is reachable from any instruction inside
1947 // the loop.
1948 // TODO: ReturnCaptures=true shouldn't be necessary here.
1949 return !PointerMayBeCapturedBefore(V, /* ReturnCaptures */ true,
1950 /* StoreCaptures */ true,
1951 L->getHeader()->getTerminator(), DT);
1952}
1953
1954/// Return true iff we can prove that a caller of this function can not inspect
1955/// the contents of the provided object in a well defined program.
1956bool isKnownNonEscaping(Value *Object, const Loop *L,
1957 const TargetLibraryInfo *TLI, DominatorTree *DT) {
1958 if (isa<AllocaInst>(Object))
1959 // Since the alloca goes out of scope, we know the caller can't retain a
1960 // reference to it and be well defined. Thus, we don't need to check for
1961 // capture.
1962 return true;
1963
1964 // For all other objects we need to know that the caller can't possibly
1965 // have gotten a reference to the object. There are two components of
1966 // that:
1967 // 1) Object can't be escaped by this function. This is what
1968 // PointerMayBeCaptured checks.
1969 // 2) Object can't have been captured at definition site. For this, we
1970 // need to know the return value is noalias. At the moment, we use a
1971 // weaker condition and handle only AllocLikeFunctions (which are
1972 // known to be noalias). TODO
1973 return isAllocLikeFn(Object, TLI) &&
1974 isNotCapturedBeforeOrInLoop(Object, L, DT);
1975}
1976
1977} // namespace
1978
1979/// Try to promote memory values to scalars by sinking stores out of the
1980/// loop and moving loads to before the loop. We do this by looping over
1981/// the stores in the loop, looking for stores to Must pointers which are
1982/// loop invariant.
1983///
1984bool llvm::promoteLoopAccessesToScalars(
1985 const SmallSetVector<Value *, 8> &PointerMustAliases,
1986 SmallVectorImpl<BasicBlock *> &ExitBlocks,
1987 SmallVectorImpl<Instruction *> &InsertPts,
1988 SmallVectorImpl<MemoryAccess *> &MSSAInsertPts, PredIteratorCache &PIC,
1989 LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
1990 Loop *CurLoop, MemorySSAUpdater *MSSAU, ICFLoopSafetyInfo *SafetyInfo,
1991 OptimizationRemarkEmitter *ORE) {
1992 // Verify inputs.
1993 assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1995, __extension__ __PRETTY_FUNCTION__))
1994 SafetyInfo != nullptr &&(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1995, __extension__ __PRETTY_FUNCTION__))
1995 "Unexpected Input to promoteLoopAccessesToScalars")(static_cast <bool> (LI != nullptr && DT != nullptr
&& CurLoop != nullptr && SafetyInfo != nullptr
&& "Unexpected Input to promoteLoopAccessesToScalars"
) ? void (0) : __assert_fail ("LI != nullptr && DT != nullptr && CurLoop != nullptr && SafetyInfo != nullptr && \"Unexpected Input to promoteLoopAccessesToScalars\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 1995, __extension__ __PRETTY_FUNCTION__))
;
1996
1997 Value *SomePtr = *PointerMustAliases.begin();
1998 BasicBlock *Preheader = CurLoop->getLoopPreheader();
1999
2000 // It is not safe to promote a load/store from the loop if the load/store is
2001 // conditional. For example, turning:
2002 //
2003 // for () { if (c) *P += 1; }
2004 //
2005 // into:
2006 //
2007 // tmp = *P; for () { if (c) tmp +=1; } *P = tmp;
2008 //
2009 // is not safe, because *P may only be valid to access if 'c' is true.
2010 //
2011 // The safety property divides into two parts:
2012 // p1) The memory may not be dereferenceable on entry to the loop. In this
2013 // case, we can't insert the required load in the preheader.
2014 // p2) The memory model does not allow us to insert a store along any dynamic
2015 // path which did not originally have one.
2016 //
2017 // If at least one store is guaranteed to execute, both properties are
2018 // satisfied, and promotion is legal.
2019 //
2020 // This, however, is not a necessary condition. Even if no store/load is
2021 // guaranteed to execute, we can still establish these properties.
2022 // We can establish (p1) by proving that hoisting the load into the preheader
2023 // is safe (i.e. proving dereferenceability on all paths through the loop). We
2024 // can use any access within the alias set to prove dereferenceability,
2025 // since they're all must alias.
2026 //
2027 // There are two ways establish (p2):
2028 // a) Prove the location is thread-local. In this case the memory model
2029 // requirement does not apply, and stores are safe to insert.
2030 // b) Prove a store dominates every exit block. In this case, if an exit
2031 // blocks is reached, the original dynamic path would have taken us through
2032 // the store, so inserting a store into the exit block is safe. Note that this
2033 // is different from the store being guaranteed to execute. For instance,
2034 // if an exception is thrown on the first iteration of the loop, the original
2035 // store is never executed, but the exit blocks are not executed either.
2036
2037 bool DereferenceableInPH = false;
2038 bool SafeToInsertStore = false;
2039
2040 SmallVector<Instruction *, 64> LoopUses;
2041
2042 // We start with an alignment of one and try to find instructions that allow
2043 // us to prove better alignment.
2044 Align Alignment;
2045 // Keep track of which types of access we see
2046 bool SawUnorderedAtomic = false;
2047 bool SawNotAtomic = false;
2048 AAMDNodes AATags;
2049
2050 const DataLayout &MDL = Preheader->getModule()->getDataLayout();
2051
2052 bool IsKnownThreadLocalObject = false;
2053 if (SafetyInfo->anyBlockMayThrow()) {
2054 // If a loop can throw, we have to insert a store along each unwind edge.
2055 // That said, we can't actually make the unwind edge explicit. Therefore,
2056 // we have to prove that the store is dead along the unwind edge. We do
2057 // this by proving that the caller can't have a reference to the object
2058 // after return and thus can't possibly load from the object.
2059 Value *Object = getUnderlyingObject(SomePtr);
2060 if (!isKnownNonEscaping(Object, CurLoop, TLI, DT))
2061 return false;
2062 // Subtlety: Alloca's aren't visible to callers, but *are* potentially
2063 // visible to other threads if captured and used during their lifetimes.
2064 IsKnownThreadLocalObject = !isa<AllocaInst>(Object);
2065 }
2066
2067 // Check that all of the pointers in the alias set have the same type. We
2068 // cannot (yet) promote a memory location that is loaded and stored in
2069 // different sizes. While we are at it, collect alignment and AA info.
2070 for (Value *ASIV : PointerMustAliases) {
2071 // Check that all of the pointers in the alias set have the same type. We
2072 // cannot (yet) promote a memory location that is loaded and stored in
2073 // different sizes.
2074 if (SomePtr->getType() != ASIV->getType())
2075 return false;
2076
2077 for (User *U : ASIV->users()) {
2078 // Ignore instructions that are outside the loop.
2079 Instruction *UI = dyn_cast<Instruction>(U);
2080 if (!UI || !CurLoop->contains(UI))
2081 continue;
2082
2083 // If there is an non-load/store instruction in the loop, we can't promote
2084 // it.
2085 if (LoadInst *Load = dyn_cast<LoadInst>(UI)) {
2086 if (!Load->isUnordered())
2087 return false;
2088
2089 SawUnorderedAtomic |= Load->isAtomic();
2090 SawNotAtomic |= !Load->isAtomic();
2091
2092 Align InstAlignment = Load->getAlign();
2093
2094 // Note that proving a load safe to speculate requires proving
2095 // sufficient alignment at the target location. Proving it guaranteed
2096 // to execute does as well. Thus we can increase our guaranteed
2097 // alignment as well.
2098 if (!DereferenceableInPH || (InstAlignment > Alignment))
2099 if (isSafeToExecuteUnconditionally(*Load, DT, TLI, CurLoop,
2100 SafetyInfo, ORE,
2101 Preheader->getTerminator())) {
2102 DereferenceableInPH = true;
2103 Alignment = std::max(Alignment, InstAlignment);
2104 }
2105 } else if (const StoreInst *Store = dyn_cast<StoreInst>(UI)) {
2106 // Stores *of* the pointer are not interesting, only stores *to* the
2107 // pointer.
2108 if (UI->getOperand(1) != ASIV)
2109 continue;
2110 if (!Store->isUnordered())
2111 return false;
2112
2113 SawUnorderedAtomic |= Store->isAtomic();
2114 SawNotAtomic |= !Store->isAtomic();
2115
2116 // If the store is guaranteed to execute, both properties are satisfied.
2117 // We may want to check if a store is guaranteed to execute even if we
2118 // already know that promotion is safe, since it may have higher
2119 // alignment than any other guaranteed stores, in which case we can
2120 // raise the alignment on the promoted store.
2121 Align InstAlignment = Store->getAlign();
2122
2123 if (!DereferenceableInPH || !SafeToInsertStore ||
2124 (InstAlignment > Alignment)) {
2125 if (SafetyInfo->isGuaranteedToExecute(*UI, DT, CurLoop)) {
2126 DereferenceableInPH = true;
2127 SafeToInsertStore = true;
2128 Alignment = std::max(Alignment, InstAlignment);
2129 }
2130 }
2131
2132 // If a store dominates all exit blocks, it is safe to sink.
2133 // As explained above, if an exit block was executed, a dominating
2134 // store must have been executed at least once, so we are not
2135 // introducing stores on paths that did not have them.
2136 // Note that this only looks at explicit exit blocks. If we ever
2137 // start sinking stores into unwind edges (see above), this will break.
2138 if (!SafeToInsertStore)
2139 SafeToInsertStore = llvm::all_of(ExitBlocks, [&](BasicBlock *Exit) {
2140 return DT->dominates(Store->getParent(), Exit);
2141 });
2142
2143 // If the store is not guaranteed to execute, we may still get
2144 // deref info through it.
2145 if (!DereferenceableInPH) {
2146 DereferenceableInPH = isDereferenceableAndAlignedPointer(
2147 Store->getPointerOperand(), Store->getValueOperand()->getType(),
2148 Store->getAlign(), MDL, Preheader->getTerminator(), DT, TLI);
2149 }
2150 } else
2151 return false; // Not a load or store.
2152
2153 // Merge the AA tags.
2154 if (LoopUses.empty()) {
2155 // On the first load/store, just take its AA tags.
2156 AATags = UI->getAAMetadata();
2157 } else if (AATags) {
2158 AATags = AATags.merge(UI->getAAMetadata());
2159 }
2160
2161 LoopUses.push_back(UI);
2162 }
2163 }
2164
2165 // If we found both an unordered atomic instruction and a non-atomic memory
2166 // access, bail. We can't blindly promote non-atomic to atomic since we
2167 // might not be able to lower the result. We can't downgrade since that
2168 // would violate memory model. Also, align 0 is an error for atomics.
2169 if (SawUnorderedAtomic && SawNotAtomic)
2170 return false;
2171
2172 // If we're inserting an atomic load in the preheader, we must be able to
2173 // lower it. We're only guaranteed to be able to lower naturally aligned
2174 // atomics.
2175 auto *SomePtrElemType = SomePtr->getType()->getPointerElementType();
2176 if (SawUnorderedAtomic &&
2177 Alignment < MDL.getTypeStoreSize(SomePtrElemType))
2178 return false;
2179
2180 // If we couldn't prove we can hoist the load, bail.
2181 if (!DereferenceableInPH)
2182 return false;
2183
2184 // We know we can hoist the load, but don't have a guaranteed store.
2185 // Check whether the location is thread-local. If it is, then we can insert
2186 // stores along paths which originally didn't have them without violating the
2187 // memory model.
2188 if (!SafeToInsertStore) {
2189 if (IsKnownThreadLocalObject)
2190 SafeToInsertStore = true;
2191 else {
2192 Value *Object = getUnderlyingObject(SomePtr);
2193 SafeToInsertStore =
2194 (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
2195 isNotCapturedBeforeOrInLoop(Object, CurLoop, DT);
2196 }
2197 }
2198
2199 // If we've still failed to prove we can sink the store, give up.
2200 if (!SafeToInsertStore)
2201 return false;
2202
2203 // Otherwise, this is safe to promote, lets do it!
2204 LLVM_DEBUG(dbgs() << "LICM: Promoting value stored to in loop: " << *SomePtrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
<< *SomePtr << '\n'; } } while (false)
2205 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("licm")) { dbgs() << "LICM: Promoting value stored to in loop: "
<< *SomePtr << '\n'; } } while (false)
;
2206 ORE->emit([&]() {
2207 return OptimizationRemark(DEBUG_TYPE"licm", "PromoteLoopAccessesToScalar",
2208 LoopUses[0])
2209 << "Moving accesses to memory location out of the loop";
2210 });
2211 ++NumPromoted;
2212
2213 // Look at all the loop uses, and try to merge their locations.
2214 std::vector<const DILocation *> LoopUsesLocs;
2215 for (auto U : LoopUses)
2216 LoopUsesLocs.push_back(U->getDebugLoc().get());
2217 auto DL = DebugLoc(DILocation::getMergedLocations(LoopUsesLocs));
2218
2219 // We use the SSAUpdater interface to insert phi nodes as required.
2220 SmallVector<PHINode *, 16> NewPHIs;
2221 SSAUpdater SSA(&NewPHIs);
2222 LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
2223 InsertPts, MSSAInsertPts, PIC, MSSAU, *LI, DL,
2224 Alignment.value(), SawUnorderedAtomic, AATags,
2225 *SafetyInfo);
2226
2227 // Set up the preheader to have a definition of the value. It is the live-out
2228 // value from the preheader that uses in the loop will use.
2229 LoadInst *PreheaderLoad = new LoadInst(
2230 SomePtr->getType()->getPointerElementType(), SomePtr,
2231 SomePtr->getName() + ".promoted", Preheader->getTerminator());
2232 if (SawUnorderedAtomic)
2233 PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
2234 PreheaderLoad->setAlignment(Alignment);
2235 PreheaderLoad->setDebugLoc(DebugLoc());
2236 if (AATags)
2237 PreheaderLoad->setAAMetadata(AATags);
2238 SSA.AddAvailableValue(Preheader, PreheaderLoad);
2239
2240 MemoryAccess *PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
2241 PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
2242 MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
2243 MSSAU->insertUse(NewMemUse, /*RenameUses=*/true);
2244
2245 if (VerifyMemorySSA)
2246 MSSAU->getMemorySSA()->verifyMemorySSA();
2247 // Rewrite all the loads in the loop and remember all the definitions from
2248 // stores in the loop.
2249 Promoter.run(LoopUses);
2250
2251 if (VerifyMemorySSA)
2252 MSSAU->getMemorySSA()->verifyMemorySSA();
2253 // If the SSAUpdater didn't use the load in the preheader, just zap it now.
2254 if (PreheaderLoad->use_empty())
2255 eraseInstruction(*PreheaderLoad, *SafetyInfo, MSSAU);
2256
2257 return true;
2258}
2259
2260static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
2261 function_ref<void(Instruction *)> Fn) {
2262 for (const BasicBlock *BB : L->blocks())
2263 if (const auto *Accesses = MSSA->getBlockAccesses(BB))
2264 for (const auto &Access : *Accesses)
2265 if (const auto *MUD = dyn_cast<MemoryUseOrDef>(&Access))
2266 Fn(MUD->getMemoryInst());
2267}
2268
2269static SmallVector<SmallSetVector<Value *, 8>, 0>
2270collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
2271 AliasSetTracker AST(*AA);
2272
2273 auto IsPotentiallyPromotable = [L](const Instruction *I) {
2274 if (const auto *SI = dyn_cast<StoreInst>(I))
2275 return L->isLoopInvariant(SI->getPointerOperand());
2276 if (const auto *LI = dyn_cast<LoadInst>(I))
2277 return L->isLoopInvariant(LI->getPointerOperand());
2278 return false;
2279 };
2280
2281 // Populate AST with potentially promotable accesses and remove them from
2282 // MaybePromotable, so they will not be checked again on the next iteration.
2283 SmallPtrSet<Value *, 16> AttemptingPromotion;
2284 foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
2285 if (IsPotentiallyPromotable(I)) {
2286 AttemptingPromotion.insert(I);
2287 AST.add(I);
2288 }
2289 });
2290
2291 // We're only interested in must-alias sets that contain a mod.
2292 SmallVector<const AliasSet *, 8> Sets;
2293 for (AliasSet &AS : AST)
2294 if (!AS.isForwardingAliasSet() && AS.isMod() && AS.isMustAlias())
2295 Sets.push_back(&AS);
2296
2297 if (Sets.empty())
2298 return {}; // Nothing to promote...
2299
2300 // Discard any sets for which there is an aliasing non-promotable access.
2301 foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
2302 if (AttemptingPromotion.contains(I))
2303 return;
2304
2305 llvm::erase_if(Sets, [&](const AliasSet *AS) {
2306 return AS->aliasesUnknownInst(I, *AA);
2307 });
2308 });
2309
2310 SmallVector<SmallSetVector<Value *, 8>, 0> Result;
2311 for (const AliasSet *Set : Sets) {
2312 SmallSetVector<Value *, 8> PointerMustAliases;
2313 for (const auto &ASI : *Set)
2314 PointerMustAliases.insert(ASI.getValue());
2315 Result.push_back(std::move(PointerMustAliases));
2316 }
2317
2318 return Result;
2319}
2320
2321static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
2322 AliasSetTracker *CurAST, Loop *CurLoop,
2323 AAResults *AA) {
2324 return CurAST->getAliasSetFor(MemLoc).isMod();
2325}
2326
2327bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
2328 Loop *CurLoop, Instruction &I,
2329 SinkAndHoistLICMFlags &Flags) {
2330 // For hoisting, use the walker to determine safety
2331 if (!Flags.getIsSink()) {
2332 MemoryAccess *Source;
2333 // See declaration of SetLicmMssaOptCap for usage details.
2334 if (Flags.tooManyClobberingCalls())
2335 Source = MU->getDefiningAccess();
2336 else {
2337 Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
2338 Flags.incrementClobberingCalls();
2339 }
2340 return !MSSA->isLiveOnEntryDef(Source) &&
2341 CurLoop->contains(Source->getBlock());
2342 }
2343
2344 // For sinking, we'd need to check all Defs below this use. The getClobbering
2345 // call will look on the backedge of the loop, but will check aliasing with
2346 // the instructions on the previous iteration.
2347 // For example:
2348 // for (i ... )
2349 // load a[i] ( Use (LoE)
2350 // store a[i] ( 1 = Def (2), with 2 = Phi for the loop.
2351 // i++;
2352 // The load sees no clobbering inside the loop, as the backedge alias check
2353 // does phi translation, and will check aliasing against store a[i-1].
2354 // However sinking the load outside the loop, below the store is incorrect.
2355
2356 // For now, only sink if there are no Defs in the loop, and the existing ones
2357 // precede the use and are in the same block.
2358 // FIXME: Increase precision: Safe to sink if Use post dominates the Def;
2359 // needs PostDominatorTreeAnalysis.
2360 // FIXME: More precise: no Defs that alias this Use.
2361 if (Flags.tooManyMemoryAccesses())
2362 return true;
2363 for (auto *BB : CurLoop->getBlocks())
2364 if (pointerInvalidatedByBlockWithMSSA(*BB, *MSSA, *MU))
2365 return true;
2366 // When sinking, the source block may not be part of the loop so check it.
2367 if (!CurLoop->contains(&I))
2368 return pointerInvalidatedByBlockWithMSSA(*I.getParent(), *MSSA, *MU);
2369
2370 return false;
2371}
2372
2373bool pointerInvalidatedByBlockWithMSSA(BasicBlock &BB, MemorySSA &MSSA,
2374 MemoryUse &MU) {
2375 if (const auto *Accesses = MSSA.getBlockDefs(&BB))
2376 for (const auto &MA : *Accesses)
2377 if (const auto *MD = dyn_cast<MemoryDef>(&MA))
2378 if (MU.getBlock() != MD->getBlock() || !MSSA.locallyDominates(MD, &MU))
2379 return true;
2380 return false;
2381}
2382
2383/// Little predicate that returns true if the specified basic block is in
2384/// a subloop of the current one, not the current one itself.
2385///
2386static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI) {
2387 assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop")(static_cast <bool> (CurLoop->contains(BB) &&
"Only valid if BB is IN the loop") ? void (0) : __assert_fail
("CurLoop->contains(BB) && \"Only valid if BB is IN the loop\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/lib/Transforms/Scalar/LICM.cpp"
, 2387, __extension__ __PRETTY_FUNCTION__))
;
2388 return LI->getLoopFor(BB) != CurLoop;
2389}

/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h

1//===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file exposes the class definitions of all of the subclasses of the
10// Instruction class. This is meant to be an easy way to get access to all
11// instruction subclasses.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_INSTRUCTIONS_H
16#define LLVM_IR_INSTRUCTIONS_H
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/Bitfields.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/ADT/iterator.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/IR/Attributes.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/CFG.h"
32#include "llvm/IR/Constant.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/InstrTypes.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/OperandTraits.h"
38#include "llvm/IR/Type.h"
39#include "llvm/IR/Use.h"
40#include "llvm/IR/User.h"
41#include "llvm/IR/Value.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstddef>
47#include <cstdint>
48#include <iterator>
49
50namespace llvm {
51
52class APInt;
53class ConstantInt;
54class DataLayout;
55class LLVMContext;
56
57//===----------------------------------------------------------------------===//
58// AllocaInst Class
59//===----------------------------------------------------------------------===//
60
61/// an instruction to allocate memory on the stack
62class AllocaInst : public UnaryInstruction {
63 Type *AllocatedType;
64
65 using AlignmentField = AlignmentBitfieldElementT<0>;
66 using UsedWithInAllocaField = BoolBitfieldElementT<AlignmentField::NextBit>;
67 using SwiftErrorField = BoolBitfieldElementT<UsedWithInAllocaField::NextBit>;
68 static_assert(Bitfield::areContiguous<AlignmentField, UsedWithInAllocaField,
69 SwiftErrorField>(),
70 "Bitfields must be contiguous");
71
72protected:
73 // Note: Instruction needs to be a friend here to call cloneImpl.
74 friend class Instruction;
75
76 AllocaInst *cloneImpl() const;
77
78public:
79 explicit AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
80 const Twine &Name, Instruction *InsertBefore);
81 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
82 const Twine &Name, BasicBlock *InsertAtEnd);
83
84 AllocaInst(Type *Ty, unsigned AddrSpace, const Twine &Name,
85 Instruction *InsertBefore);
86 AllocaInst(Type *Ty, unsigned AddrSpace,
87 const Twine &Name, BasicBlock *InsertAtEnd);
88
89 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
90 const Twine &Name = "", Instruction *InsertBefore = nullptr);
91 AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize, Align Align,
92 const Twine &Name, BasicBlock *InsertAtEnd);
93
94 /// Return true if there is an allocation size parameter to the allocation
95 /// instruction that is not 1.
96 bool isArrayAllocation() const;
97
98 /// Get the number of elements allocated. For a simple allocation of a single
99 /// element, this will return a constant 1 value.
100 const Value *getArraySize() const { return getOperand(0); }
101 Value *getArraySize() { return getOperand(0); }
102
103 /// Overload to return most specific pointer type.
104 PointerType *getType() const {
105 return cast<PointerType>(Instruction::getType());
106 }
107
108 /// Get allocation size in bits. Returns None if size can't be determined,
109 /// e.g. in case of a VLA.
110 Optional<TypeSize> getAllocationSizeInBits(const DataLayout &DL) const;
111
112 /// Return the type that is being allocated by the instruction.
113 Type *getAllocatedType() const { return AllocatedType; }
114 /// for use only in special circumstances that need to generically
115 /// transform a whole instruction (eg: IR linking and vectorization).
116 void setAllocatedType(Type *Ty) { AllocatedType = Ty; }
117
118 /// Return the alignment of the memory that is being allocated by the
119 /// instruction.
120 Align getAlign() const {
121 return Align(1ULL << getSubclassData<AlignmentField>());
122 }
123
124 void setAlignment(Align Align) {
125 setSubclassData<AlignmentField>(Log2(Align));
126 }
127
128 // FIXME: Remove this one transition to Align is over.
129 unsigned getAlignment() const { return getAlign().value(); }
130
131 /// Return true if this alloca is in the entry block of the function and is a
132 /// constant size. If so, the code generator will fold it into the
133 /// prolog/epilog code, so it is basically free.
134 bool isStaticAlloca() const;
135
136 /// Return true if this alloca is used as an inalloca argument to a call. Such
137 /// allocas are never considered static even if they are in the entry block.
138 bool isUsedWithInAlloca() const {
139 return getSubclassData<UsedWithInAllocaField>();
140 }
141
142 /// Specify whether this alloca is used to represent the arguments to a call.
143 void setUsedWithInAlloca(bool V) {
144 setSubclassData<UsedWithInAllocaField>(V);
145 }
146
147 /// Return true if this alloca is used as a swifterror argument to a call.
148 bool isSwiftError() const { return getSubclassData<SwiftErrorField>(); }
149 /// Specify whether this alloca is used to represent a swifterror.
150 void setSwiftError(bool V) { setSubclassData<SwiftErrorField>(V); }
151
152 // Methods for support type inquiry through isa, cast, and dyn_cast:
153 static bool classof(const Instruction *I) {
154 return (I->getOpcode() == Instruction::Alloca);
155 }
156 static bool classof(const Value *V) {
157 return isa<Instruction>(V) && classof(cast<Instruction>(V));
158 }
159
160private:
161 // Shadow Instruction::setInstructionSubclassData with a private forwarding
162 // method so that subclasses cannot accidentally use it.
163 template <typename Bitfield>
164 void setSubclassData(typename Bitfield::Type Value) {
165 Instruction::setSubclassData<Bitfield>(Value);
166 }
167};
168
169//===----------------------------------------------------------------------===//
170// LoadInst Class
171//===----------------------------------------------------------------------===//
172
173/// An instruction for reading from memory. This uses the SubclassData field in
174/// Value to store whether or not the load is volatile.
175class LoadInst : public UnaryInstruction {
176 using VolatileField = BoolBitfieldElementT<0>;
177 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
178 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
179 static_assert(
180 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
181 "Bitfields must be contiguous");
182
183 void AssertOK();
184
185protected:
186 // Note: Instruction needs to be a friend here to call cloneImpl.
187 friend class Instruction;
188
189 LoadInst *cloneImpl() const;
190
191public:
192 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr,
193 Instruction *InsertBefore);
194 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
195 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
196 Instruction *InsertBefore);
197 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
198 BasicBlock *InsertAtEnd);
199 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
200 Align Align, Instruction *InsertBefore = nullptr);
201 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
202 Align Align, BasicBlock *InsertAtEnd);
203 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
204 Align Align, AtomicOrdering Order,
205 SyncScope::ID SSID = SyncScope::System,
206 Instruction *InsertBefore = nullptr);
207 LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
208 Align Align, AtomicOrdering Order, SyncScope::ID SSID,
209 BasicBlock *InsertAtEnd);
210
211 /// Return true if this is a load from a volatile memory location.
212 bool isVolatile() const { return getSubclassData<VolatileField>(); }
213
214 /// Specify whether this is a volatile load or not.
215 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
216
217 /// Return the alignment of the access that is being performed.
218 /// FIXME: Remove this function once transition to Align is over.
219 /// Use getAlign() instead.
220 unsigned getAlignment() const { return getAlign().value(); }
221
222 /// Return the alignment of the access that is being performed.
223 Align getAlign() const {
224 return Align(1ULL << (getSubclassData<AlignmentField>()));
225 }
226
227 void setAlignment(Align Align) {
228 setSubclassData<AlignmentField>(Log2(Align));
229 }
230
231 /// Returns the ordering constraint of this load instruction.
232 AtomicOrdering getOrdering() const {
233 return getSubclassData<OrderingField>();
234 }
235 /// Sets the ordering constraint of this load instruction. May not be Release
236 /// or AcquireRelease.
237 void setOrdering(AtomicOrdering Ordering) {
238 setSubclassData<OrderingField>(Ordering);
239 }
240
241 /// Returns the synchronization scope ID of this load instruction.
242 SyncScope::ID getSyncScopeID() const {
243 return SSID;
244 }
245
246 /// Sets the synchronization scope ID of this load instruction.
247 void setSyncScopeID(SyncScope::ID SSID) {
248 this->SSID = SSID;
249 }
250
251 /// Sets the ordering constraint and the synchronization scope ID of this load
252 /// instruction.
253 void setAtomic(AtomicOrdering Ordering,
254 SyncScope::ID SSID = SyncScope::System) {
255 setOrdering(Ordering);
256 setSyncScopeID(SSID);
257 }
258
259 bool isSimple() const { return !isAtomic() && !isVolatile(); }
260
261 bool isUnordered() const {
262 return (getOrdering() == AtomicOrdering::NotAtomic ||
16
Assuming the condition is true
18
Returning the value 1, which participates in a condition later
263 getOrdering() == AtomicOrdering::Unordered) &&
264 !isVolatile();
17
Assuming the condition is true
265 }
266
267 Value *getPointerOperand() { return getOperand(0); }
268 const Value *getPointerOperand() const { return getOperand(0); }
269 static unsigned getPointerOperandIndex() { return 0U; }
270 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
271
272 /// Returns the address space of the pointer operand.
273 unsigned getPointerAddressSpace() const {
274 return getPointerOperandType()->getPointerAddressSpace();
275 }
276
277 // Methods for support type inquiry through isa, cast, and dyn_cast:
278 static bool classof(const Instruction *I) {
279 return I->getOpcode() == Instruction::Load;
280 }
281 static bool classof(const Value *V) {
282 return isa<Instruction>(V) && classof(cast<Instruction>(V));
283 }
284
285private:
286 // Shadow Instruction::setInstructionSubclassData with a private forwarding
287 // method so that subclasses cannot accidentally use it.
288 template <typename Bitfield>
289 void setSubclassData(typename Bitfield::Type Value) {
290 Instruction::setSubclassData<Bitfield>(Value);
291 }
292
293 /// The synchronization scope ID of this load instruction. Not quite enough
294 /// room in SubClassData for everything, so synchronization scope ID gets its
295 /// own field.
296 SyncScope::ID SSID;
297};
298
299//===----------------------------------------------------------------------===//
300// StoreInst Class
301//===----------------------------------------------------------------------===//
302
303/// An instruction for storing to memory.
304class StoreInst : public Instruction {
305 using VolatileField = BoolBitfieldElementT<0>;
306 using AlignmentField = AlignmentBitfieldElementT<VolatileField::NextBit>;
307 using OrderingField = AtomicOrderingBitfieldElementT<AlignmentField::NextBit>;
308 static_assert(
309 Bitfield::areContiguous<VolatileField, AlignmentField, OrderingField>(),
310 "Bitfields must be contiguous");
311
312 void AssertOK();
313
314protected:
315 // Note: Instruction needs to be a friend here to call cloneImpl.
316 friend class Instruction;
317
318 StoreInst *cloneImpl() const;
319
320public:
321 StoreInst(Value *Val, Value *Ptr, Instruction *InsertBefore);
322 StoreInst(Value *Val, Value *Ptr, BasicBlock *InsertAtEnd);
323 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Instruction *InsertBefore);
324 StoreInst(Value *Val, Value *Ptr, bool isVolatile, BasicBlock *InsertAtEnd);
325 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
326 Instruction *InsertBefore = nullptr);
327 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
328 BasicBlock *InsertAtEnd);
329 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
330 AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
331 Instruction *InsertBefore = nullptr);
332 StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align,
333 AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd);
334
335 // allocate space for exactly two operands
336 void *operator new(size_t S) { return User::operator new(S, 2); }
337 void operator delete(void *Ptr) { User::operator delete(Ptr); }
338
339 /// Return true if this is a store to a volatile memory location.
340 bool isVolatile() const { return getSubclassData<VolatileField>(); }
341
342 /// Specify whether this is a volatile store or not.
343 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
344
345 /// Transparently provide more efficient getOperand methods.
346 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
347
348 /// Return the alignment of the access that is being performed
349 /// FIXME: Remove this function once transition to Align is over.
350 /// Use getAlign() instead.
351 unsigned getAlignment() const { return getAlign().value(); }
352
353 Align getAlign() const {
354 return Align(1ULL << (getSubclassData<AlignmentField>()));
355 }
356
357 void setAlignment(Align Align) {
358 setSubclassData<AlignmentField>(Log2(Align));
359 }
360
361 /// Returns the ordering constraint of this store instruction.
362 AtomicOrdering getOrdering() const {
363 return getSubclassData<OrderingField>();
364 }
365
366 /// Sets the ordering constraint of this store instruction. May not be
367 /// Acquire or AcquireRelease.
368 void setOrdering(AtomicOrdering Ordering) {
369 setSubclassData<OrderingField>(Ordering);
370 }
371
372 /// Returns the synchronization scope ID of this store instruction.
373 SyncScope::ID getSyncScopeID() const {
374 return SSID;
375 }
376
377 /// Sets the synchronization scope ID of this store instruction.
378 void setSyncScopeID(SyncScope::ID SSID) {
379 this->SSID = SSID;
380 }
381
382 /// Sets the ordering constraint and the synchronization scope ID of this
383 /// store instruction.
384 void setAtomic(AtomicOrdering Ordering,
385 SyncScope::ID SSID = SyncScope::System) {
386 setOrdering(Ordering);
387 setSyncScopeID(SSID);
388 }
389
390 bool isSimple() const { return !isAtomic() && !isVolatile(); }
391
392 bool isUnordered() const {
393 return (getOrdering() == AtomicOrdering::NotAtomic ||
394 getOrdering() == AtomicOrdering::Unordered) &&
395 !isVolatile();
396 }
397
398 Value *getValueOperand() { return getOperand(0); }
399 const Value *getValueOperand() const { return getOperand(0); }
400
401 Value *getPointerOperand() { return getOperand(1); }
402 const Value *getPointerOperand() const { return getOperand(1); }
403 static unsigned getPointerOperandIndex() { return 1U; }
404 Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
405
406 /// Returns the address space of the pointer operand.
407 unsigned getPointerAddressSpace() const {
408 return getPointerOperandType()->getPointerAddressSpace();
409 }
410
411 // Methods for support type inquiry through isa, cast, and dyn_cast:
412 static bool classof(const Instruction *I) {
413 return I->getOpcode() == Instruction::Store;
414 }
415 static bool classof(const Value *V) {
416 return isa<Instruction>(V) && classof(cast<Instruction>(V));
417 }
418
419private:
420 // Shadow Instruction::setInstructionSubclassData with a private forwarding
421 // method so that subclasses cannot accidentally use it.
422 template <typename Bitfield>
423 void setSubclassData(typename Bitfield::Type Value) {
424 Instruction::setSubclassData<Bitfield>(Value);
425 }
426
427 /// The synchronization scope ID of this store instruction. Not quite enough
428 /// room in SubClassData for everything, so synchronization scope ID gets its
429 /// own field.
430 SyncScope::ID SSID;
431};
432
433template <>
434struct OperandTraits<StoreInst> : public FixedNumOperandTraits<StoreInst, 2> {
435};
436
437DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value)StoreInst::op_iterator StoreInst::op_begin() { return OperandTraits
<StoreInst>::op_begin(this); } StoreInst::const_op_iterator
StoreInst::op_begin() const { return OperandTraits<StoreInst
>::op_begin(const_cast<StoreInst*>(this)); } StoreInst
::op_iterator StoreInst::op_end() { return OperandTraits<StoreInst
>::op_end(this); } StoreInst::const_op_iterator StoreInst::
op_end() const { return OperandTraits<StoreInst>::op_end
(const_cast<StoreInst*>(this)); } Value *StoreInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<StoreInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 437, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<StoreInst>::op_begin(const_cast
<StoreInst*>(this))[i_nocapture].get()); } void StoreInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<StoreInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<StoreInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 437, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
StoreInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned StoreInst::getNumOperands() const { return OperandTraits
<StoreInst>::operands(this); } template <int Idx_nocapture
> Use &StoreInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
StoreInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
438
439//===----------------------------------------------------------------------===//
440// FenceInst Class
441//===----------------------------------------------------------------------===//
442
443/// An instruction for ordering other memory operations.
444class FenceInst : public Instruction {
445 using OrderingField = AtomicOrderingBitfieldElementT<0>;
446
447 void Init(AtomicOrdering Ordering, SyncScope::ID SSID);
448
449protected:
450 // Note: Instruction needs to be a friend here to call cloneImpl.
451 friend class Instruction;
452
453 FenceInst *cloneImpl() const;
454
455public:
456 // Ordering may only be Acquire, Release, AcquireRelease, or
457 // SequentiallyConsistent.
458 FenceInst(LLVMContext &C, AtomicOrdering Ordering,
459 SyncScope::ID SSID = SyncScope::System,
460 Instruction *InsertBefore = nullptr);
461 FenceInst(LLVMContext &C, AtomicOrdering Ordering, SyncScope::ID SSID,
462 BasicBlock *InsertAtEnd);
463
464 // allocate space for exactly zero operands
465 void *operator new(size_t S) { return User::operator new(S, 0); }
466 void operator delete(void *Ptr) { User::operator delete(Ptr); }
467
468 /// Returns the ordering constraint of this fence instruction.
469 AtomicOrdering getOrdering() const {
470 return getSubclassData<OrderingField>();
471 }
472
473 /// Sets the ordering constraint of this fence instruction. May only be
474 /// Acquire, Release, AcquireRelease, or SequentiallyConsistent.
475 void setOrdering(AtomicOrdering Ordering) {
476 setSubclassData<OrderingField>(Ordering);
477 }
478
479 /// Returns the synchronization scope ID of this fence instruction.
480 SyncScope::ID getSyncScopeID() const {
481 return SSID;
482 }
483
484 /// Sets the synchronization scope ID of this fence instruction.
485 void setSyncScopeID(SyncScope::ID SSID) {
486 this->SSID = SSID;
487 }
488
489 // Methods for support type inquiry through isa, cast, and dyn_cast:
490 static bool classof(const Instruction *I) {
491 return I->getOpcode() == Instruction::Fence;
492 }
493 static bool classof(const Value *V) {
494 return isa<Instruction>(V) && classof(cast<Instruction>(V));
495 }
496
497private:
498 // Shadow Instruction::setInstructionSubclassData with a private forwarding
499 // method so that subclasses cannot accidentally use it.
500 template <typename Bitfield>
501 void setSubclassData(typename Bitfield::Type Value) {
502 Instruction::setSubclassData<Bitfield>(Value);
503 }
504
505 /// The synchronization scope ID of this fence instruction. Not quite enough
506 /// room in SubClassData for everything, so synchronization scope ID gets its
507 /// own field.
508 SyncScope::ID SSID;
509};
510
511//===----------------------------------------------------------------------===//
512// AtomicCmpXchgInst Class
513//===----------------------------------------------------------------------===//
514
515/// An instruction that atomically checks whether a
516/// specified value is in a memory location, and, if it is, stores a new value
517/// there. The value returned by this instruction is a pair containing the
518/// original value as first element, and an i1 indicating success (true) or
519/// failure (false) as second element.
520///
521class AtomicCmpXchgInst : public Instruction {
522 void Init(Value *Ptr, Value *Cmp, Value *NewVal, Align Align,
523 AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
524 SyncScope::ID SSID);
525
526 template <unsigned Offset>
527 using AtomicOrderingBitfieldElement =
528 typename Bitfield::Element<AtomicOrdering, Offset, 3,
529 AtomicOrdering::LAST>;
530
531protected:
532 // Note: Instruction needs to be a friend here to call cloneImpl.
533 friend class Instruction;
534
535 AtomicCmpXchgInst *cloneImpl() const;
536
537public:
538 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
539 AtomicOrdering SuccessOrdering,
540 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
541 Instruction *InsertBefore = nullptr);
542 AtomicCmpXchgInst(Value *Ptr, Value *Cmp, Value *NewVal, Align Alignment,
543 AtomicOrdering SuccessOrdering,
544 AtomicOrdering FailureOrdering, SyncScope::ID SSID,
545 BasicBlock *InsertAtEnd);
546
547 // allocate space for exactly three operands
548 void *operator new(size_t S) { return User::operator new(S, 3); }
549 void operator delete(void *Ptr) { User::operator delete(Ptr); }
550
551 using VolatileField = BoolBitfieldElementT<0>;
552 using WeakField = BoolBitfieldElementT<VolatileField::NextBit>;
553 using SuccessOrderingField =
554 AtomicOrderingBitfieldElementT<WeakField::NextBit>;
555 using FailureOrderingField =
556 AtomicOrderingBitfieldElementT<SuccessOrderingField::NextBit>;
557 using AlignmentField =
558 AlignmentBitfieldElementT<FailureOrderingField::NextBit>;
559 static_assert(
560 Bitfield::areContiguous<VolatileField, WeakField, SuccessOrderingField,
561 FailureOrderingField, AlignmentField>(),
562 "Bitfields must be contiguous");
563
564 /// Return the alignment of the memory that is being allocated by the
565 /// instruction.
566 Align getAlign() const {
567 return Align(1ULL << getSubclassData<AlignmentField>());
568 }
569
570 void setAlignment(Align Align) {
571 setSubclassData<AlignmentField>(Log2(Align));
572 }
573
574 /// Return true if this is a cmpxchg from a volatile memory
575 /// location.
576 ///
577 bool isVolatile() const { return getSubclassData<VolatileField>(); }
578
579 /// Specify whether this is a volatile cmpxchg.
580 ///
581 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
582
583 /// Return true if this cmpxchg may spuriously fail.
584 bool isWeak() const { return getSubclassData<WeakField>(); }
585
586 void setWeak(bool IsWeak) { setSubclassData<WeakField>(IsWeak); }
587
588 /// Transparently provide more efficient getOperand methods.
589 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
590
591 static bool isValidSuccessOrdering(AtomicOrdering Ordering) {
592 return Ordering != AtomicOrdering::NotAtomic &&
593 Ordering != AtomicOrdering::Unordered;
594 }
595
596 static bool isValidFailureOrdering(AtomicOrdering Ordering) {
597 return Ordering != AtomicOrdering::NotAtomic &&
598 Ordering != AtomicOrdering::Unordered &&
599 Ordering != AtomicOrdering::AcquireRelease &&
600 Ordering != AtomicOrdering::Release;
601 }
602
603 /// Returns the success ordering constraint of this cmpxchg instruction.
604 AtomicOrdering getSuccessOrdering() const {
605 return getSubclassData<SuccessOrderingField>();
606 }
607
608 /// Sets the success ordering constraint of this cmpxchg instruction.
609 void setSuccessOrdering(AtomicOrdering Ordering) {
610 assert(isValidSuccessOrdering(Ordering) &&(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 611, __extension__ __PRETTY_FUNCTION__))
611 "invalid CmpXchg success ordering")(static_cast <bool> (isValidSuccessOrdering(Ordering) &&
"invalid CmpXchg success ordering") ? void (0) : __assert_fail
("isValidSuccessOrdering(Ordering) && \"invalid CmpXchg success ordering\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 611, __extension__ __PRETTY_FUNCTION__))
;
612 setSubclassData<SuccessOrderingField>(Ordering);
613 }
614
615 /// Returns the failure ordering constraint of this cmpxchg instruction.
616 AtomicOrdering getFailureOrdering() const {
617 return getSubclassData<FailureOrderingField>();
618 }
619
620 /// Sets the failure ordering constraint of this cmpxchg instruction.
621 void setFailureOrdering(AtomicOrdering Ordering) {
622 assert(isValidFailureOrdering(Ordering) &&(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 623, __extension__ __PRETTY_FUNCTION__))
623 "invalid CmpXchg failure ordering")(static_cast <bool> (isValidFailureOrdering(Ordering) &&
"invalid CmpXchg failure ordering") ? void (0) : __assert_fail
("isValidFailureOrdering(Ordering) && \"invalid CmpXchg failure ordering\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 623, __extension__ __PRETTY_FUNCTION__))
;
624 setSubclassData<FailureOrderingField>(Ordering);
625 }
626
627 /// Returns a single ordering which is at least as strong as both the
628 /// success and failure orderings for this cmpxchg.
629 AtomicOrdering getMergedOrdering() const {
630 if (getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
631 return AtomicOrdering::SequentiallyConsistent;
632 if (getFailureOrdering() == AtomicOrdering::Acquire) {
633 if (getSuccessOrdering() == AtomicOrdering::Monotonic)
634 return AtomicOrdering::Acquire;
635 if (getSuccessOrdering() == AtomicOrdering::Release)
636 return AtomicOrdering::AcquireRelease;
637 }
638 return getSuccessOrdering();
639 }
640
641 /// Returns the synchronization scope ID of this cmpxchg instruction.
642 SyncScope::ID getSyncScopeID() const {
643 return SSID;
644 }
645
646 /// Sets the synchronization scope ID of this cmpxchg instruction.
647 void setSyncScopeID(SyncScope::ID SSID) {
648 this->SSID = SSID;
649 }
650
651 Value *getPointerOperand() { return getOperand(0); }
652 const Value *getPointerOperand() const { return getOperand(0); }
653 static unsigned getPointerOperandIndex() { return 0U; }
654
655 Value *getCompareOperand() { return getOperand(1); }
656 const Value *getCompareOperand() const { return getOperand(1); }
657
658 Value *getNewValOperand() { return getOperand(2); }
659 const Value *getNewValOperand() const { return getOperand(2); }
660
661 /// Returns the address space of the pointer operand.
662 unsigned getPointerAddressSpace() const {
663 return getPointerOperand()->getType()->getPointerAddressSpace();
664 }
665
666 /// Returns the strongest permitted ordering on failure, given the
667 /// desired ordering on success.
668 ///
669 /// If the comparison in a cmpxchg operation fails, there is no atomic store
670 /// so release semantics cannot be provided. So this function drops explicit
671 /// Release requests from the AtomicOrdering. A SequentiallyConsistent
672 /// operation would remain SequentiallyConsistent.
673 static AtomicOrdering
674 getStrongestFailureOrdering(AtomicOrdering SuccessOrdering) {
675 switch (SuccessOrdering) {
676 default:
677 llvm_unreachable("invalid cmpxchg success ordering")::llvm::llvm_unreachable_internal("invalid cmpxchg success ordering"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 677)
;
678 case AtomicOrdering::Release:
679 case AtomicOrdering::Monotonic:
680 return AtomicOrdering::Monotonic;
681 case AtomicOrdering::AcquireRelease:
682 case AtomicOrdering::Acquire:
683 return AtomicOrdering::Acquire;
684 case AtomicOrdering::SequentiallyConsistent:
685 return AtomicOrdering::SequentiallyConsistent;
686 }
687 }
688
689 // Methods for support type inquiry through isa, cast, and dyn_cast:
690 static bool classof(const Instruction *I) {
691 return I->getOpcode() == Instruction::AtomicCmpXchg;
692 }
693 static bool classof(const Value *V) {
694 return isa<Instruction>(V) && classof(cast<Instruction>(V));
695 }
696
697private:
698 // Shadow Instruction::setInstructionSubclassData with a private forwarding
699 // method so that subclasses cannot accidentally use it.
700 template <typename Bitfield>
701 void setSubclassData(typename Bitfield::Type Value) {
702 Instruction::setSubclassData<Bitfield>(Value);
703 }
704
705 /// The synchronization scope ID of this cmpxchg instruction. Not quite
706 /// enough room in SubClassData for everything, so synchronization scope ID
707 /// gets its own field.
708 SyncScope::ID SSID;
709};
710
711template <>
712struct OperandTraits<AtomicCmpXchgInst> :
713 public FixedNumOperandTraits<AtomicCmpXchgInst, 3> {
714};
715
716DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicCmpXchgInst, Value)AtomicCmpXchgInst::op_iterator AtomicCmpXchgInst::op_begin() {
return OperandTraits<AtomicCmpXchgInst>::op_begin(this
); } AtomicCmpXchgInst::const_op_iterator AtomicCmpXchgInst::
op_begin() const { return OperandTraits<AtomicCmpXchgInst>
::op_begin(const_cast<AtomicCmpXchgInst*>(this)); } AtomicCmpXchgInst
::op_iterator AtomicCmpXchgInst::op_end() { return OperandTraits
<AtomicCmpXchgInst>::op_end(this); } AtomicCmpXchgInst::
const_op_iterator AtomicCmpXchgInst::op_end() const { return OperandTraits
<AtomicCmpXchgInst>::op_end(const_cast<AtomicCmpXchgInst
*>(this)); } Value *AtomicCmpXchgInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 716, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<AtomicCmpXchgInst>::op_begin
(const_cast<AtomicCmpXchgInst*>(this))[i_nocapture].get
()); } void AtomicCmpXchgInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicCmpXchgInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicCmpXchgInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 716, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
AtomicCmpXchgInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned AtomicCmpXchgInst::getNumOperands() const { return
OperandTraits<AtomicCmpXchgInst>::operands(this); } template
<int Idx_nocapture> Use &AtomicCmpXchgInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &AtomicCmpXchgInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
717
718//===----------------------------------------------------------------------===//
719// AtomicRMWInst Class
720//===----------------------------------------------------------------------===//
721
722/// an instruction that atomically reads a memory location,
723/// combines it with another value, and then stores the result back. Returns
724/// the old value.
725///
726class AtomicRMWInst : public Instruction {
727protected:
728 // Note: Instruction needs to be a friend here to call cloneImpl.
729 friend class Instruction;
730
731 AtomicRMWInst *cloneImpl() const;
732
733public:
734 /// This enumeration lists the possible modifications atomicrmw can make. In
735 /// the descriptions, 'p' is the pointer to the instruction's memory location,
736 /// 'old' is the initial value of *p, and 'v' is the other value passed to the
737 /// instruction. These instructions always return 'old'.
738 enum BinOp : unsigned {
739 /// *p = v
740 Xchg,
741 /// *p = old + v
742 Add,
743 /// *p = old - v
744 Sub,
745 /// *p = old & v
746 And,
747 /// *p = ~(old & v)
748 Nand,
749 /// *p = old | v
750 Or,
751 /// *p = old ^ v
752 Xor,
753 /// *p = old >signed v ? old : v
754 Max,
755 /// *p = old <signed v ? old : v
756 Min,
757 /// *p = old >unsigned v ? old : v
758 UMax,
759 /// *p = old <unsigned v ? old : v
760 UMin,
761
762 /// *p = old + v
763 FAdd,
764
765 /// *p = old - v
766 FSub,
767
768 FIRST_BINOP = Xchg,
769 LAST_BINOP = FSub,
770 BAD_BINOP
771 };
772
773private:
774 template <unsigned Offset>
775 using AtomicOrderingBitfieldElement =
776 typename Bitfield::Element<AtomicOrdering, Offset, 3,
777 AtomicOrdering::LAST>;
778
779 template <unsigned Offset>
780 using BinOpBitfieldElement =
781 typename Bitfield::Element<BinOp, Offset, 4, BinOp::LAST_BINOP>;
782
783public:
784 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
785 AtomicOrdering Ordering, SyncScope::ID SSID,
786 Instruction *InsertBefore = nullptr);
787 AtomicRMWInst(BinOp Operation, Value *Ptr, Value *Val, Align Alignment,
788 AtomicOrdering Ordering, SyncScope::ID SSID,
789 BasicBlock *InsertAtEnd);
790
791 // allocate space for exactly two operands
792 void *operator new(size_t S) { return User::operator new(S, 2); }
793 void operator delete(void *Ptr) { User::operator delete(Ptr); }
794
795 using VolatileField = BoolBitfieldElementT<0>;
796 using AtomicOrderingField =
797 AtomicOrderingBitfieldElementT<VolatileField::NextBit>;
798 using OperationField = BinOpBitfieldElement<AtomicOrderingField::NextBit>;
799 using AlignmentField = AlignmentBitfieldElementT<OperationField::NextBit>;
800 static_assert(Bitfield::areContiguous<VolatileField, AtomicOrderingField,
801 OperationField, AlignmentField>(),
802 "Bitfields must be contiguous");
803
804 BinOp getOperation() const { return getSubclassData<OperationField>(); }
805
806 static StringRef getOperationName(BinOp Op);
807
808 static bool isFPOperation(BinOp Op) {
809 switch (Op) {
810 case AtomicRMWInst::FAdd:
811 case AtomicRMWInst::FSub:
812 return true;
813 default:
814 return false;
815 }
816 }
817
818 void setOperation(BinOp Operation) {
819 setSubclassData<OperationField>(Operation);
820 }
821
822 /// Return the alignment of the memory that is being allocated by the
823 /// instruction.
824 Align getAlign() const {
825 return Align(1ULL << getSubclassData<AlignmentField>());
826 }
827
828 void setAlignment(Align Align) {
829 setSubclassData<AlignmentField>(Log2(Align));
830 }
831
832 /// Return true if this is a RMW on a volatile memory location.
833 ///
834 bool isVolatile() const { return getSubclassData<VolatileField>(); }
835
836 /// Specify whether this is a volatile RMW or not.
837 ///
838 void setVolatile(bool V) { setSubclassData<VolatileField>(V); }
839
840 /// Transparently provide more efficient getOperand methods.
841 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
842
843 /// Returns the ordering constraint of this rmw instruction.
844 AtomicOrdering getOrdering() const {
845 return getSubclassData<AtomicOrderingField>();
846 }
847
848 /// Sets the ordering constraint of this rmw instruction.
849 void setOrdering(AtomicOrdering Ordering) {
850 assert(Ordering != AtomicOrdering::NotAtomic &&(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 851, __extension__ __PRETTY_FUNCTION__))
851 "atomicrmw instructions can only be atomic.")(static_cast <bool> (Ordering != AtomicOrdering::NotAtomic
&& "atomicrmw instructions can only be atomic.") ? void
(0) : __assert_fail ("Ordering != AtomicOrdering::NotAtomic && \"atomicrmw instructions can only be atomic.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 851, __extension__ __PRETTY_FUNCTION__))
;
852 setSubclassData<AtomicOrderingField>(Ordering);
853 }
854
855 /// Returns the synchronization scope ID of this rmw instruction.
856 SyncScope::ID getSyncScopeID() const {
857 return SSID;
858 }
859
860 /// Sets the synchronization scope ID of this rmw instruction.
861 void setSyncScopeID(SyncScope::ID SSID) {
862 this->SSID = SSID;
863 }
864
865 Value *getPointerOperand() { return getOperand(0); }
866 const Value *getPointerOperand() const { return getOperand(0); }
867 static unsigned getPointerOperandIndex() { return 0U; }
868
869 Value *getValOperand() { return getOperand(1); }
870 const Value *getValOperand() const { return getOperand(1); }
871
872 /// Returns the address space of the pointer operand.
873 unsigned getPointerAddressSpace() const {
874 return getPointerOperand()->getType()->getPointerAddressSpace();
875 }
876
877 bool isFloatingPointOperation() const {
878 return isFPOperation(getOperation());
879 }
880
881 // Methods for support type inquiry through isa, cast, and dyn_cast:
882 static bool classof(const Instruction *I) {
883 return I->getOpcode() == Instruction::AtomicRMW;
884 }
885 static bool classof(const Value *V) {
886 return isa<Instruction>(V) && classof(cast<Instruction>(V));
887 }
888
889private:
890 void Init(BinOp Operation, Value *Ptr, Value *Val, Align Align,
891 AtomicOrdering Ordering, SyncScope::ID SSID);
892
893 // Shadow Instruction::setInstructionSubclassData with a private forwarding
894 // method so that subclasses cannot accidentally use it.
895 template <typename Bitfield>
896 void setSubclassData(typename Bitfield::Type Value) {
897 Instruction::setSubclassData<Bitfield>(Value);
898 }
899
900 /// The synchronization scope ID of this rmw instruction. Not quite enough
901 /// room in SubClassData for everything, so synchronization scope ID gets its
902 /// own field.
903 SyncScope::ID SSID;
904};
905
906template <>
907struct OperandTraits<AtomicRMWInst>
908 : public FixedNumOperandTraits<AtomicRMWInst,2> {
909};
910
911DEFINE_TRANSPARENT_OPERAND_ACCESSORS(AtomicRMWInst, Value)AtomicRMWInst::op_iterator AtomicRMWInst::op_begin() { return
OperandTraits<AtomicRMWInst>::op_begin(this); } AtomicRMWInst
::const_op_iterator AtomicRMWInst::op_begin() const { return OperandTraits
<AtomicRMWInst>::op_begin(const_cast<AtomicRMWInst*>
(this)); } AtomicRMWInst::op_iterator AtomicRMWInst::op_end()
{ return OperandTraits<AtomicRMWInst>::op_end(this); }
AtomicRMWInst::const_op_iterator AtomicRMWInst::op_end() const
{ return OperandTraits<AtomicRMWInst>::op_end(const_cast
<AtomicRMWInst*>(this)); } Value *AtomicRMWInst::getOperand
(unsigned i_nocapture) const { (static_cast <bool> (i_nocapture
< OperandTraits<AtomicRMWInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 911, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<AtomicRMWInst>::op_begin(const_cast
<AtomicRMWInst*>(this))[i_nocapture].get()); } void AtomicRMWInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<AtomicRMWInst
>::operands(this) && "setOperand() out of range!")
? void (0) : __assert_fail ("i_nocapture < OperandTraits<AtomicRMWInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 911, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
AtomicRMWInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned AtomicRMWInst::getNumOperands() const { return OperandTraits
<AtomicRMWInst>::operands(this); } template <int Idx_nocapture
> Use &AtomicRMWInst::Op() { return this->OpFrom<
Idx_nocapture>(this); } template <int Idx_nocapture>
const Use &AtomicRMWInst::Op() const { return this->OpFrom
<Idx_nocapture>(this); }
912
913//===----------------------------------------------------------------------===//
914// GetElementPtrInst Class
915//===----------------------------------------------------------------------===//
916
917// checkGEPType - Simple wrapper function to give a better assertion failure
918// message on bad indexes for a gep instruction.
919//
920inline Type *checkGEPType(Type *Ty) {
921 assert(Ty && "Invalid GetElementPtrInst indices for type!")(static_cast <bool> (Ty && "Invalid GetElementPtrInst indices for type!"
) ? void (0) : __assert_fail ("Ty && \"Invalid GetElementPtrInst indices for type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 921, __extension__ __PRETTY_FUNCTION__))
;
922 return Ty;
923}
924
925/// an instruction for type-safe pointer arithmetic to
926/// access elements of arrays and structs
927///
928class GetElementPtrInst : public Instruction {
929 Type *SourceElementType;
930 Type *ResultElementType;
931
932 GetElementPtrInst(const GetElementPtrInst &GEPI);
933
934 /// Constructors - Create a getelementptr instruction with a base pointer an
935 /// list of indices. The first ctor can optionally insert before an existing
936 /// instruction, the second appends the new instruction to the specified
937 /// BasicBlock.
938 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
939 ArrayRef<Value *> IdxList, unsigned Values,
940 const Twine &NameStr, Instruction *InsertBefore);
941 inline GetElementPtrInst(Type *PointeeType, Value *Ptr,
942 ArrayRef<Value *> IdxList, unsigned Values,
943 const Twine &NameStr, BasicBlock *InsertAtEnd);
944
945 void init(Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr);
946
947protected:
948 // Note: Instruction needs to be a friend here to call cloneImpl.
949 friend class Instruction;
950
951 GetElementPtrInst *cloneImpl() const;
952
953public:
954 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
955 ArrayRef<Value *> IdxList,
956 const Twine &NameStr = "",
957 Instruction *InsertBefore = nullptr) {
958 unsigned Values = 1 + unsigned(IdxList.size());
959 assert(PointeeType && "Must specify element type")(static_cast <bool> (PointeeType && "Must specify element type"
) ? void (0) : __assert_fail ("PointeeType && \"Must specify element type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 959, __extension__ __PRETTY_FUNCTION__))
;
960 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 961, __extension__ __PRETTY_FUNCTION__))
961 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 961, __extension__ __PRETTY_FUNCTION__))
;
962 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
963 NameStr, InsertBefore);
964 }
965
966 static GetElementPtrInst *Create(Type *PointeeType, Value *Ptr,
967 ArrayRef<Value *> IdxList,
968 const Twine &NameStr,
969 BasicBlock *InsertAtEnd) {
970 unsigned Values = 1 + unsigned(IdxList.size());
971 assert(PointeeType && "Must specify element type")(static_cast <bool> (PointeeType && "Must specify element type"
) ? void (0) : __assert_fail ("PointeeType && \"Must specify element type\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 971, __extension__ __PRETTY_FUNCTION__))
;
972 assert(cast<PointerType>(Ptr->getType()->getScalarType())(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 973, __extension__ __PRETTY_FUNCTION__))
973 ->isOpaqueOrPointeeTypeMatches(PointeeType))(static_cast <bool> (cast<PointerType>(Ptr->getType
()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType
)) ? void (0) : __assert_fail ("cast<PointerType>(Ptr->getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(PointeeType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 973, __extension__ __PRETTY_FUNCTION__))
;
974 return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values,
975 NameStr, InsertAtEnd);
976 }
977
978 LLVM_ATTRIBUTE_DEPRECATED(static GetElementPtrInst *CreateInBounds([[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
979 Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr = "",[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
980 Instruction *InsertBefore = nullptr),[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
981 "Use the version with explicit element type instead")[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr = "", Instruction
*InsertBefore = nullptr)
{
982 return CreateInBounds(
983 Ptr->getType()->getScalarType()->getPointerElementType(), Ptr, IdxList,
984 NameStr, InsertBefore);
985 }
986
987 /// Create an "inbounds" getelementptr. See the documentation for the
988 /// "inbounds" flag in LangRef.html for details.
989 static GetElementPtrInst *
990 CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef<Value *> IdxList,
991 const Twine &NameStr = "",
992 Instruction *InsertBefore = nullptr) {
993 GetElementPtrInst *GEP =
994 Create(PointeeType, Ptr, IdxList, NameStr, InsertBefore);
995 GEP->setIsInBounds(true);
996 return GEP;
997 }
998
999 LLVM_ATTRIBUTE_DEPRECATED(static GetElementPtrInst *CreateInBounds([[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
1000 Value *Ptr, ArrayRef<Value *> IdxList, const Twine &NameStr,[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
1001 BasicBlock *InsertAtEnd),[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
1002 "Use the version with explicit element type instead")[[deprecated("Use the version with explicit element type instead"
)]] static GetElementPtrInst *CreateInBounds( Value *Ptr, ArrayRef
<Value *> IdxList, const Twine &NameStr, BasicBlock
*InsertAtEnd)
{
1003 return CreateInBounds(
1004 Ptr->getType()->getScalarType()->getPointerElementType(), Ptr, IdxList,
1005 NameStr, InsertAtEnd);
1006 }
1007
1008 static GetElementPtrInst *CreateInBounds(Type *PointeeType, Value *Ptr,
1009 ArrayRef<Value *> IdxList,
1010 const Twine &NameStr,
1011 BasicBlock *InsertAtEnd) {
1012 GetElementPtrInst *GEP =
1013 Create(PointeeType, Ptr, IdxList, NameStr, InsertAtEnd);
1014 GEP->setIsInBounds(true);
1015 return GEP;
1016 }
1017
1018 /// Transparently provide more efficient getOperand methods.
1019 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1020
1021 Type *getSourceElementType() const { return SourceElementType; }
1022
1023 void setSourceElementType(Type *Ty) { SourceElementType = Ty; }
1024 void setResultElementType(Type *Ty) { ResultElementType = Ty; }
1025
1026 Type *getResultElementType() const {
1027 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1028, __extension__ __PRETTY_FUNCTION__))
1028 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1028, __extension__ __PRETTY_FUNCTION__))
;
1029 return ResultElementType;
1030 }
1031
1032 /// Returns the address space of this instruction's pointer type.
1033 unsigned getAddressSpace() const {
1034 // Note that this is always the same as the pointer operand's address space
1035 // and that is cheaper to compute, so cheat here.
1036 return getPointerAddressSpace();
1037 }
1038
1039 /// Returns the result type of a getelementptr with the given source
1040 /// element type and indexes.
1041 ///
1042 /// Null is returned if the indices are invalid for the specified
1043 /// source element type.
1044 static Type *getIndexedType(Type *Ty, ArrayRef<Value *> IdxList);
1045 static Type *getIndexedType(Type *Ty, ArrayRef<Constant *> IdxList);
1046 static Type *getIndexedType(Type *Ty, ArrayRef<uint64_t> IdxList);
1047
1048 /// Return the type of the element at the given index of an indexable
1049 /// type. This is equivalent to "getIndexedType(Agg, {Zero, Idx})".
1050 ///
1051 /// Returns null if the type can't be indexed, or the given index is not
1052 /// legal for the given type.
1053 static Type *getTypeAtIndex(Type *Ty, Value *Idx);
1054 static Type *getTypeAtIndex(Type *Ty, uint64_t Idx);
1055
1056 inline op_iterator idx_begin() { return op_begin()+1; }
1057 inline const_op_iterator idx_begin() const { return op_begin()+1; }
1058 inline op_iterator idx_end() { return op_end(); }
1059 inline const_op_iterator idx_end() const { return op_end(); }
1060
1061 inline iterator_range<op_iterator> indices() {
1062 return make_range(idx_begin(), idx_end());
1063 }
1064
1065 inline iterator_range<const_op_iterator> indices() const {
1066 return make_range(idx_begin(), idx_end());
1067 }
1068
1069 Value *getPointerOperand() {
1070 return getOperand(0);
1071 }
1072 const Value *getPointerOperand() const {
1073 return getOperand(0);
1074 }
1075 static unsigned getPointerOperandIndex() {
1076 return 0U; // get index for modifying correct operand.
1077 }
1078
1079 /// Method to return the pointer operand as a
1080 /// PointerType.
1081 Type *getPointerOperandType() const {
1082 return getPointerOperand()->getType();
1083 }
1084
1085 /// Returns the address space of the pointer operand.
1086 unsigned getPointerAddressSpace() const {
1087 return getPointerOperandType()->getPointerAddressSpace();
1088 }
1089
1090 /// Returns the pointer type returned by the GEP
1091 /// instruction, which may be a vector of pointers.
1092 static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
1093 ArrayRef<Value *> IdxList) {
1094 PointerType *OrigPtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
1095 unsigned AddrSpace = OrigPtrTy->getAddressSpace();
1096 Type *ResultElemTy = checkGEPType(getIndexedType(ElTy, IdxList));
1097 Type *PtrTy = OrigPtrTy->isOpaque()
1098 ? PointerType::get(OrigPtrTy->getContext(), AddrSpace)
1099 : PointerType::get(ResultElemTy, AddrSpace);
1100 // Vector GEP
1101 if (auto *PtrVTy = dyn_cast<VectorType>(Ptr->getType())) {
1102 ElementCount EltCount = PtrVTy->getElementCount();
1103 return VectorType::get(PtrTy, EltCount);
1104 }
1105 for (Value *Index : IdxList)
1106 if (auto *IndexVTy = dyn_cast<VectorType>(Index->getType())) {
1107 ElementCount EltCount = IndexVTy->getElementCount();
1108 return VectorType::get(PtrTy, EltCount);
1109 }
1110 // Scalar GEP
1111 return PtrTy;
1112 }
1113
1114 unsigned getNumIndices() const { // Note: always non-negative
1115 return getNumOperands() - 1;
1116 }
1117
1118 bool hasIndices() const {
1119 return getNumOperands() > 1;
1120 }
1121
1122 /// Return true if all of the indices of this GEP are
1123 /// zeros. If so, the result pointer and the first operand have the same
1124 /// value, just potentially different types.
1125 bool hasAllZeroIndices() const;
1126
1127 /// Return true if all of the indices of this GEP are
1128 /// constant integers. If so, the result pointer and the first operand have
1129 /// a constant offset between them.
1130 bool hasAllConstantIndices() const;
1131
1132 /// Set or clear the inbounds flag on this GEP instruction.
1133 /// See LangRef.html for the meaning of inbounds on a getelementptr.
1134 void setIsInBounds(bool b = true);
1135
1136 /// Determine whether the GEP has the inbounds flag.
1137 bool isInBounds() const;
1138
1139 /// Accumulate the constant address offset of this GEP if possible.
1140 ///
1141 /// This routine accepts an APInt into which it will accumulate the constant
1142 /// offset of this GEP if the GEP is in fact constant. If the GEP is not
1143 /// all-constant, it returns false and the value of the offset APInt is
1144 /// undefined (it is *not* preserved!). The APInt passed into this routine
1145 /// must be at least as wide as the IntPtr type for the address space of
1146 /// the base GEP pointer.
1147 bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const;
1148 bool collectOffset(const DataLayout &DL, unsigned BitWidth,
1149 MapVector<Value *, APInt> &VariableOffsets,
1150 APInt &ConstantOffset) const;
1151 // Methods for support type inquiry through isa, cast, and dyn_cast:
1152 static bool classof(const Instruction *I) {
1153 return (I->getOpcode() == Instruction::GetElementPtr);
1154 }
1155 static bool classof(const Value *V) {
1156 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1157 }
1158};
1159
1160template <>
1161struct OperandTraits<GetElementPtrInst> :
1162 public VariadicOperandTraits<GetElementPtrInst, 1> {
1163};
1164
1165GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1166 ArrayRef<Value *> IdxList, unsigned Values,
1167 const Twine &NameStr,
1168 Instruction *InsertBefore)
1169 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1170 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1171 Values, InsertBefore),
1172 SourceElementType(PointeeType),
1173 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1174 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1175, __extension__ __PRETTY_FUNCTION__))
1175 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1175, __extension__ __PRETTY_FUNCTION__))
;
1176 init(Ptr, IdxList, NameStr);
1177}
1178
1179GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr,
1180 ArrayRef<Value *> IdxList, unsigned Values,
1181 const Twine &NameStr,
1182 BasicBlock *InsertAtEnd)
1183 : Instruction(getGEPReturnType(PointeeType, Ptr, IdxList), GetElementPtr,
1184 OperandTraits<GetElementPtrInst>::op_end(this) - Values,
1185 Values, InsertAtEnd),
1186 SourceElementType(PointeeType),
1187 ResultElementType(getIndexedType(PointeeType, IdxList)) {
1188 assert(cast<PointerType>(getType()->getScalarType())(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1189, __extension__ __PRETTY_FUNCTION__))
1189 ->isOpaqueOrPointeeTypeMatches(ResultElementType))(static_cast <bool> (cast<PointerType>(getType()->
getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType
)) ? void (0) : __assert_fail ("cast<PointerType>(getType()->getScalarType()) ->isOpaqueOrPointeeTypeMatches(ResultElementType)"
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1189, __extension__ __PRETTY_FUNCTION__))
;
1190 init(Ptr, IdxList, NameStr);
1191}
1192
1193DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value)GetElementPtrInst::op_iterator GetElementPtrInst::op_begin() {
return OperandTraits<GetElementPtrInst>::op_begin(this
); } GetElementPtrInst::const_op_iterator GetElementPtrInst::
op_begin() const { return OperandTraits<GetElementPtrInst>
::op_begin(const_cast<GetElementPtrInst*>(this)); } GetElementPtrInst
::op_iterator GetElementPtrInst::op_end() { return OperandTraits
<GetElementPtrInst>::op_end(this); } GetElementPtrInst::
const_op_iterator GetElementPtrInst::op_end() const { return OperandTraits
<GetElementPtrInst>::op_end(const_cast<GetElementPtrInst
*>(this)); } Value *GetElementPtrInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<GetElementPtrInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1193, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<GetElementPtrInst>::op_begin
(const_cast<GetElementPtrInst*>(this))[i_nocapture].get
()); } void GetElementPtrInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<GetElementPtrInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<GetElementPtrInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1193, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
GetElementPtrInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned GetElementPtrInst::getNumOperands() const { return
OperandTraits<GetElementPtrInst>::operands(this); } template
<int Idx_nocapture> Use &GetElementPtrInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &GetElementPtrInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1194
1195//===----------------------------------------------------------------------===//
1196// ICmpInst Class
1197//===----------------------------------------------------------------------===//
1198
1199/// This instruction compares its operands according to the predicate given
1200/// to the constructor. It only operates on integers or pointers. The operands
1201/// must be identical types.
1202/// Represent an integer comparison operator.
1203class ICmpInst: public CmpInst {
1204 void AssertOK() {
1205 assert(isIntPredicate() &&(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1206, __extension__ __PRETTY_FUNCTION__))
1206 "Invalid ICmp predicate value")(static_cast <bool> (isIntPredicate() && "Invalid ICmp predicate value"
) ? void (0) : __assert_fail ("isIntPredicate() && \"Invalid ICmp predicate value\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1206, __extension__ __PRETTY_FUNCTION__))
;
1207 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1208, __extension__ __PRETTY_FUNCTION__))
1208 "Both operands to ICmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to ICmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to ICmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1208, __extension__ __PRETTY_FUNCTION__))
;
1209 // Check that the operands are the right type
1210 assert((getOperand(0)->getType()->isIntOrIntVectorTy() ||(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1212, __extension__ __PRETTY_FUNCTION__))
1211 getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1212, __extension__ __PRETTY_FUNCTION__))
1212 "Invalid operand types for ICmp instruction")(static_cast <bool> ((getOperand(0)->getType()->isIntOrIntVectorTy
() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) &&
"Invalid operand types for ICmp instruction") ? void (0) : __assert_fail
("(getOperand(0)->getType()->isIntOrIntVectorTy() || getOperand(0)->getType()->isPtrOrPtrVectorTy()) && \"Invalid operand types for ICmp instruction\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1212, __extension__ __PRETTY_FUNCTION__))
;
1213 }
1214
1215protected:
1216 // Note: Instruction needs to be a friend here to call cloneImpl.
1217 friend class Instruction;
1218
1219 /// Clone an identical ICmpInst
1220 ICmpInst *cloneImpl() const;
1221
1222public:
1223 /// Constructor with insert-before-instruction semantics.
1224 ICmpInst(
1225 Instruction *InsertBefore, ///< Where to insert
1226 Predicate pred, ///< The predicate to use for the comparison
1227 Value *LHS, ///< The left-hand-side of the expression
1228 Value *RHS, ///< The right-hand-side of the expression
1229 const Twine &NameStr = "" ///< Name of the instruction
1230 ) : CmpInst(makeCmpResultType(LHS->getType()),
1231 Instruction::ICmp, pred, LHS, RHS, NameStr,
1232 InsertBefore) {
1233#ifndef NDEBUG
1234 AssertOK();
1235#endif
1236 }
1237
1238 /// Constructor with insert-at-end semantics.
1239 ICmpInst(
1240 BasicBlock &InsertAtEnd, ///< Block to insert into.
1241 Predicate pred, ///< The predicate to use for the comparison
1242 Value *LHS, ///< The left-hand-side of the expression
1243 Value *RHS, ///< The right-hand-side of the expression
1244 const Twine &NameStr = "" ///< Name of the instruction
1245 ) : CmpInst(makeCmpResultType(LHS->getType()),
1246 Instruction::ICmp, pred, LHS, RHS, NameStr,
1247 &InsertAtEnd) {
1248#ifndef NDEBUG
1249 AssertOK();
1250#endif
1251 }
1252
1253 /// Constructor with no-insertion semantics
1254 ICmpInst(
1255 Predicate pred, ///< The predicate to use for the comparison
1256 Value *LHS, ///< The left-hand-side of the expression
1257 Value *RHS, ///< The right-hand-side of the expression
1258 const Twine &NameStr = "" ///< Name of the instruction
1259 ) : CmpInst(makeCmpResultType(LHS->getType()),
1260 Instruction::ICmp, pred, LHS, RHS, NameStr) {
1261#ifndef NDEBUG
1262 AssertOK();
1263#endif
1264 }
1265
1266 /// For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
1267 /// @returns the predicate that would be the result if the operand were
1268 /// regarded as signed.
1269 /// Return the signed version of the predicate
1270 Predicate getSignedPredicate() const {
1271 return getSignedPredicate(getPredicate());
1272 }
1273
1274 /// This is a static version that you can use without an instruction.
1275 /// Return the signed version of the predicate.
1276 static Predicate getSignedPredicate(Predicate pred);
1277
1278 /// For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
1279 /// @returns the predicate that would be the result if the operand were
1280 /// regarded as unsigned.
1281 /// Return the unsigned version of the predicate
1282 Predicate getUnsignedPredicate() const {
1283 return getUnsignedPredicate(getPredicate());
1284 }
1285
1286 /// This is a static version that you can use without an instruction.
1287 /// Return the unsigned version of the predicate.
1288 static Predicate getUnsignedPredicate(Predicate pred);
1289
1290 /// Return true if this predicate is either EQ or NE. This also
1291 /// tests for commutativity.
1292 static bool isEquality(Predicate P) {
1293 return P == ICMP_EQ || P == ICMP_NE;
1294 }
1295
1296 /// Return true if this predicate is either EQ or NE. This also
1297 /// tests for commutativity.
1298 bool isEquality() const {
1299 return isEquality(getPredicate());
1300 }
1301
1302 /// @returns true if the predicate of this ICmpInst is commutative
1303 /// Determine if this relation is commutative.
1304 bool isCommutative() const { return isEquality(); }
1305
1306 /// Return true if the predicate is relational (not EQ or NE).
1307 ///
1308 bool isRelational() const {
1309 return !isEquality();
1310 }
1311
1312 /// Return true if the predicate is relational (not EQ or NE).
1313 ///
1314 static bool isRelational(Predicate P) {
1315 return !isEquality(P);
1316 }
1317
1318 /// Return true if the predicate is SGT or UGT.
1319 ///
1320 static bool isGT(Predicate P) {
1321 return P == ICMP_SGT || P == ICMP_UGT;
1322 }
1323
1324 /// Return true if the predicate is SLT or ULT.
1325 ///
1326 static bool isLT(Predicate P) {
1327 return P == ICMP_SLT || P == ICMP_ULT;
1328 }
1329
1330 /// Return true if the predicate is SGE or UGE.
1331 ///
1332 static bool isGE(Predicate P) {
1333 return P == ICMP_SGE || P == ICMP_UGE;
1334 }
1335
1336 /// Return true if the predicate is SLE or ULE.
1337 ///
1338 static bool isLE(Predicate P) {
1339 return P == ICMP_SLE || P == ICMP_ULE;
1340 }
1341
1342 /// Exchange the two operands to this instruction in such a way that it does
1343 /// not modify the semantics of the instruction. The predicate value may be
1344 /// changed to retain the same result if the predicate is order dependent
1345 /// (e.g. ult).
1346 /// Swap operands and adjust predicate.
1347 void swapOperands() {
1348 setPredicate(getSwappedPredicate());
1349 Op<0>().swap(Op<1>());
1350 }
1351
1352 // Methods for support type inquiry through isa, cast, and dyn_cast:
1353 static bool classof(const Instruction *I) {
1354 return I->getOpcode() == Instruction::ICmp;
1355 }
1356 static bool classof(const Value *V) {
1357 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1358 }
1359};
1360
1361//===----------------------------------------------------------------------===//
1362// FCmpInst Class
1363//===----------------------------------------------------------------------===//
1364
1365/// This instruction compares its operands according to the predicate given
1366/// to the constructor. It only operates on floating point values or packed
1367/// vectors of floating point values. The operands must be identical types.
1368/// Represents a floating point comparison operator.
1369class FCmpInst: public CmpInst {
1370 void AssertOK() {
1371 assert(isFPPredicate() && "Invalid FCmp predicate value")(static_cast <bool> (isFPPredicate() && "Invalid FCmp predicate value"
) ? void (0) : __assert_fail ("isFPPredicate() && \"Invalid FCmp predicate value\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1371, __extension__ __PRETTY_FUNCTION__))
;
1372 assert(getOperand(0)->getType() == getOperand(1)->getType() &&(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1373, __extension__ __PRETTY_FUNCTION__))
1373 "Both operands to FCmp instruction are not of the same type!")(static_cast <bool> (getOperand(0)->getType() == getOperand
(1)->getType() && "Both operands to FCmp instruction are not of the same type!"
) ? void (0) : __assert_fail ("getOperand(0)->getType() == getOperand(1)->getType() && \"Both operands to FCmp instruction are not of the same type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1373, __extension__ __PRETTY_FUNCTION__))
;
1374 // Check that the operands are the right type
1375 assert(getOperand(0)->getType()->isFPOrFPVectorTy() &&(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1376, __extension__ __PRETTY_FUNCTION__))
1376 "Invalid operand types for FCmp instruction")(static_cast <bool> (getOperand(0)->getType()->isFPOrFPVectorTy
() && "Invalid operand types for FCmp instruction") ?
void (0) : __assert_fail ("getOperand(0)->getType()->isFPOrFPVectorTy() && \"Invalid operand types for FCmp instruction\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1376, __extension__ __PRETTY_FUNCTION__))
;
1377 }
1378
1379protected:
1380 // Note: Instruction needs to be a friend here to call cloneImpl.
1381 friend class Instruction;
1382
1383 /// Clone an identical FCmpInst
1384 FCmpInst *cloneImpl() const;
1385
1386public:
1387 /// Constructor with insert-before-instruction semantics.
1388 FCmpInst(
1389 Instruction *InsertBefore, ///< Where to insert
1390 Predicate pred, ///< The predicate to use for the comparison
1391 Value *LHS, ///< The left-hand-side of the expression
1392 Value *RHS, ///< The right-hand-side of the expression
1393 const Twine &NameStr = "" ///< Name of the instruction
1394 ) : CmpInst(makeCmpResultType(LHS->getType()),
1395 Instruction::FCmp, pred, LHS, RHS, NameStr,
1396 InsertBefore) {
1397 AssertOK();
1398 }
1399
1400 /// Constructor with insert-at-end semantics.
1401 FCmpInst(
1402 BasicBlock &InsertAtEnd, ///< Block to insert into.
1403 Predicate pred, ///< The predicate to use for the comparison
1404 Value *LHS, ///< The left-hand-side of the expression
1405 Value *RHS, ///< The right-hand-side of the expression
1406 const Twine &NameStr = "" ///< Name of the instruction
1407 ) : CmpInst(makeCmpResultType(LHS->getType()),
1408 Instruction::FCmp, pred, LHS, RHS, NameStr,
1409 &InsertAtEnd) {
1410 AssertOK();
1411 }
1412
1413 /// Constructor with no-insertion semantics
1414 FCmpInst(
1415 Predicate Pred, ///< The predicate to use for the comparison
1416 Value *LHS, ///< The left-hand-side of the expression
1417 Value *RHS, ///< The right-hand-side of the expression
1418 const Twine &NameStr = "", ///< Name of the instruction
1419 Instruction *FlagsSource = nullptr
1420 ) : CmpInst(makeCmpResultType(LHS->getType()), Instruction::FCmp, Pred, LHS,
1421 RHS, NameStr, nullptr, FlagsSource) {
1422 AssertOK();
1423 }
1424
1425 /// @returns true if the predicate of this instruction is EQ or NE.
1426 /// Determine if this is an equality predicate.
1427 static bool isEquality(Predicate Pred) {
1428 return Pred == FCMP_OEQ || Pred == FCMP_ONE || Pred == FCMP_UEQ ||
1429 Pred == FCMP_UNE;
1430 }
1431
1432 /// @returns true if the predicate of this instruction is EQ or NE.
1433 /// Determine if this is an equality predicate.
1434 bool isEquality() const { return isEquality(getPredicate()); }
1435
1436 /// @returns true if the predicate of this instruction is commutative.
1437 /// Determine if this is a commutative predicate.
1438 bool isCommutative() const {
1439 return isEquality() ||
1440 getPredicate() == FCMP_FALSE ||
1441 getPredicate() == FCMP_TRUE ||
1442 getPredicate() == FCMP_ORD ||
1443 getPredicate() == FCMP_UNO;
1444 }
1445
1446 /// @returns true if the predicate is relational (not EQ or NE).
1447 /// Determine if this a relational predicate.
1448 bool isRelational() const { return !isEquality(); }
1449
1450 /// Exchange the two operands to this instruction in such a way that it does
1451 /// not modify the semantics of the instruction. The predicate value may be
1452 /// changed to retain the same result if the predicate is order dependent
1453 /// (e.g. ult).
1454 /// Swap operands and adjust predicate.
1455 void swapOperands() {
1456 setPredicate(getSwappedPredicate());
1457 Op<0>().swap(Op<1>());
1458 }
1459
1460 /// Methods for support type inquiry through isa, cast, and dyn_cast:
1461 static bool classof(const Instruction *I) {
1462 return I->getOpcode() == Instruction::FCmp;
1463 }
1464 static bool classof(const Value *V) {
1465 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1466 }
1467};
1468
1469//===----------------------------------------------------------------------===//
1470/// This class represents a function call, abstracting a target
1471/// machine's calling convention. This class uses low bit of the SubClassData
1472/// field to indicate whether or not this is a tail call. The rest of the bits
1473/// hold the calling convention of the call.
1474///
1475class CallInst : public CallBase {
1476 CallInst(const CallInst &CI);
1477
1478 /// Construct a CallInst given a range of arguments.
1479 /// Construct a CallInst from a range of arguments
1480 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1481 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1482 Instruction *InsertBefore);
1483
1484 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1485 const Twine &NameStr, Instruction *InsertBefore)
1486 : CallInst(Ty, Func, Args, None, NameStr, InsertBefore) {}
1487
1488 /// Construct a CallInst given a range of arguments.
1489 /// Construct a CallInst from a range of arguments
1490 inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1491 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1492 BasicBlock *InsertAtEnd);
1493
1494 explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
1495 Instruction *InsertBefore);
1496
1497 CallInst(FunctionType *ty, Value *F, const Twine &NameStr,
1498 BasicBlock *InsertAtEnd);
1499
1500 void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
1501 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
1502 void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
1503
1504 /// Compute the number of operands to allocate.
1505 static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
1506 // We need one operand for the called function, plus the input operand
1507 // counts provided.
1508 return 1 + NumArgs + NumBundleInputs;
1509 }
1510
1511protected:
1512 // Note: Instruction needs to be a friend here to call cloneImpl.
1513 friend class Instruction;
1514
1515 CallInst *cloneImpl() const;
1516
1517public:
1518 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
1519 Instruction *InsertBefore = nullptr) {
1520 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertBefore);
1521 }
1522
1523 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1524 const Twine &NameStr,
1525 Instruction *InsertBefore = nullptr) {
1526 return new (ComputeNumOperands(Args.size()))
1527 CallInst(Ty, Func, Args, None, NameStr, InsertBefore);
1528 }
1529
1530 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1531 ArrayRef<OperandBundleDef> Bundles = None,
1532 const Twine &NameStr = "",
1533 Instruction *InsertBefore = nullptr) {
1534 const int NumOperands =
1535 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1536 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1537
1538 return new (NumOperands, DescriptorBytes)
1539 CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
1540 }
1541
1542 static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr,
1543 BasicBlock *InsertAtEnd) {
1544 return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertAtEnd);
1545 }
1546
1547 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1548 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1549 return new (ComputeNumOperands(Args.size()))
1550 CallInst(Ty, Func, Args, None, NameStr, InsertAtEnd);
1551 }
1552
1553 static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1554 ArrayRef<OperandBundleDef> Bundles,
1555 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1556 const int NumOperands =
1557 ComputeNumOperands(Args.size(), CountBundleInputs(Bundles));
1558 const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
1559
1560 return new (NumOperands, DescriptorBytes)
1561 CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd);
1562 }
1563
1564 static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "",
1565 Instruction *InsertBefore = nullptr) {
1566 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1567 InsertBefore);
1568 }
1569
1570 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1571 ArrayRef<OperandBundleDef> Bundles = None,
1572 const Twine &NameStr = "",
1573 Instruction *InsertBefore = nullptr) {
1574 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1575 NameStr, InsertBefore);
1576 }
1577
1578 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1579 const Twine &NameStr,
1580 Instruction *InsertBefore = nullptr) {
1581 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1582 InsertBefore);
1583 }
1584
1585 static CallInst *Create(FunctionCallee Func, const Twine &NameStr,
1586 BasicBlock *InsertAtEnd) {
1587 return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
1588 InsertAtEnd);
1589 }
1590
1591 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1592 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1593 return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
1594 InsertAtEnd);
1595 }
1596
1597 static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
1598 ArrayRef<OperandBundleDef> Bundles,
1599 const Twine &NameStr, BasicBlock *InsertAtEnd) {
1600 return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
1601 NameStr, InsertAtEnd);
1602 }
1603
1604 /// Create a clone of \p CI with a different set of operand bundles and
1605 /// insert it before \p InsertPt.
1606 ///
1607 /// The returned call instruction is identical \p CI in every way except that
1608 /// the operand bundles for the new instruction are set to the operand bundles
1609 /// in \p Bundles.
1610 static CallInst *Create(CallInst *CI, ArrayRef<OperandBundleDef> Bundles,
1611 Instruction *InsertPt = nullptr);
1612
1613 /// Generate the IR for a call to malloc:
1614 /// 1. Compute the malloc call's argument as the specified type's size,
1615 /// possibly multiplied by the array size if the array size is not
1616 /// constant 1.
1617 /// 2. Call malloc with that argument.
1618 /// 3. Bitcast the result of the malloc call to the specified type.
1619 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1620 Type *AllocTy, Value *AllocSize,
1621 Value *ArraySize = nullptr,
1622 Function *MallocF = nullptr,
1623 const Twine &Name = "");
1624 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1625 Type *AllocTy, Value *AllocSize,
1626 Value *ArraySize = nullptr,
1627 Function *MallocF = nullptr,
1628 const Twine &Name = "");
1629 static Instruction *CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy,
1630 Type *AllocTy, Value *AllocSize,
1631 Value *ArraySize = nullptr,
1632 ArrayRef<OperandBundleDef> Bundles = None,
1633 Function *MallocF = nullptr,
1634 const Twine &Name = "");
1635 static Instruction *CreateMalloc(BasicBlock *InsertAtEnd, Type *IntPtrTy,
1636 Type *AllocTy, Value *AllocSize,
1637 Value *ArraySize = nullptr,
1638 ArrayRef<OperandBundleDef> Bundles = None,
1639 Function *MallocF = nullptr,
1640 const Twine &Name = "");
1641 /// Generate the IR for a call to the builtin free function.
1642 static Instruction *CreateFree(Value *Source, Instruction *InsertBefore);
1643 static Instruction *CreateFree(Value *Source, BasicBlock *InsertAtEnd);
1644 static Instruction *CreateFree(Value *Source,
1645 ArrayRef<OperandBundleDef> Bundles,
1646 Instruction *InsertBefore);
1647 static Instruction *CreateFree(Value *Source,
1648 ArrayRef<OperandBundleDef> Bundles,
1649 BasicBlock *InsertAtEnd);
1650
1651 // Note that 'musttail' implies 'tail'.
1652 enum TailCallKind : unsigned {
1653 TCK_None = 0,
1654 TCK_Tail = 1,
1655 TCK_MustTail = 2,
1656 TCK_NoTail = 3,
1657 TCK_LAST = TCK_NoTail
1658 };
1659
1660 using TailCallKindField = Bitfield::Element<TailCallKind, 0, 2, TCK_LAST>;
1661 static_assert(
1662 Bitfield::areContiguous<TailCallKindField, CallBase::CallingConvField>(),
1663 "Bitfields must be contiguous");
1664
1665 TailCallKind getTailCallKind() const {
1666 return getSubclassData<TailCallKindField>();
1667 }
1668
1669 bool isTailCall() const {
1670 TailCallKind Kind = getTailCallKind();
1671 return Kind == TCK_Tail || Kind == TCK_MustTail;
1672 }
1673
1674 bool isMustTailCall() const { return getTailCallKind() == TCK_MustTail; }
1675
1676 bool isNoTailCall() const { return getTailCallKind() == TCK_NoTail; }
1677
1678 void setTailCallKind(TailCallKind TCK) {
1679 setSubclassData<TailCallKindField>(TCK);
1680 }
1681
1682 void setTailCall(bool IsTc = true) {
1683 setTailCallKind(IsTc ? TCK_Tail : TCK_None);
1684 }
1685
1686 /// Return true if the call can return twice
1687 bool canReturnTwice() const { return hasFnAttr(Attribute::ReturnsTwice); }
1688 void setCanReturnTwice() { addFnAttr(Attribute::ReturnsTwice); }
1689
1690 // Methods for support type inquiry through isa, cast, and dyn_cast:
1691 static bool classof(const Instruction *I) {
1692 return I->getOpcode() == Instruction::Call;
1693 }
1694 static bool classof(const Value *V) {
1695 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1696 }
1697
1698 /// Updates profile metadata by scaling it by \p S / \p T.
1699 void updateProfWeight(uint64_t S, uint64_t T);
1700
1701private:
1702 // Shadow Instruction::setInstructionSubclassData with a private forwarding
1703 // method so that subclasses cannot accidentally use it.
1704 template <typename Bitfield>
1705 void setSubclassData(typename Bitfield::Type Value) {
1706 Instruction::setSubclassData<Bitfield>(Value);
1707 }
1708};
1709
1710CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1711 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1712 BasicBlock *InsertAtEnd)
1713 : CallBase(Ty->getReturnType(), Instruction::Call,
1714 OperandTraits<CallBase>::op_end(this) -
1715 (Args.size() + CountBundleInputs(Bundles) + 1),
1716 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1717 InsertAtEnd) {
1718 init(Ty, Func, Args, Bundles, NameStr);
1719}
1720
1721CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
1722 ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
1723 Instruction *InsertBefore)
1724 : CallBase(Ty->getReturnType(), Instruction::Call,
1725 OperandTraits<CallBase>::op_end(this) -
1726 (Args.size() + CountBundleInputs(Bundles) + 1),
1727 unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
1728 InsertBefore) {
1729 init(Ty, Func, Args, Bundles, NameStr);
1730}
1731
1732//===----------------------------------------------------------------------===//
1733// SelectInst Class
1734//===----------------------------------------------------------------------===//
1735
1736/// This class represents the LLVM 'select' instruction.
1737///
1738class SelectInst : public Instruction {
1739 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1740 Instruction *InsertBefore)
1741 : Instruction(S1->getType(), Instruction::Select,
1742 &Op<0>(), 3, InsertBefore) {
1743 init(C, S1, S2);
1744 setName(NameStr);
1745 }
1746
1747 SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
1748 BasicBlock *InsertAtEnd)
1749 : Instruction(S1->getType(), Instruction::Select,
1750 &Op<0>(), 3, InsertAtEnd) {
1751 init(C, S1, S2);
1752 setName(NameStr);
1753 }
1754
1755 void init(Value *C, Value *S1, Value *S2) {
1756 assert(!areInvalidOperands(C, S1, S2) && "Invalid operands for select")(static_cast <bool> (!areInvalidOperands(C, S1, S2) &&
"Invalid operands for select") ? void (0) : __assert_fail ("!areInvalidOperands(C, S1, S2) && \"Invalid operands for select\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1756, __extension__ __PRETTY_FUNCTION__))
;
1757 Op<0>() = C;
1758 Op<1>() = S1;
1759 Op<2>() = S2;
1760 }
1761
1762protected:
1763 // Note: Instruction needs to be a friend here to call cloneImpl.
1764 friend class Instruction;
1765
1766 SelectInst *cloneImpl() const;
1767
1768public:
1769 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1770 const Twine &NameStr = "",
1771 Instruction *InsertBefore = nullptr,
1772 Instruction *MDFrom = nullptr) {
1773 SelectInst *Sel = new(3) SelectInst(C, S1, S2, NameStr, InsertBefore);
1774 if (MDFrom)
1775 Sel->copyMetadata(*MDFrom);
1776 return Sel;
1777 }
1778
1779 static SelectInst *Create(Value *C, Value *S1, Value *S2,
1780 const Twine &NameStr,
1781 BasicBlock *InsertAtEnd) {
1782 return new(3) SelectInst(C, S1, S2, NameStr, InsertAtEnd);
1783 }
1784
1785 const Value *getCondition() const { return Op<0>(); }
1786 const Value *getTrueValue() const { return Op<1>(); }
1787 const Value *getFalseValue() const { return Op<2>(); }
1788 Value *getCondition() { return Op<0>(); }
1789 Value *getTrueValue() { return Op<1>(); }
1790 Value *getFalseValue() { return Op<2>(); }
1791
1792 void setCondition(Value *V) { Op<0>() = V; }
1793 void setTrueValue(Value *V) { Op<1>() = V; }
1794 void setFalseValue(Value *V) { Op<2>() = V; }
1795
1796 /// Swap the true and false values of the select instruction.
1797 /// This doesn't swap prof metadata.
1798 void swapValues() { Op<1>().swap(Op<2>()); }
1799
1800 /// Return a string if the specified operands are invalid
1801 /// for a select operation, otherwise return null.
1802 static const char *areInvalidOperands(Value *Cond, Value *True, Value *False);
1803
1804 /// Transparently provide more efficient getOperand methods.
1805 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1806
1807 OtherOps getOpcode() const {
1808 return static_cast<OtherOps>(Instruction::getOpcode());
1809 }
1810
1811 // Methods for support type inquiry through isa, cast, and dyn_cast:
1812 static bool classof(const Instruction *I) {
1813 return I->getOpcode() == Instruction::Select;
1814 }
1815 static bool classof(const Value *V) {
1816 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1817 }
1818};
1819
1820template <>
1821struct OperandTraits<SelectInst> : public FixedNumOperandTraits<SelectInst, 3> {
1822};
1823
1824DEFINE_TRANSPARENT_OPERAND_ACCESSORS(SelectInst, Value)SelectInst::op_iterator SelectInst::op_begin() { return OperandTraits
<SelectInst>::op_begin(this); } SelectInst::const_op_iterator
SelectInst::op_begin() const { return OperandTraits<SelectInst
>::op_begin(const_cast<SelectInst*>(this)); } SelectInst
::op_iterator SelectInst::op_end() { return OperandTraits<
SelectInst>::op_end(this); } SelectInst::const_op_iterator
SelectInst::op_end() const { return OperandTraits<SelectInst
>::op_end(const_cast<SelectInst*>(this)); } Value *SelectInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<SelectInst>::operands
(this) && "getOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1824, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<SelectInst>::op_begin(const_cast
<SelectInst*>(this))[i_nocapture].get()); } void SelectInst
::setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<SelectInst>
::operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<SelectInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1824, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
SelectInst>::op_begin(this)[i_nocapture] = Val_nocapture; }
unsigned SelectInst::getNumOperands() const { return OperandTraits
<SelectInst>::operands(this); } template <int Idx_nocapture
> Use &SelectInst::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
SelectInst::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
1825
1826//===----------------------------------------------------------------------===//
1827// VAArgInst Class
1828//===----------------------------------------------------------------------===//
1829
1830/// This class represents the va_arg llvm instruction, which returns
1831/// an argument of the specified type given a va_list and increments that list
1832///
1833class VAArgInst : public UnaryInstruction {
1834protected:
1835 // Note: Instruction needs to be a friend here to call cloneImpl.
1836 friend class Instruction;
1837
1838 VAArgInst *cloneImpl() const;
1839
1840public:
1841 VAArgInst(Value *List, Type *Ty, const Twine &NameStr = "",
1842 Instruction *InsertBefore = nullptr)
1843 : UnaryInstruction(Ty, VAArg, List, InsertBefore) {
1844 setName(NameStr);
1845 }
1846
1847 VAArgInst(Value *List, Type *Ty, const Twine &NameStr,
1848 BasicBlock *InsertAtEnd)
1849 : UnaryInstruction(Ty, VAArg, List, InsertAtEnd) {
1850 setName(NameStr);
1851 }
1852
1853 Value *getPointerOperand() { return getOperand(0); }
1854 const Value *getPointerOperand() const { return getOperand(0); }
1855 static unsigned getPointerOperandIndex() { return 0U; }
1856
1857 // Methods for support type inquiry through isa, cast, and dyn_cast:
1858 static bool classof(const Instruction *I) {
1859 return I->getOpcode() == VAArg;
1860 }
1861 static bool classof(const Value *V) {
1862 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1863 }
1864};
1865
1866//===----------------------------------------------------------------------===//
1867// ExtractElementInst Class
1868//===----------------------------------------------------------------------===//
1869
1870/// This instruction extracts a single (scalar)
1871/// element from a VectorType value
1872///
1873class ExtractElementInst : public Instruction {
1874 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr = "",
1875 Instruction *InsertBefore = nullptr);
1876 ExtractElementInst(Value *Vec, Value *Idx, const Twine &NameStr,
1877 BasicBlock *InsertAtEnd);
1878
1879protected:
1880 // Note: Instruction needs to be a friend here to call cloneImpl.
1881 friend class Instruction;
1882
1883 ExtractElementInst *cloneImpl() const;
1884
1885public:
1886 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1887 const Twine &NameStr = "",
1888 Instruction *InsertBefore = nullptr) {
1889 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertBefore);
1890 }
1891
1892 static ExtractElementInst *Create(Value *Vec, Value *Idx,
1893 const Twine &NameStr,
1894 BasicBlock *InsertAtEnd) {
1895 return new(2) ExtractElementInst(Vec, Idx, NameStr, InsertAtEnd);
1896 }
1897
1898 /// Return true if an extractelement instruction can be
1899 /// formed with the specified operands.
1900 static bool isValidOperands(const Value *Vec, const Value *Idx);
1901
1902 Value *getVectorOperand() { return Op<0>(); }
1903 Value *getIndexOperand() { return Op<1>(); }
1904 const Value *getVectorOperand() const { return Op<0>(); }
1905 const Value *getIndexOperand() const { return Op<1>(); }
1906
1907 VectorType *getVectorOperandType() const {
1908 return cast<VectorType>(getVectorOperand()->getType());
1909 }
1910
1911 /// Transparently provide more efficient getOperand methods.
1912 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1913
1914 // Methods for support type inquiry through isa, cast, and dyn_cast:
1915 static bool classof(const Instruction *I) {
1916 return I->getOpcode() == Instruction::ExtractElement;
1917 }
1918 static bool classof(const Value *V) {
1919 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1920 }
1921};
1922
1923template <>
1924struct OperandTraits<ExtractElementInst> :
1925 public FixedNumOperandTraits<ExtractElementInst, 2> {
1926};
1927
1928DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ExtractElementInst, Value)ExtractElementInst::op_iterator ExtractElementInst::op_begin(
) { return OperandTraits<ExtractElementInst>::op_begin(
this); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_begin() const { return OperandTraits<ExtractElementInst
>::op_begin(const_cast<ExtractElementInst*>(this)); }
ExtractElementInst::op_iterator ExtractElementInst::op_end()
{ return OperandTraits<ExtractElementInst>::op_end(this
); } ExtractElementInst::const_op_iterator ExtractElementInst
::op_end() const { return OperandTraits<ExtractElementInst
>::op_end(const_cast<ExtractElementInst*>(this)); } Value
*ExtractElementInst::getOperand(unsigned i_nocapture) const {
(static_cast <bool> (i_nocapture < OperandTraits<
ExtractElementInst>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1928, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ExtractElementInst>::op_begin
(const_cast<ExtractElementInst*>(this))[i_nocapture].get
()); } void ExtractElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ExtractElementInst>::operands(this)
&& "setOperand() out of range!") ? void (0) : __assert_fail
("i_nocapture < OperandTraits<ExtractElementInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1928, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ExtractElementInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned ExtractElementInst::getNumOperands() const { return
OperandTraits<ExtractElementInst>::operands(this); } template
<int Idx_nocapture> Use &ExtractElementInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ExtractElementInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1929
1930//===----------------------------------------------------------------------===//
1931// InsertElementInst Class
1932//===----------------------------------------------------------------------===//
1933
1934/// This instruction inserts a single (scalar)
1935/// element into a VectorType value
1936///
1937class InsertElementInst : public Instruction {
1938 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx,
1939 const Twine &NameStr = "",
1940 Instruction *InsertBefore = nullptr);
1941 InsertElementInst(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr,
1942 BasicBlock *InsertAtEnd);
1943
1944protected:
1945 // Note: Instruction needs to be a friend here to call cloneImpl.
1946 friend class Instruction;
1947
1948 InsertElementInst *cloneImpl() const;
1949
1950public:
1951 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1952 const Twine &NameStr = "",
1953 Instruction *InsertBefore = nullptr) {
1954 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertBefore);
1955 }
1956
1957 static InsertElementInst *Create(Value *Vec, Value *NewElt, Value *Idx,
1958 const Twine &NameStr,
1959 BasicBlock *InsertAtEnd) {
1960 return new(3) InsertElementInst(Vec, NewElt, Idx, NameStr, InsertAtEnd);
1961 }
1962
1963 /// Return true if an insertelement instruction can be
1964 /// formed with the specified operands.
1965 static bool isValidOperands(const Value *Vec, const Value *NewElt,
1966 const Value *Idx);
1967
1968 /// Overload to return most specific vector type.
1969 ///
1970 VectorType *getType() const {
1971 return cast<VectorType>(Instruction::getType());
1972 }
1973
1974 /// Transparently provide more efficient getOperand methods.
1975 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
1976
1977 // Methods for support type inquiry through isa, cast, and dyn_cast:
1978 static bool classof(const Instruction *I) {
1979 return I->getOpcode() == Instruction::InsertElement;
1980 }
1981 static bool classof(const Value *V) {
1982 return isa<Instruction>(V) && classof(cast<Instruction>(V));
1983 }
1984};
1985
1986template <>
1987struct OperandTraits<InsertElementInst> :
1988 public FixedNumOperandTraits<InsertElementInst, 3> {
1989};
1990
1991DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertElementInst, Value)InsertElementInst::op_iterator InsertElementInst::op_begin() {
return OperandTraits<InsertElementInst>::op_begin(this
); } InsertElementInst::const_op_iterator InsertElementInst::
op_begin() const { return OperandTraits<InsertElementInst>
::op_begin(const_cast<InsertElementInst*>(this)); } InsertElementInst
::op_iterator InsertElementInst::op_end() { return OperandTraits
<InsertElementInst>::op_end(this); } InsertElementInst::
const_op_iterator InsertElementInst::op_end() const { return OperandTraits
<InsertElementInst>::op_end(const_cast<InsertElementInst
*>(this)); } Value *InsertElementInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<InsertElementInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1991, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InsertElementInst>::op_begin
(const_cast<InsertElementInst*>(this))[i_nocapture].get
()); } void InsertElementInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<InsertElementInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertElementInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 1991, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InsertElementInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned InsertElementInst::getNumOperands() const { return
OperandTraits<InsertElementInst>::operands(this); } template
<int Idx_nocapture> Use &InsertElementInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &InsertElementInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
1992
1993//===----------------------------------------------------------------------===//
1994// ShuffleVectorInst Class
1995//===----------------------------------------------------------------------===//
1996
1997constexpr int UndefMaskElem = -1;
1998
1999/// This instruction constructs a fixed permutation of two
2000/// input vectors.
2001///
2002/// For each element of the result vector, the shuffle mask selects an element
2003/// from one of the input vectors to copy to the result. Non-negative elements
2004/// in the mask represent an index into the concatenated pair of input vectors.
2005/// UndefMaskElem (-1) specifies that the result element is undefined.
2006///
2007/// For scalable vectors, all the elements of the mask must be 0 or -1. This
2008/// requirement may be relaxed in the future.
2009class ShuffleVectorInst : public Instruction {
2010 SmallVector<int, 4> ShuffleMask;
2011 Constant *ShuffleMaskForBitcode;
2012
2013protected:
2014 // Note: Instruction needs to be a friend here to call cloneImpl.
2015 friend class Instruction;
2016
2017 ShuffleVectorInst *cloneImpl() const;
2018
2019public:
2020 ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr = "",
2021 Instruction *InsertBefore = nullptr);
2022 ShuffleVectorInst(Value *V1, Value *Mask, const Twine &NameStr,
2023 BasicBlock *InsertAtEnd);
2024 ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr = "",
2025 Instruction *InsertBefore = nullptr);
2026 ShuffleVectorInst(Value *V1, ArrayRef<int> Mask, const Twine &NameStr,
2027 BasicBlock *InsertAtEnd);
2028 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2029 const Twine &NameStr = "",
2030 Instruction *InsertBefor = nullptr);
2031 ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
2032 const Twine &NameStr, BasicBlock *InsertAtEnd);
2033 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2034 const Twine &NameStr = "",
2035 Instruction *InsertBefor = nullptr);
2036 ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
2037 const Twine &NameStr, BasicBlock *InsertAtEnd);
2038
2039 void *operator new(size_t S) { return User::operator new(S, 2); }
2040 void operator delete(void *Ptr) { return User::operator delete(Ptr); }
2041
2042 /// Swap the operands and adjust the mask to preserve the semantics
2043 /// of the instruction.
2044 void commute();
2045
2046 /// Return true if a shufflevector instruction can be
2047 /// formed with the specified operands.
2048 static bool isValidOperands(const Value *V1, const Value *V2,
2049 const Value *Mask);
2050 static bool isValidOperands(const Value *V1, const Value *V2,
2051 ArrayRef<int> Mask);
2052
2053 /// Overload to return most specific vector type.
2054 ///
2055 VectorType *getType() const {
2056 return cast<VectorType>(Instruction::getType());
2057 }
2058
2059 /// Transparently provide more efficient getOperand methods.
2060 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2061
2062 /// Return the shuffle mask value of this instruction for the given element
2063 /// index. Return UndefMaskElem if the element is undef.
2064 int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
2065
2066 /// Convert the input shuffle mask operand to a vector of integers. Undefined
2067 /// elements of the mask are returned as UndefMaskElem.
2068 static void getShuffleMask(const Constant *Mask,
2069 SmallVectorImpl<int> &Result);
2070
2071 /// Return the mask for this instruction as a vector of integers. Undefined
2072 /// elements of the mask are returned as UndefMaskElem.
2073 void getShuffleMask(SmallVectorImpl<int> &Result) const {
2074 Result.assign(ShuffleMask.begin(), ShuffleMask.end());
2075 }
2076
2077 /// Return the mask for this instruction, for use in bitcode.
2078 ///
2079 /// TODO: This is temporary until we decide a new bitcode encoding for
2080 /// shufflevector.
2081 Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
2082
2083 static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
2084 Type *ResultTy);
2085
2086 void setShuffleMask(ArrayRef<int> Mask);
2087
2088 ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
2089
2090 /// Return true if this shuffle returns a vector with a different number of
2091 /// elements than its source vectors.
2092 /// Examples: shufflevector <4 x n> A, <4 x n> B, <1,2,3>
2093 /// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
2094 bool changesLength() const {
2095 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2096 ->getElementCount()
2097 .getKnownMinValue();
2098 unsigned NumMaskElts = ShuffleMask.size();
2099 return NumSourceElts != NumMaskElts;
2100 }
2101
2102 /// Return true if this shuffle returns a vector with a greater number of
2103 /// elements than its source vectors.
2104 /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
2105 bool increasesLength() const {
2106 unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
2107 ->getElementCount()
2108 .getKnownMinValue();
2109 unsigned NumMaskElts = ShuffleMask.size();
2110 return NumSourceElts < NumMaskElts;
2111 }
2112
2113 /// Return true if this shuffle mask chooses elements from exactly one source
2114 /// vector.
2115 /// Example: <7,5,undef,7>
2116 /// This assumes that vector operands are the same length as the mask.
2117 static bool isSingleSourceMask(ArrayRef<int> Mask);
2118 static bool isSingleSourceMask(const Constant *Mask) {
2119 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2119, __extension__ __PRETTY_FUNCTION__))
;
2120 SmallVector<int, 16> MaskAsInts;
2121 getShuffleMask(Mask, MaskAsInts);
2122 return isSingleSourceMask(MaskAsInts);
2123 }
2124
2125 /// Return true if this shuffle chooses elements from exactly one source
2126 /// vector without changing the length of that vector.
2127 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
2128 /// TODO: Optionally allow length-changing shuffles.
2129 bool isSingleSource() const {
2130 return !changesLength() && isSingleSourceMask(ShuffleMask);
2131 }
2132
2133 /// Return true if this shuffle mask chooses elements from exactly one source
2134 /// vector without lane crossings. A shuffle using this mask is not
2135 /// necessarily a no-op because it may change the number of elements from its
2136 /// input vectors or it may provide demanded bits knowledge via undef lanes.
2137 /// Example: <undef,undef,2,3>
2138 static bool isIdentityMask(ArrayRef<int> Mask);
2139 static bool isIdentityMask(const Constant *Mask) {
2140 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2140, __extension__ __PRETTY_FUNCTION__))
;
2141 SmallVector<int, 16> MaskAsInts;
2142 getShuffleMask(Mask, MaskAsInts);
2143 return isIdentityMask(MaskAsInts);
2144 }
2145
2146 /// Return true if this shuffle chooses elements from exactly one source
2147 /// vector without lane crossings and does not change the number of elements
2148 /// from its input vectors.
2149 /// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
2150 bool isIdentity() const {
2151 return !changesLength() && isIdentityMask(ShuffleMask);
2152 }
2153
2154 /// Return true if this shuffle lengthens exactly one source vector with
2155 /// undefs in the high elements.
2156 bool isIdentityWithPadding() const;
2157
2158 /// Return true if this shuffle extracts the first N elements of exactly one
2159 /// source vector.
2160 bool isIdentityWithExtract() const;
2161
2162 /// Return true if this shuffle concatenates its 2 source vectors. This
2163 /// returns false if either input is undefined. In that case, the shuffle is
2164 /// is better classified as an identity with padding operation.
2165 bool isConcat() const;
2166
2167 /// Return true if this shuffle mask chooses elements from its source vectors
2168 /// without lane crossings. A shuffle using this mask would be
2169 /// equivalent to a vector select with a constant condition operand.
2170 /// Example: <4,1,6,undef>
2171 /// This returns false if the mask does not choose from both input vectors.
2172 /// In that case, the shuffle is better classified as an identity shuffle.
2173 /// This assumes that vector operands are the same length as the mask
2174 /// (a length-changing shuffle can never be equivalent to a vector select).
2175 static bool isSelectMask(ArrayRef<int> Mask);
2176 static bool isSelectMask(const Constant *Mask) {
2177 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2177, __extension__ __PRETTY_FUNCTION__))
;
2178 SmallVector<int, 16> MaskAsInts;
2179 getShuffleMask(Mask, MaskAsInts);
2180 return isSelectMask(MaskAsInts);
2181 }
2182
2183 /// Return true if this shuffle chooses elements from its source vectors
2184 /// without lane crossings and all operands have the same number of elements.
2185 /// In other words, this shuffle is equivalent to a vector select with a
2186 /// constant condition operand.
2187 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,1,6,3>
2188 /// This returns false if the mask does not choose from both input vectors.
2189 /// In that case, the shuffle is better classified as an identity shuffle.
2190 /// TODO: Optionally allow length-changing shuffles.
2191 bool isSelect() const {
2192 return !changesLength() && isSelectMask(ShuffleMask);
2193 }
2194
2195 /// Return true if this shuffle mask swaps the order of elements from exactly
2196 /// one source vector.
2197 /// Example: <7,6,undef,4>
2198 /// This assumes that vector operands are the same length as the mask.
2199 static bool isReverseMask(ArrayRef<int> Mask);
2200 static bool isReverseMask(const Constant *Mask) {
2201 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2201, __extension__ __PRETTY_FUNCTION__))
;
2202 SmallVector<int, 16> MaskAsInts;
2203 getShuffleMask(Mask, MaskAsInts);
2204 return isReverseMask(MaskAsInts);
2205 }
2206
2207 /// Return true if this shuffle swaps the order of elements from exactly
2208 /// one source vector.
2209 /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
2210 /// TODO: Optionally allow length-changing shuffles.
2211 bool isReverse() const {
2212 return !changesLength() && isReverseMask(ShuffleMask);
2213 }
2214
2215 /// Return true if this shuffle mask chooses all elements with the same value
2216 /// as the first element of exactly one source vector.
2217 /// Example: <4,undef,undef,4>
2218 /// This assumes that vector operands are the same length as the mask.
2219 static bool isZeroEltSplatMask(ArrayRef<int> Mask);
2220 static bool isZeroEltSplatMask(const Constant *Mask) {
2221 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2221, __extension__ __PRETTY_FUNCTION__))
;
2222 SmallVector<int, 16> MaskAsInts;
2223 getShuffleMask(Mask, MaskAsInts);
2224 return isZeroEltSplatMask(MaskAsInts);
2225 }
2226
2227 /// Return true if all elements of this shuffle are the same value as the
2228 /// first element of exactly one source vector without changing the length
2229 /// of that vector.
2230 /// Example: shufflevector <4 x n> A, <4 x n> B, <undef,0,undef,0>
2231 /// TODO: Optionally allow length-changing shuffles.
2232 /// TODO: Optionally allow splats from other elements.
2233 bool isZeroEltSplat() const {
2234 return !changesLength() && isZeroEltSplatMask(ShuffleMask);
2235 }
2236
2237 /// Return true if this shuffle mask is a transpose mask.
2238 /// Transpose vector masks transpose a 2xn matrix. They read corresponding
2239 /// even- or odd-numbered vector elements from two n-dimensional source
2240 /// vectors and write each result into consecutive elements of an
2241 /// n-dimensional destination vector. Two shuffles are necessary to complete
2242 /// the transpose, one for the even elements and another for the odd elements.
2243 /// This description closely follows how the TRN1 and TRN2 AArch64
2244 /// instructions operate.
2245 ///
2246 /// For example, a simple 2x2 matrix can be transposed with:
2247 ///
2248 /// ; Original matrix
2249 /// m0 = < a, b >
2250 /// m1 = < c, d >
2251 ///
2252 /// ; Transposed matrix
2253 /// t0 = < a, c > = shufflevector m0, m1, < 0, 2 >
2254 /// t1 = < b, d > = shufflevector m0, m1, < 1, 3 >
2255 ///
2256 /// For matrices having greater than n columns, the resulting nx2 transposed
2257 /// matrix is stored in two result vectors such that one vector contains
2258 /// interleaved elements from all the even-numbered rows and the other vector
2259 /// contains interleaved elements from all the odd-numbered rows. For example,
2260 /// a 2x4 matrix can be transposed with:
2261 ///
2262 /// ; Original matrix
2263 /// m0 = < a, b, c, d >
2264 /// m1 = < e, f, g, h >
2265 ///
2266 /// ; Transposed matrix
2267 /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 >
2268 /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 >
2269 static bool isTransposeMask(ArrayRef<int> Mask);
2270 static bool isTransposeMask(const Constant *Mask) {
2271 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2271, __extension__ __PRETTY_FUNCTION__))
;
2272 SmallVector<int, 16> MaskAsInts;
2273 getShuffleMask(Mask, MaskAsInts);
2274 return isTransposeMask(MaskAsInts);
2275 }
2276
2277 /// Return true if this shuffle transposes the elements of its inputs without
2278 /// changing the length of the vectors. This operation may also be known as a
2279 /// merge or interleave. See the description for isTransposeMask() for the
2280 /// exact specification.
2281 /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
2282 bool isTranspose() const {
2283 return !changesLength() && isTransposeMask(ShuffleMask);
2284 }
2285
2286 /// Return true if this shuffle mask is an extract subvector mask.
2287 /// A valid extract subvector mask returns a smaller vector from a single
2288 /// source operand. The base extraction index is returned as well.
2289 static bool isExtractSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2290 int &Index);
2291 static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
2292 int &Index) {
2293 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2293, __extension__ __PRETTY_FUNCTION__))
;
2294 // Not possible to express a shuffle mask for a scalable vector for this
2295 // case.
2296 if (isa<ScalableVectorType>(Mask->getType()))
2297 return false;
2298 SmallVector<int, 16> MaskAsInts;
2299 getShuffleMask(Mask, MaskAsInts);
2300 return isExtractSubvectorMask(MaskAsInts, NumSrcElts, Index);
2301 }
2302
2303 /// Return true if this shuffle mask is an extract subvector mask.
2304 bool isExtractSubvectorMask(int &Index) const {
2305 // Not possible to express a shuffle mask for a scalable vector for this
2306 // case.
2307 if (isa<ScalableVectorType>(getType()))
2308 return false;
2309
2310 int NumSrcElts =
2311 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2312 return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
2313 }
2314
2315 /// Return true if this shuffle mask is an insert subvector mask.
2316 /// A valid insert subvector mask inserts the lowest elements of a second
2317 /// source operand into an in-place first source operand operand.
2318 /// Both the sub vector width and the insertion index is returned.
2319 static bool isInsertSubvectorMask(ArrayRef<int> Mask, int NumSrcElts,
2320 int &NumSubElts, int &Index);
2321 static bool isInsertSubvectorMask(const Constant *Mask, int NumSrcElts,
2322 int &NumSubElts, int &Index) {
2323 assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.")(static_cast <bool> (Mask->getType()->isVectorTy(
) && "Shuffle needs vector constant.") ? void (0) : __assert_fail
("Mask->getType()->isVectorTy() && \"Shuffle needs vector constant.\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2323, __extension__ __PRETTY_FUNCTION__))
;
2324 // Not possible to express a shuffle mask for a scalable vector for this
2325 // case.
2326 if (isa<ScalableVectorType>(Mask->getType()))
2327 return false;
2328 SmallVector<int, 16> MaskAsInts;
2329 getShuffleMask(Mask, MaskAsInts);
2330 return isInsertSubvectorMask(MaskAsInts, NumSrcElts, NumSubElts, Index);
2331 }
2332
2333 /// Return true if this shuffle mask is an insert subvector mask.
2334 bool isInsertSubvectorMask(int &NumSubElts, int &Index) const {
2335 // Not possible to express a shuffle mask for a scalable vector for this
2336 // case.
2337 if (isa<ScalableVectorType>(getType()))
2338 return false;
2339
2340 int NumSrcElts =
2341 cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
2342 return isInsertSubvectorMask(ShuffleMask, NumSrcElts, NumSubElts, Index);
2343 }
2344
2345 /// Change values in a shuffle permute mask assuming the two vector operands
2346 /// of length InVecNumElts have swapped position.
2347 static void commuteShuffleMask(MutableArrayRef<int> Mask,
2348 unsigned InVecNumElts) {
2349 for (int &Idx : Mask) {
2350 if (Idx == -1)
2351 continue;
2352 Idx = Idx < (int)InVecNumElts ? Idx + InVecNumElts : Idx - InVecNumElts;
2353 assert(Idx >= 0 && Idx < (int)InVecNumElts * 2 &&(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2354, __extension__ __PRETTY_FUNCTION__))
2354 "shufflevector mask index out of range")(static_cast <bool> (Idx >= 0 && Idx < (int
)InVecNumElts * 2 && "shufflevector mask index out of range"
) ? void (0) : __assert_fail ("Idx >= 0 && Idx < (int)InVecNumElts * 2 && \"shufflevector mask index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2354, __extension__ __PRETTY_FUNCTION__))
;
2355 }
2356 }
2357
2358 // Methods for support type inquiry through isa, cast, and dyn_cast:
2359 static bool classof(const Instruction *I) {
2360 return I->getOpcode() == Instruction::ShuffleVector;
2361 }
2362 static bool classof(const Value *V) {
2363 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2364 }
2365};
2366
2367template <>
2368struct OperandTraits<ShuffleVectorInst>
2369 : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
2370
2371DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)ShuffleVectorInst::op_iterator ShuffleVectorInst::op_begin() {
return OperandTraits<ShuffleVectorInst>::op_begin(this
); } ShuffleVectorInst::const_op_iterator ShuffleVectorInst::
op_begin() const { return OperandTraits<ShuffleVectorInst>
::op_begin(const_cast<ShuffleVectorInst*>(this)); } ShuffleVectorInst
::op_iterator ShuffleVectorInst::op_end() { return OperandTraits
<ShuffleVectorInst>::op_end(this); } ShuffleVectorInst::
const_op_iterator ShuffleVectorInst::op_end() const { return OperandTraits
<ShuffleVectorInst>::op_end(const_cast<ShuffleVectorInst
*>(this)); } Value *ShuffleVectorInst::getOperand(unsigned
i_nocapture) const { (static_cast <bool> (i_nocapture <
OperandTraits<ShuffleVectorInst>::operands(this) &&
"getOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2371, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<ShuffleVectorInst>::op_begin
(const_cast<ShuffleVectorInst*>(this))[i_nocapture].get
()); } void ShuffleVectorInst::setOperand(unsigned i_nocapture
, Value *Val_nocapture) { (static_cast <bool> (i_nocapture
< OperandTraits<ShuffleVectorInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<ShuffleVectorInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2371, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
ShuffleVectorInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned ShuffleVectorInst::getNumOperands() const { return
OperandTraits<ShuffleVectorInst>::operands(this); } template
<int Idx_nocapture> Use &ShuffleVectorInst::Op() {
return this->OpFrom<Idx_nocapture>(this); } template
<int Idx_nocapture> const Use &ShuffleVectorInst::
Op() const { return this->OpFrom<Idx_nocapture>(this
); }
2372
2373//===----------------------------------------------------------------------===//
2374// ExtractValueInst Class
2375//===----------------------------------------------------------------------===//
2376
2377/// This instruction extracts a struct member or array
2378/// element value from an aggregate value.
2379///
2380class ExtractValueInst : public UnaryInstruction {
2381 SmallVector<unsigned, 4> Indices;
2382
2383 ExtractValueInst(const ExtractValueInst &EVI);
2384
2385 /// Constructors - Create a extractvalue instruction with a base aggregate
2386 /// value and a list of indices. The first ctor can optionally insert before
2387 /// an existing instruction, the second appends the new instruction to the
2388 /// specified BasicBlock.
2389 inline ExtractValueInst(Value *Agg,
2390 ArrayRef<unsigned> Idxs,
2391 const Twine &NameStr,
2392 Instruction *InsertBefore);
2393 inline ExtractValueInst(Value *Agg,
2394 ArrayRef<unsigned> Idxs,
2395 const Twine &NameStr, BasicBlock *InsertAtEnd);
2396
2397 void init(ArrayRef<unsigned> Idxs, const Twine &NameStr);
2398
2399protected:
2400 // Note: Instruction needs to be a friend here to call cloneImpl.
2401 friend class Instruction;
2402
2403 ExtractValueInst *cloneImpl() const;
2404
2405public:
2406 static ExtractValueInst *Create(Value *Agg,
2407 ArrayRef<unsigned> Idxs,
2408 const Twine &NameStr = "",
2409 Instruction *InsertBefore = nullptr) {
2410 return new
2411 ExtractValueInst(Agg, Idxs, NameStr, InsertBefore);
2412 }
2413
2414 static ExtractValueInst *Create(Value *Agg,
2415 ArrayRef<unsigned> Idxs,
2416 const Twine &NameStr,
2417 BasicBlock *InsertAtEnd) {
2418 return new ExtractValueInst(Agg, Idxs, NameStr, InsertAtEnd);
2419 }
2420
2421 /// Returns the type of the element that would be extracted
2422 /// with an extractvalue instruction with the specified parameters.
2423 ///
2424 /// Null is returned if the indices are invalid for the specified type.
2425 static Type *getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs);
2426
2427 using idx_iterator = const unsigned*;
2428
2429 inline idx_iterator idx_begin() const { return Indices.begin(); }
2430 inline idx_iterator idx_end() const { return Indices.end(); }
2431 inline iterator_range<idx_iterator> indices() const {
2432 return make_range(idx_begin(), idx_end());
2433 }
2434
2435 Value *getAggregateOperand() {
2436 return getOperand(0);
2437 }
2438 const Value *getAggregateOperand() const {
2439 return getOperand(0);
2440 }
2441 static unsigned getAggregateOperandIndex() {
2442 return 0U; // get index for modifying correct operand
2443 }
2444
2445 ArrayRef<unsigned> getIndices() const {
2446 return Indices;
2447 }
2448
2449 unsigned getNumIndices() const {
2450 return (unsigned)Indices.size();
2451 }
2452
2453 bool hasIndices() const {
2454 return true;
2455 }
2456
2457 // Methods for support type inquiry through isa, cast, and dyn_cast:
2458 static bool classof(const Instruction *I) {
2459 return I->getOpcode() == Instruction::ExtractValue;
2460 }
2461 static bool classof(const Value *V) {
2462 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2463 }
2464};
2465
2466ExtractValueInst::ExtractValueInst(Value *Agg,
2467 ArrayRef<unsigned> Idxs,
2468 const Twine &NameStr,
2469 Instruction *InsertBefore)
2470 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2471 ExtractValue, Agg, InsertBefore) {
2472 init(Idxs, NameStr);
2473}
2474
2475ExtractValueInst::ExtractValueInst(Value *Agg,
2476 ArrayRef<unsigned> Idxs,
2477 const Twine &NameStr,
2478 BasicBlock *InsertAtEnd)
2479 : UnaryInstruction(checkGEPType(getIndexedType(Agg->getType(), Idxs)),
2480 ExtractValue, Agg, InsertAtEnd) {
2481 init(Idxs, NameStr);
2482}
2483
2484//===----------------------------------------------------------------------===//
2485// InsertValueInst Class
2486//===----------------------------------------------------------------------===//
2487
2488/// This instruction inserts a struct field of array element
2489/// value into an aggregate value.
2490///
2491class InsertValueInst : public Instruction {
2492 SmallVector<unsigned, 4> Indices;
2493
2494 InsertValueInst(const InsertValueInst &IVI);
2495
2496 /// Constructors - Create a insertvalue instruction with a base aggregate
2497 /// value, a value to insert, and a list of indices. The first ctor can
2498 /// optionally insert before an existing instruction, the second appends
2499 /// the new instruction to the specified BasicBlock.
2500 inline InsertValueInst(Value *Agg, Value *Val,
2501 ArrayRef<unsigned> Idxs,
2502 const Twine &NameStr,
2503 Instruction *InsertBefore);
2504 inline InsertValueInst(Value *Agg, Value *Val,
2505 ArrayRef<unsigned> Idxs,
2506 const Twine &NameStr, BasicBlock *InsertAtEnd);
2507
2508 /// Constructors - These two constructors are convenience methods because one
2509 /// and two index insertvalue instructions are so common.
2510 InsertValueInst(Value *Agg, Value *Val, unsigned Idx,
2511 const Twine &NameStr = "",
2512 Instruction *InsertBefore = nullptr);
2513 InsertValueInst(Value *Agg, Value *Val, unsigned Idx, const Twine &NameStr,
2514 BasicBlock *InsertAtEnd);
2515
2516 void init(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
2517 const Twine &NameStr);
2518
2519protected:
2520 // Note: Instruction needs to be a friend here to call cloneImpl.
2521 friend class Instruction;
2522
2523 InsertValueInst *cloneImpl() const;
2524
2525public:
2526 // allocate space for exactly two operands
2527 void *operator new(size_t S) { return User::operator new(S, 2); }
2528 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2529
2530 static InsertValueInst *Create(Value *Agg, Value *Val,
2531 ArrayRef<unsigned> Idxs,
2532 const Twine &NameStr = "",
2533 Instruction *InsertBefore = nullptr) {
2534 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertBefore);
2535 }
2536
2537 static InsertValueInst *Create(Value *Agg, Value *Val,
2538 ArrayRef<unsigned> Idxs,
2539 const Twine &NameStr,
2540 BasicBlock *InsertAtEnd) {
2541 return new InsertValueInst(Agg, Val, Idxs, NameStr, InsertAtEnd);
2542 }
2543
2544 /// Transparently provide more efficient getOperand methods.
2545 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2546
2547 using idx_iterator = const unsigned*;
2548
2549 inline idx_iterator idx_begin() const { return Indices.begin(); }
2550 inline idx_iterator idx_end() const { return Indices.end(); }
2551 inline iterator_range<idx_iterator> indices() const {
2552 return make_range(idx_begin(), idx_end());
2553 }
2554
2555 Value *getAggregateOperand() {
2556 return getOperand(0);
2557 }
2558 const Value *getAggregateOperand() const {
2559 return getOperand(0);
2560 }
2561 static unsigned getAggregateOperandIndex() {
2562 return 0U; // get index for modifying correct operand
2563 }
2564
2565 Value *getInsertedValueOperand() {
2566 return getOperand(1);
2567 }
2568 const Value *getInsertedValueOperand() const {
2569 return getOperand(1);
2570 }
2571 static unsigned getInsertedValueOperandIndex() {
2572 return 1U; // get index for modifying correct operand
2573 }
2574
2575 ArrayRef<unsigned> getIndices() const {
2576 return Indices;
2577 }
2578
2579 unsigned getNumIndices() const {
2580 return (unsigned)Indices.size();
2581 }
2582
2583 bool hasIndices() const {
2584 return true;
2585 }
2586
2587 // Methods for support type inquiry through isa, cast, and dyn_cast:
2588 static bool classof(const Instruction *I) {
2589 return I->getOpcode() == Instruction::InsertValue;
2590 }
2591 static bool classof(const Value *V) {
2592 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2593 }
2594};
2595
2596template <>
2597struct OperandTraits<InsertValueInst> :
2598 public FixedNumOperandTraits<InsertValueInst, 2> {
2599};
2600
2601InsertValueInst::InsertValueInst(Value *Agg,
2602 Value *Val,
2603 ArrayRef<unsigned> Idxs,
2604 const Twine &NameStr,
2605 Instruction *InsertBefore)
2606 : Instruction(Agg->getType(), InsertValue,
2607 OperandTraits<InsertValueInst>::op_begin(this),
2608 2, InsertBefore) {
2609 init(Agg, Val, Idxs, NameStr);
2610}
2611
2612InsertValueInst::InsertValueInst(Value *Agg,
2613 Value *Val,
2614 ArrayRef<unsigned> Idxs,
2615 const Twine &NameStr,
2616 BasicBlock *InsertAtEnd)
2617 : Instruction(Agg->getType(), InsertValue,
2618 OperandTraits<InsertValueInst>::op_begin(this),
2619 2, InsertAtEnd) {
2620 init(Agg, Val, Idxs, NameStr);
2621}
2622
2623DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)InsertValueInst::op_iterator InsertValueInst::op_begin() { return
OperandTraits<InsertValueInst>::op_begin(this); } InsertValueInst
::const_op_iterator InsertValueInst::op_begin() const { return
OperandTraits<InsertValueInst>::op_begin(const_cast<
InsertValueInst*>(this)); } InsertValueInst::op_iterator InsertValueInst
::op_end() { return OperandTraits<InsertValueInst>::op_end
(this); } InsertValueInst::const_op_iterator InsertValueInst::
op_end() const { return OperandTraits<InsertValueInst>::
op_end(const_cast<InsertValueInst*>(this)); } Value *InsertValueInst
::getOperand(unsigned i_nocapture) const { (static_cast <bool
> (i_nocapture < OperandTraits<InsertValueInst>::
operands(this) && "getOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2623, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<InsertValueInst>::op_begin
(const_cast<InsertValueInst*>(this))[i_nocapture].get()
); } void InsertValueInst::setOperand(unsigned i_nocapture, Value
*Val_nocapture) { (static_cast <bool> (i_nocapture <
OperandTraits<InsertValueInst>::operands(this) &&
"setOperand() out of range!") ? void (0) : __assert_fail ("i_nocapture < OperandTraits<InsertValueInst>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2623, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
InsertValueInst>::op_begin(this)[i_nocapture] = Val_nocapture
; } unsigned InsertValueInst::getNumOperands() const { return
OperandTraits<InsertValueInst>::operands(this); } template
<int Idx_nocapture> Use &InsertValueInst::Op() { return
this->OpFrom<Idx_nocapture>(this); } template <int
Idx_nocapture> const Use &InsertValueInst::Op() const
{ return this->OpFrom<Idx_nocapture>(this); }
2624
2625//===----------------------------------------------------------------------===//
2626// PHINode Class
2627//===----------------------------------------------------------------------===//
2628
2629// PHINode - The PHINode class is used to represent the magical mystical PHI
2630// node, that can not exist in nature, but can be synthesized in a computer
2631// scientist's overactive imagination.
2632//
2633class PHINode : public Instruction {
2634 /// The number of operands actually allocated. NumOperands is
2635 /// the number actually in use.
2636 unsigned ReservedSpace;
2637
2638 PHINode(const PHINode &PN);
2639
2640 explicit PHINode(Type *Ty, unsigned NumReservedValues,
2641 const Twine &NameStr = "",
2642 Instruction *InsertBefore = nullptr)
2643 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertBefore),
2644 ReservedSpace(NumReservedValues) {
2645 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2645, __extension__ __PRETTY_FUNCTION__))
;
2646 setName(NameStr);
2647 allocHungoffUses(ReservedSpace);
2648 }
2649
2650 PHINode(Type *Ty, unsigned NumReservedValues, const Twine &NameStr,
2651 BasicBlock *InsertAtEnd)
2652 : Instruction(Ty, Instruction::PHI, nullptr, 0, InsertAtEnd),
2653 ReservedSpace(NumReservedValues) {
2654 assert(!Ty->isTokenTy() && "PHI nodes cannot have token type!")(static_cast <bool> (!Ty->isTokenTy() && "PHI nodes cannot have token type!"
) ? void (0) : __assert_fail ("!Ty->isTokenTy() && \"PHI nodes cannot have token type!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2654, __extension__ __PRETTY_FUNCTION__))
;
2655 setName(NameStr);
2656 allocHungoffUses(ReservedSpace);
2657 }
2658
2659protected:
2660 // Note: Instruction needs to be a friend here to call cloneImpl.
2661 friend class Instruction;
2662
2663 PHINode *cloneImpl() const;
2664
2665 // allocHungoffUses - this is more complicated than the generic
2666 // User::allocHungoffUses, because we have to allocate Uses for the incoming
2667 // values and pointers to the incoming blocks, all in one allocation.
2668 void allocHungoffUses(unsigned N) {
2669 User::allocHungoffUses(N, /* IsPhi */ true);
2670 }
2671
2672public:
2673 /// Constructors - NumReservedValues is a hint for the number of incoming
2674 /// edges that this phi node will have (use 0 if you really have no idea).
2675 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2676 const Twine &NameStr = "",
2677 Instruction *InsertBefore = nullptr) {
2678 return new PHINode(Ty, NumReservedValues, NameStr, InsertBefore);
2679 }
2680
2681 static PHINode *Create(Type *Ty, unsigned NumReservedValues,
2682 const Twine &NameStr, BasicBlock *InsertAtEnd) {
2683 return new PHINode(Ty, NumReservedValues, NameStr, InsertAtEnd);
2684 }
2685
2686 /// Provide fast operand accessors
2687 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value)public: inline Value *getOperand(unsigned) const; inline void
setOperand(unsigned, Value*); inline op_iterator op_begin();
inline const_op_iterator op_begin() const; inline op_iterator
op_end(); inline const_op_iterator op_end() const; protected
: template <int> inline Use &Op(); template <int
> inline const Use &Op() const; public: inline unsigned
getNumOperands() const
;
2688
2689 // Block iterator interface. This provides access to the list of incoming
2690 // basic blocks, which parallels the list of incoming values.
2691
2692 using block_iterator = BasicBlock **;
2693 using const_block_iterator = BasicBlock * const *;
2694
2695 block_iterator block_begin() {
2696 return reinterpret_cast<block_iterator>(op_begin() + ReservedSpace);
2697 }
2698
2699 const_block_iterator block_begin() const {
2700 return reinterpret_cast<const_block_iterator>(op_begin() + ReservedSpace);
2701 }
2702
2703 block_iterator block_end() {
2704 return block_begin() + getNumOperands();
2705 }
2706
2707 const_block_iterator block_end() const {
2708 return block_begin() + getNumOperands();
2709 }
2710
2711 iterator_range<block_iterator> blocks() {
2712 return make_range(block_begin(), block_end());
2713 }
2714
2715 iterator_range<const_block_iterator> blocks() const {
2716 return make_range(block_begin(), block_end());
2717 }
2718
2719 op_range incoming_values() { return operands(); }
2720
2721 const_op_range incoming_values() const { return operands(); }
2722
2723 /// Return the number of incoming edges
2724 ///
2725 unsigned getNumIncomingValues() const { return getNumOperands(); }
2726
2727 /// Return incoming value number x
2728 ///
2729 Value *getIncomingValue(unsigned i) const {
2730 return getOperand(i);
2731 }
2732 void setIncomingValue(unsigned i, Value *V) {
2733 assert(V && "PHI node got a null value!")(static_cast <bool> (V && "PHI node got a null value!"
) ? void (0) : __assert_fail ("V && \"PHI node got a null value!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2733, __extension__ __PRETTY_FUNCTION__))
;
2734 assert(getType() == V->getType() &&(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2735, __extension__ __PRETTY_FUNCTION__))
2735 "All operands to PHI node must be the same type as the PHI node!")(static_cast <bool> (getType() == V->getType() &&
"All operands to PHI node must be the same type as the PHI node!"
) ? void (0) : __assert_fail ("getType() == V->getType() && \"All operands to PHI node must be the same type as the PHI node!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2735, __extension__ __PRETTY_FUNCTION__))
;
2736 setOperand(i, V);
2737 }
2738
2739 static unsigned getOperandNumForIncomingValue(unsigned i) {
2740 return i;
2741 }
2742
2743 static unsigned getIncomingValueNumForOperand(unsigned i) {
2744 return i;
2745 }
2746
2747 /// Return incoming basic block number @p i.
2748 ///
2749 BasicBlock *getIncomingBlock(unsigned i) const {
2750 return block_begin()[i];
2751 }
2752
2753 /// Return incoming basic block corresponding
2754 /// to an operand of the PHI.
2755 ///
2756 BasicBlock *getIncomingBlock(const Use &U) const {
2757 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?")(static_cast <bool> (this == U.getUser() && "Iterator doesn't point to PHI's Uses?"
) ? void (0) : __assert_fail ("this == U.getUser() && \"Iterator doesn't point to PHI's Uses?\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2757, __extension__ __PRETTY_FUNCTION__))
;
2758 return getIncomingBlock(unsigned(&U - op_begin()));
2759 }
2760
2761 /// Return incoming basic block corresponding
2762 /// to value use iterator.
2763 ///
2764 BasicBlock *getIncomingBlock(Value::const_user_iterator I) const {
2765 return getIncomingBlock(I.getUse());
2766 }
2767
2768 void setIncomingBlock(unsigned i, BasicBlock *BB) {
2769 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2769, __extension__ __PRETTY_FUNCTION__))
;
2770 block_begin()[i] = BB;
2771 }
2772
2773 /// Replace every incoming basic block \p Old to basic block \p New.
2774 void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
2775 assert(New && Old && "PHI node got a null basic block!")(static_cast <bool> (New && Old && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("New && Old && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2775, __extension__ __PRETTY_FUNCTION__))
;
2776 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2777 if (getIncomingBlock(Op) == Old)
2778 setIncomingBlock(Op, New);
2779 }
2780
2781 /// Add an incoming value to the end of the PHI list
2782 ///
2783 void addIncoming(Value *V, BasicBlock *BB) {
2784 if (getNumOperands() == ReservedSpace)
2785 growOperands(); // Get more space!
2786 // Initialize some new operands.
2787 setNumHungOffUseOperands(getNumOperands() + 1);
2788 setIncomingValue(getNumOperands() - 1, V);
2789 setIncomingBlock(getNumOperands() - 1, BB);
2790 }
2791
2792 /// Remove an incoming value. This is useful if a
2793 /// predecessor basic block is deleted. The value removed is returned.
2794 ///
2795 /// If the last incoming value for a PHI node is removed (and DeletePHIIfEmpty
2796 /// is true), the PHI node is destroyed and any uses of it are replaced with
2797 /// dummy values. The only time there should be zero incoming values to a PHI
2798 /// node is when the block is dead, so this strategy is sound.
2799 ///
2800 Value *removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty = true);
2801
2802 Value *removeIncomingValue(const BasicBlock *BB, bool DeletePHIIfEmpty=true) {
2803 int Idx = getBasicBlockIndex(BB);
2804 assert(Idx >= 0 && "Invalid basic block argument to remove!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument to remove!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument to remove!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2804, __extension__ __PRETTY_FUNCTION__))
;
2805 return removeIncomingValue(Idx, DeletePHIIfEmpty);
2806 }
2807
2808 /// Return the first index of the specified basic
2809 /// block in the value list for this PHI. Returns -1 if no instance.
2810 ///
2811 int getBasicBlockIndex(const BasicBlock *BB) const {
2812 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
2813 if (block_begin()[i] == BB)
2814 return i;
2815 return -1;
2816 }
2817
2818 Value *getIncomingValueForBlock(const BasicBlock *BB) const {
2819 int Idx = getBasicBlockIndex(BB);
2820 assert(Idx >= 0 && "Invalid basic block argument!")(static_cast <bool> (Idx >= 0 && "Invalid basic block argument!"
) ? void (0) : __assert_fail ("Idx >= 0 && \"Invalid basic block argument!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2820, __extension__ __PRETTY_FUNCTION__))
;
2821 return getIncomingValue(Idx);
2822 }
2823
2824 /// Set every incoming value(s) for block \p BB to \p V.
2825 void setIncomingValueForBlock(const BasicBlock *BB, Value *V) {
2826 assert(BB && "PHI node got a null basic block!")(static_cast <bool> (BB && "PHI node got a null basic block!"
) ? void (0) : __assert_fail ("BB && \"PHI node got a null basic block!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2826, __extension__ __PRETTY_FUNCTION__))
;
2827 bool Found = false;
2828 for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op)
2829 if (getIncomingBlock(Op) == BB) {
2830 Found = true;
2831 setIncomingValue(Op, V);
2832 }
2833 (void)Found;
2834 assert(Found && "Invalid basic block argument to set!")(static_cast <bool> (Found && "Invalid basic block argument to set!"
) ? void (0) : __assert_fail ("Found && \"Invalid basic block argument to set!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2834, __extension__ __PRETTY_FUNCTION__))
;
2835 }
2836
2837 /// If the specified PHI node always merges together the
2838 /// same value, return the value, otherwise return null.
2839 Value *hasConstantValue() const;
2840
2841 /// Whether the specified PHI node always merges
2842 /// together the same value, assuming undefs are equal to a unique
2843 /// non-undef value.
2844 bool hasConstantOrUndefValue() const;
2845
2846 /// If the PHI node is complete which means all of its parent's predecessors
2847 /// have incoming value in this PHI, return true, otherwise return false.
2848 bool isComplete() const {
2849 return llvm::all_of(predecessors(getParent()),
2850 [this](const BasicBlock *Pred) {
2851 return getBasicBlockIndex(Pred) >= 0;
2852 });
2853 }
2854
2855 /// Methods for support type inquiry through isa, cast, and dyn_cast:
2856 static bool classof(const Instruction *I) {
2857 return I->getOpcode() == Instruction::PHI;
2858 }
2859 static bool classof(const Value *V) {
2860 return isa<Instruction>(V) && classof(cast<Instruction>(V));
2861 }
2862
2863private:
2864 void growOperands();
2865};
2866
2867template <>
2868struct OperandTraits<PHINode> : public HungoffOperandTraits<2> {
2869};
2870
2871DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PHINode, Value)PHINode::op_iterator PHINode::op_begin() { return OperandTraits
<PHINode>::op_begin(this); } PHINode::const_op_iterator
PHINode::op_begin() const { return OperandTraits<PHINode>
::op_begin(const_cast<PHINode*>(this)); } PHINode::op_iterator
PHINode::op_end() { return OperandTraits<PHINode>::op_end
(this); } PHINode::const_op_iterator PHINode::op_end() const {
return OperandTraits<PHINode>::op_end(const_cast<PHINode
*>(this)); } Value *PHINode::getOperand(unsigned i_nocapture
) const { (static_cast <bool> (i_nocapture < OperandTraits
<PHINode>::operands(this) && "getOperand() out of range!"
) ? void (0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2871, __extension__ __PRETTY_FUNCTION__)); return cast_or_null
<Value>( OperandTraits<PHINode>::op_begin(const_cast
<PHINode*>(this))[i_nocapture].get()); } void PHINode::
setOperand(unsigned i_nocapture, Value *Val_nocapture) { (static_cast
<bool> (i_nocapture < OperandTraits<PHINode>::
operands(this) && "setOperand() out of range!") ? void
(0) : __assert_fail ("i_nocapture < OperandTraits<PHINode>::operands(this) && \"setOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20210926122410+d23fd8ae8906/llvm/include/llvm/IR/Instructions.h"
, 2871, __extension__ __PRETTY_FUNCTION__)); OperandTraits<
PHINode>::op_begin(this)[i_nocapture] = Val_nocapture; } unsigned
PHINode::getNumOperands() const { return OperandTraits<PHINode
>::operands(this); } template <int Idx_nocapture> Use
&PHINode::Op() { return this->OpFrom<Idx_nocapture
>(this); } template <int Idx_nocapture> const Use &
PHINode::Op() const { return this->OpFrom<Idx_nocapture
>(this); }
2872
2873//===----------------------------------------------------------------------===//
2874// LandingPadInst Class
2875//===----------------------------------------------------------------------===//
2876
2877//===---------------------------------------------------------------------------
2878/// The landingpad instruction holds all of the information
2879/// necessary to generate correct exception handling. The landingpad instruction
2880/// cannot be moved from the top of a landing pad block, which itself is
2881/// accessible only from the 'unwind' edge of an invoke. This uses the
2882/// SubclassData field in Value to store whether or not the landingpad is a
2883/// cleanup.
2884///
2885class LandingPadInst : public Instruction {
2886 using CleanupField = BoolBitfieldElementT<0>;
2887
2888 /// The number of operands actually allocated. NumOperands is
2889 /// the number actually in use.
2890 unsigned ReservedSpace;
2891
2892 LandingPadInst(const LandingPadInst &LP);
2893
2894public:
2895 enum ClauseType { Catch, Filter };
2896
2897private:
2898 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2899 const Twine &NameStr, Instruction *InsertBefore);
2900 explicit LandingPadInst(Type *RetTy, unsigned NumReservedValues,
2901 const Twine &NameStr, BasicBlock *InsertAtEnd);
2902
2903 // Allocate space for exactly zero operands.
2904 void *operator new(size_t S) { return User::operator new(S); }
2905
2906 void growOperands(unsigned Size);
2907 void init(unsigned NumReservedValues, const Twine &NameStr);
2908
2909protected:
2910 // Note: Instruction needs to be a friend here to call cloneImpl.
2911 friend class Instruction;
2912
2913 LandingPadInst *cloneImpl() const;
2914
2915public:
2916 void operator delete(void *Ptr) { User::operator delete(Ptr); }
2917
2918 /// Constructors - NumReservedClauses is a hint for the number of incoming
2919 /// clauses that this landingpad will have (use 0 if you really have no idea).
2920 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2921 const Twine &NameStr = "",
2922 Instruction *InsertBefore = nullptr);
2923 static LandingPadInst *Create(Type *RetTy, unsigned NumReservedClauses,
2924 const Twine &NameStr, BasicBlock *InsertAtEnd);
2925