File: | llvm/lib/CodeGen/IfConversion.cpp |
Warning: | line 758, column 7 The expression is an uninitialized value. The computed value will also be garbage |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- IfConversion.cpp - Machine code if conversion pass -----------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the machine instruction level if-conversion pass, which | |||
10 | // tries to convert conditional branches into predicated instructions. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "BranchFolding.h" | |||
15 | #include "llvm/ADT/STLExtras.h" | |||
16 | #include "llvm/ADT/ScopeExit.h" | |||
17 | #include "llvm/ADT/SmallSet.h" | |||
18 | #include "llvm/ADT/SmallVector.h" | |||
19 | #include "llvm/ADT/SparseSet.h" | |||
20 | #include "llvm/ADT/Statistic.h" | |||
21 | #include "llvm/ADT/iterator_range.h" | |||
22 | #include "llvm/Analysis/ProfileSummaryInfo.h" | |||
23 | #include "llvm/CodeGen/LivePhysRegs.h" | |||
24 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
25 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" | |||
26 | #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" | |||
27 | #include "llvm/CodeGen/MachineFunction.h" | |||
28 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
29 | #include "llvm/CodeGen/MachineInstr.h" | |||
30 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
31 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
32 | #include "llvm/CodeGen/MachineOperand.h" | |||
33 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
34 | #include "llvm/CodeGen/MBFIWrapper.h" | |||
35 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
36 | #include "llvm/CodeGen/TargetLowering.h" | |||
37 | #include "llvm/CodeGen/TargetRegisterInfo.h" | |||
38 | #include "llvm/CodeGen/TargetSchedule.h" | |||
39 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
40 | #include "llvm/IR/Attributes.h" | |||
41 | #include "llvm/IR/DebugLoc.h" | |||
42 | #include "llvm/InitializePasses.h" | |||
43 | #include "llvm/MC/MCRegisterInfo.h" | |||
44 | #include "llvm/Pass.h" | |||
45 | #include "llvm/Support/BranchProbability.h" | |||
46 | #include "llvm/Support/CommandLine.h" | |||
47 | #include "llvm/Support/Debug.h" | |||
48 | #include "llvm/Support/ErrorHandling.h" | |||
49 | #include "llvm/Support/raw_ostream.h" | |||
50 | #include <algorithm> | |||
51 | #include <cassert> | |||
52 | #include <functional> | |||
53 | #include <iterator> | |||
54 | #include <memory> | |||
55 | #include <utility> | |||
56 | #include <vector> | |||
57 | ||||
58 | using namespace llvm; | |||
59 | ||||
60 | #define DEBUG_TYPE"if-converter" "if-converter" | |||
61 | ||||
62 | // Hidden options for help debugging. | |||
63 | static cl::opt<int> IfCvtFnStart("ifcvt-fn-start", cl::init(-1), cl::Hidden); | |||
64 | static cl::opt<int> IfCvtFnStop("ifcvt-fn-stop", cl::init(-1), cl::Hidden); | |||
65 | static cl::opt<int> IfCvtLimit("ifcvt-limit", cl::init(-1), cl::Hidden); | |||
66 | static cl::opt<bool> DisableSimple("disable-ifcvt-simple", | |||
67 | cl::init(false), cl::Hidden); | |||
68 | static cl::opt<bool> DisableSimpleF("disable-ifcvt-simple-false", | |||
69 | cl::init(false), cl::Hidden); | |||
70 | static cl::opt<bool> DisableTriangle("disable-ifcvt-triangle", | |||
71 | cl::init(false), cl::Hidden); | |||
72 | static cl::opt<bool> DisableTriangleR("disable-ifcvt-triangle-rev", | |||
73 | cl::init(false), cl::Hidden); | |||
74 | static cl::opt<bool> DisableTriangleF("disable-ifcvt-triangle-false", | |||
75 | cl::init(false), cl::Hidden); | |||
76 | static cl::opt<bool> DisableTriangleFR("disable-ifcvt-triangle-false-rev", | |||
77 | cl::init(false), cl::Hidden); | |||
78 | static cl::opt<bool> DisableDiamond("disable-ifcvt-diamond", | |||
79 | cl::init(false), cl::Hidden); | |||
80 | static cl::opt<bool> DisableForkedDiamond("disable-ifcvt-forked-diamond", | |||
81 | cl::init(false), cl::Hidden); | |||
82 | static cl::opt<bool> IfCvtBranchFold("ifcvt-branch-fold", | |||
83 | cl::init(true), cl::Hidden); | |||
84 | ||||
85 | STATISTIC(NumSimple, "Number of simple if-conversions performed")static llvm::Statistic NumSimple = {"if-converter", "NumSimple" , "Number of simple if-conversions performed"}; | |||
86 | STATISTIC(NumSimpleFalse, "Number of simple (F) if-conversions performed")static llvm::Statistic NumSimpleFalse = {"if-converter", "NumSimpleFalse" , "Number of simple (F) if-conversions performed"}; | |||
87 | STATISTIC(NumTriangle, "Number of triangle if-conversions performed")static llvm::Statistic NumTriangle = {"if-converter", "NumTriangle" , "Number of triangle if-conversions performed"}; | |||
88 | STATISTIC(NumTriangleRev, "Number of triangle (R) if-conversions performed")static llvm::Statistic NumTriangleRev = {"if-converter", "NumTriangleRev" , "Number of triangle (R) if-conversions performed"}; | |||
89 | STATISTIC(NumTriangleFalse,"Number of triangle (F) if-conversions performed")static llvm::Statistic NumTriangleFalse = {"if-converter", "NumTriangleFalse" , "Number of triangle (F) if-conversions performed"}; | |||
90 | STATISTIC(NumTriangleFRev, "Number of triangle (F/R) if-conversions performed")static llvm::Statistic NumTriangleFRev = {"if-converter", "NumTriangleFRev" , "Number of triangle (F/R) if-conversions performed"}; | |||
91 | STATISTIC(NumDiamonds, "Number of diamond if-conversions performed")static llvm::Statistic NumDiamonds = {"if-converter", "NumDiamonds" , "Number of diamond if-conversions performed"}; | |||
92 | STATISTIC(NumForkedDiamonds, "Number of forked-diamond if-conversions performed")static llvm::Statistic NumForkedDiamonds = {"if-converter", "NumForkedDiamonds" , "Number of forked-diamond if-conversions performed"}; | |||
93 | STATISTIC(NumIfConvBBs, "Number of if-converted blocks")static llvm::Statistic NumIfConvBBs = {"if-converter", "NumIfConvBBs" , "Number of if-converted blocks"}; | |||
94 | STATISTIC(NumDupBBs, "Number of duplicated blocks")static llvm::Statistic NumDupBBs = {"if-converter", "NumDupBBs" , "Number of duplicated blocks"}; | |||
95 | STATISTIC(NumUnpred, "Number of true blocks of diamonds unpredicated")static llvm::Statistic NumUnpred = {"if-converter", "NumUnpred" , "Number of true blocks of diamonds unpredicated"}; | |||
96 | ||||
97 | namespace { | |||
98 | ||||
99 | class IfConverter : public MachineFunctionPass { | |||
100 | enum IfcvtKind { | |||
101 | ICNotClassfied, // BB data valid, but not classified. | |||
102 | ICSimpleFalse, // Same as ICSimple, but on the false path. | |||
103 | ICSimple, // BB is entry of an one split, no rejoin sub-CFG. | |||
104 | ICTriangleFRev, // Same as ICTriangleFalse, but false path rev condition. | |||
105 | ICTriangleRev, // Same as ICTriangle, but true path rev condition. | |||
106 | ICTriangleFalse, // Same as ICTriangle, but on the false path. | |||
107 | ICTriangle, // BB is entry of a triangle sub-CFG. | |||
108 | ICDiamond, // BB is entry of a diamond sub-CFG. | |||
109 | ICForkedDiamond // BB is entry of an almost diamond sub-CFG, with a | |||
110 | // common tail that can be shared. | |||
111 | }; | |||
112 | ||||
113 | /// One per MachineBasicBlock, this is used to cache the result | |||
114 | /// if-conversion feasibility analysis. This includes results from | |||
115 | /// TargetInstrInfo::analyzeBranch() (i.e. TBB, FBB, and Cond), and its | |||
116 | /// classification, and common tail block of its successors (if it's a | |||
117 | /// diamond shape), its size, whether it's predicable, and whether any | |||
118 | /// instruction can clobber the 'would-be' predicate. | |||
119 | /// | |||
120 | /// IsDone - True if BB is not to be considered for ifcvt. | |||
121 | /// IsBeingAnalyzed - True if BB is currently being analyzed. | |||
122 | /// IsAnalyzed - True if BB has been analyzed (info is still valid). | |||
123 | /// IsEnqueued - True if BB has been enqueued to be ifcvt'ed. | |||
124 | /// IsBrAnalyzable - True if analyzeBranch() returns false. | |||
125 | /// HasFallThrough - True if BB may fallthrough to the following BB. | |||
126 | /// IsUnpredicable - True if BB is known to be unpredicable. | |||
127 | /// ClobbersPred - True if BB could modify predicates (e.g. has | |||
128 | /// cmp, call, etc.) | |||
129 | /// NonPredSize - Number of non-predicated instructions. | |||
130 | /// ExtraCost - Extra cost for multi-cycle instructions. | |||
131 | /// ExtraCost2 - Some instructions are slower when predicated | |||
132 | /// BB - Corresponding MachineBasicBlock. | |||
133 | /// TrueBB / FalseBB- See analyzeBranch(). | |||
134 | /// BrCond - Conditions for end of block conditional branches. | |||
135 | /// Predicate - Predicate used in the BB. | |||
136 | struct BBInfo { | |||
137 | bool IsDone : 1; | |||
138 | bool IsBeingAnalyzed : 1; | |||
139 | bool IsAnalyzed : 1; | |||
140 | bool IsEnqueued : 1; | |||
141 | bool IsBrAnalyzable : 1; | |||
142 | bool IsBrReversible : 1; | |||
143 | bool HasFallThrough : 1; | |||
144 | bool IsUnpredicable : 1; | |||
145 | bool CannotBeCopied : 1; | |||
146 | bool ClobbersPred : 1; | |||
147 | unsigned NonPredSize = 0; | |||
148 | unsigned ExtraCost = 0; | |||
149 | unsigned ExtraCost2 = 0; | |||
150 | MachineBasicBlock *BB = nullptr; | |||
151 | MachineBasicBlock *TrueBB = nullptr; | |||
152 | MachineBasicBlock *FalseBB = nullptr; | |||
153 | SmallVector<MachineOperand, 4> BrCond; | |||
154 | SmallVector<MachineOperand, 4> Predicate; | |||
155 | ||||
156 | BBInfo() : IsDone(false), IsBeingAnalyzed(false), | |||
157 | IsAnalyzed(false), IsEnqueued(false), IsBrAnalyzable(false), | |||
158 | IsBrReversible(false), HasFallThrough(false), | |||
159 | IsUnpredicable(false), CannotBeCopied(false), | |||
160 | ClobbersPred(false) {} | |||
161 | }; | |||
162 | ||||
163 | /// Record information about pending if-conversions to attempt: | |||
164 | /// BBI - Corresponding BBInfo. | |||
165 | /// Kind - Type of block. See IfcvtKind. | |||
166 | /// NeedSubsumption - True if the to-be-predicated BB has already been | |||
167 | /// predicated. | |||
168 | /// NumDups - Number of instructions that would be duplicated due | |||
169 | /// to this if-conversion. (For diamonds, the number of | |||
170 | /// identical instructions at the beginnings of both | |||
171 | /// paths). | |||
172 | /// NumDups2 - For diamonds, the number of identical instructions | |||
173 | /// at the ends of both paths. | |||
174 | struct IfcvtToken { | |||
175 | BBInfo &BBI; | |||
176 | IfcvtKind Kind; | |||
177 | unsigned NumDups; | |||
178 | unsigned NumDups2; | |||
179 | bool NeedSubsumption : 1; | |||
180 | bool TClobbersPred : 1; | |||
181 | bool FClobbersPred : 1; | |||
182 | ||||
183 | IfcvtToken(BBInfo &b, IfcvtKind k, bool s, unsigned d, unsigned d2 = 0, | |||
184 | bool tc = false, bool fc = false) | |||
185 | : BBI(b), Kind(k), NumDups(d), NumDups2(d2), NeedSubsumption(s), | |||
186 | TClobbersPred(tc), FClobbersPred(fc) {} | |||
187 | }; | |||
188 | ||||
189 | /// Results of if-conversion feasibility analysis indexed by basic block | |||
190 | /// number. | |||
191 | std::vector<BBInfo> BBAnalysis; | |||
192 | TargetSchedModel SchedModel; | |||
193 | ||||
194 | const TargetLoweringBase *TLI; | |||
195 | const TargetInstrInfo *TII; | |||
196 | const TargetRegisterInfo *TRI; | |||
197 | const MachineBranchProbabilityInfo *MBPI; | |||
198 | MachineRegisterInfo *MRI; | |||
199 | ||||
200 | LivePhysRegs Redefs; | |||
201 | ||||
202 | bool PreRegAlloc; | |||
203 | bool MadeChange; | |||
204 | int FnNum = -1; | |||
205 | std::function<bool(const MachineFunction &)> PredicateFtor; | |||
206 | ||||
207 | public: | |||
208 | static char ID; | |||
209 | ||||
210 | IfConverter(std::function<bool(const MachineFunction &)> Ftor = nullptr) | |||
211 | : MachineFunctionPass(ID), PredicateFtor(std::move(Ftor)) { | |||
212 | initializeIfConverterPass(*PassRegistry::getPassRegistry()); | |||
213 | } | |||
214 | ||||
215 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
216 | AU.addRequired<MachineBlockFrequencyInfo>(); | |||
217 | AU.addRequired<MachineBranchProbabilityInfo>(); | |||
218 | AU.addRequired<ProfileSummaryInfoWrapperPass>(); | |||
219 | MachineFunctionPass::getAnalysisUsage(AU); | |||
220 | } | |||
221 | ||||
222 | bool runOnMachineFunction(MachineFunction &MF) override; | |||
223 | ||||
224 | MachineFunctionProperties getRequiredProperties() const override { | |||
225 | return MachineFunctionProperties().set( | |||
226 | MachineFunctionProperties::Property::NoVRegs); | |||
227 | } | |||
228 | ||||
229 | private: | |||
230 | bool reverseBranchCondition(BBInfo &BBI) const; | |||
231 | bool ValidSimple(BBInfo &TrueBBI, unsigned &Dups, | |||
232 | BranchProbability Prediction) const; | |||
233 | bool ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
234 | bool FalseBranch, unsigned &Dups, | |||
235 | BranchProbability Prediction) const; | |||
236 | bool CountDuplicatedInstructions( | |||
237 | MachineBasicBlock::iterator &TIB, MachineBasicBlock::iterator &FIB, | |||
238 | MachineBasicBlock::iterator &TIE, MachineBasicBlock::iterator &FIE, | |||
239 | unsigned &Dups1, unsigned &Dups2, | |||
240 | MachineBasicBlock &TBB, MachineBasicBlock &FBB, | |||
241 | bool SkipUnconditionalBranches) const; | |||
242 | bool ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
243 | unsigned &Dups1, unsigned &Dups2, | |||
244 | BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const; | |||
245 | bool ValidForkedDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
246 | unsigned &Dups1, unsigned &Dups2, | |||
247 | BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const; | |||
248 | void AnalyzeBranches(BBInfo &BBI); | |||
249 | void ScanInstructions(BBInfo &BBI, | |||
250 | MachineBasicBlock::iterator &Begin, | |||
251 | MachineBasicBlock::iterator &End, | |||
252 | bool BranchUnpredicable = false) const; | |||
253 | bool RescanInstructions( | |||
254 | MachineBasicBlock::iterator &TIB, MachineBasicBlock::iterator &FIB, | |||
255 | MachineBasicBlock::iterator &TIE, MachineBasicBlock::iterator &FIE, | |||
256 | BBInfo &TrueBBI, BBInfo &FalseBBI) const; | |||
257 | void AnalyzeBlock(MachineBasicBlock &MBB, | |||
258 | std::vector<std::unique_ptr<IfcvtToken>> &Tokens); | |||
259 | bool FeasibilityAnalysis(BBInfo &BBI, SmallVectorImpl<MachineOperand> &Pred, | |||
260 | bool isTriangle = false, bool RevBranch = false, | |||
261 | bool hasCommonTail = false); | |||
262 | void AnalyzeBlocks(MachineFunction &MF, | |||
263 | std::vector<std::unique_ptr<IfcvtToken>> &Tokens); | |||
264 | void InvalidatePreds(MachineBasicBlock &MBB); | |||
265 | bool IfConvertSimple(BBInfo &BBI, IfcvtKind Kind); | |||
266 | bool IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind); | |||
267 | bool IfConvertDiamondCommon(BBInfo &BBI, BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
268 | unsigned NumDups1, unsigned NumDups2, | |||
269 | bool TClobbersPred, bool FClobbersPred, | |||
270 | bool RemoveBranch, bool MergeAddEdges); | |||
271 | bool IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind, | |||
272 | unsigned NumDups1, unsigned NumDups2, | |||
273 | bool TClobbers, bool FClobbers); | |||
274 | bool IfConvertForkedDiamond(BBInfo &BBI, IfcvtKind Kind, | |||
275 | unsigned NumDups1, unsigned NumDups2, | |||
276 | bool TClobbers, bool FClobbers); | |||
277 | void PredicateBlock(BBInfo &BBI, | |||
278 | MachineBasicBlock::iterator E, | |||
279 | SmallVectorImpl<MachineOperand> &Cond, | |||
280 | SmallSet<MCPhysReg, 4> *LaterRedefs = nullptr); | |||
281 | void CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, | |||
282 | SmallVectorImpl<MachineOperand> &Cond, | |||
283 | bool IgnoreBr = false); | |||
284 | void MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges = true); | |||
285 | ||||
286 | bool MeetIfcvtSizeLimit(MachineBasicBlock &BB, | |||
287 | unsigned Cycle, unsigned Extra, | |||
288 | BranchProbability Prediction) const { | |||
289 | return Cycle > 0 && TII->isProfitableToIfCvt(BB, Cycle, Extra, | |||
290 | Prediction); | |||
291 | } | |||
292 | ||||
293 | bool MeetIfcvtSizeLimit(BBInfo &TBBInfo, BBInfo &FBBInfo, | |||
294 | MachineBasicBlock &CommBB, unsigned Dups, | |||
295 | BranchProbability Prediction, bool Forked) const { | |||
296 | const MachineFunction &MF = *TBBInfo.BB->getParent(); | |||
297 | if (MF.getFunction().hasMinSize()) { | |||
| ||||
298 | MachineBasicBlock::iterator TIB = TBBInfo.BB->begin(); | |||
299 | MachineBasicBlock::iterator FIB = FBBInfo.BB->begin(); | |||
300 | MachineBasicBlock::iterator TIE = TBBInfo.BB->end(); | |||
301 | MachineBasicBlock::iterator FIE = FBBInfo.BB->end(); | |||
302 | ||||
303 | unsigned Dups1, Dups2; | |||
304 | if (!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2, | |||
305 | *TBBInfo.BB, *FBBInfo.BB, | |||
306 | /*SkipUnconditionalBranches*/ true)) | |||
307 | llvm_unreachable("should already have been checked by ValidDiamond")::llvm::llvm_unreachable_internal("should already have been checked by ValidDiamond" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 307); | |||
308 | ||||
309 | unsigned BranchBytes = 0; | |||
310 | unsigned CommonBytes = 0; | |||
311 | ||||
312 | // Count common instructions at the start of the true and false blocks. | |||
313 | for (auto &I : make_range(TBBInfo.BB->begin(), TIB)) { | |||
314 | LLVM_DEBUG(dbgs() << "Common inst: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Common inst: " << I ; } } while (false); | |||
315 | CommonBytes += TII->getInstSizeInBytes(I); | |||
316 | } | |||
317 | for (auto &I : make_range(FBBInfo.BB->begin(), FIB)) { | |||
318 | LLVM_DEBUG(dbgs() << "Common inst: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Common inst: " << I ; } } while (false); | |||
319 | CommonBytes += TII->getInstSizeInBytes(I); | |||
320 | } | |||
321 | ||||
322 | // Count instructions at the end of the true and false blocks, after | |||
323 | // the ones we plan to predicate. Analyzable branches will be removed | |||
324 | // (unless this is a forked diamond), and all other instructions are | |||
325 | // common between the two blocks. | |||
326 | for (auto &I : make_range(TIE, TBBInfo.BB->end())) { | |||
327 | if (I.isBranch() && TBBInfo.IsBrAnalyzable && !Forked) { | |||
328 | LLVM_DEBUG(dbgs() << "Saving branch: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Saving branch: " << I; } } while (false); | |||
329 | BranchBytes += TII->predictBranchSizeForIfCvt(I); | |||
330 | } else { | |||
331 | LLVM_DEBUG(dbgs() << "Common inst: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Common inst: " << I ; } } while (false); | |||
332 | CommonBytes += TII->getInstSizeInBytes(I); | |||
333 | } | |||
334 | } | |||
335 | for (auto &I : make_range(FIE, FBBInfo.BB->end())) { | |||
336 | if (I.isBranch() && FBBInfo.IsBrAnalyzable && !Forked) { | |||
337 | LLVM_DEBUG(dbgs() << "Saving branch: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Saving branch: " << I; } } while (false); | |||
338 | BranchBytes += TII->predictBranchSizeForIfCvt(I); | |||
339 | } else { | |||
340 | LLVM_DEBUG(dbgs() << "Common inst: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Common inst: " << I ; } } while (false); | |||
341 | CommonBytes += TII->getInstSizeInBytes(I); | |||
342 | } | |||
343 | } | |||
344 | for (auto &I : CommBB.terminators()) { | |||
345 | if (I.isBranch()) { | |||
346 | LLVM_DEBUG(dbgs() << "Saving branch: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Saving branch: " << I; } } while (false); | |||
347 | BranchBytes += TII->predictBranchSizeForIfCvt(I); | |||
348 | } | |||
349 | } | |||
350 | ||||
351 | // The common instructions in one branch will be eliminated, halving | |||
352 | // their code size. | |||
353 | CommonBytes /= 2; | |||
354 | ||||
355 | // Count the instructions which we need to predicate. | |||
356 | unsigned NumPredicatedInstructions = 0; | |||
357 | for (auto &I : make_range(TIB, TIE)) { | |||
358 | if (!I.isDebugInstr()) { | |||
359 | LLVM_DEBUG(dbgs() << "Predicating: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Predicating: " << I ; } } while (false); | |||
360 | NumPredicatedInstructions++; | |||
361 | } | |||
362 | } | |||
363 | for (auto &I : make_range(FIB, FIE)) { | |||
364 | if (!I.isDebugInstr()) { | |||
365 | LLVM_DEBUG(dbgs() << "Predicating: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Predicating: " << I ; } } while (false); | |||
366 | NumPredicatedInstructions++; | |||
367 | } | |||
368 | } | |||
369 | ||||
370 | // Even though we're optimising for size at the expense of performance, | |||
371 | // avoid creating really long predicated blocks. | |||
372 | if (NumPredicatedInstructions > 15) | |||
373 | return false; | |||
374 | ||||
375 | // Some targets (e.g. Thumb2) need to insert extra instructions to | |||
376 | // start predicated blocks. | |||
377 | unsigned ExtraPredicateBytes = TII->extraSizeToPredicateInstructions( | |||
378 | MF, NumPredicatedInstructions); | |||
379 | ||||
380 | LLVM_DEBUG(dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytesdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes << ", CommonBytes=" << CommonBytes << ", NumPredicatedInstructions=" << NumPredicatedInstructions << ", ExtraPredicateBytes=" << ExtraPredicateBytes << ")\n"; } } while (false) | |||
381 | << ", CommonBytes=" << CommonBytesdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes << ", CommonBytes=" << CommonBytes << ", NumPredicatedInstructions=" << NumPredicatedInstructions << ", ExtraPredicateBytes=" << ExtraPredicateBytes << ")\n"; } } while (false) | |||
382 | << ", NumPredicatedInstructions="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes << ", CommonBytes=" << CommonBytes << ", NumPredicatedInstructions=" << NumPredicatedInstructions << ", ExtraPredicateBytes=" << ExtraPredicateBytes << ")\n"; } } while (false) | |||
383 | << NumPredicatedInstructionsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes << ", CommonBytes=" << CommonBytes << ", NumPredicatedInstructions=" << NumPredicatedInstructions << ", ExtraPredicateBytes=" << ExtraPredicateBytes << ")\n"; } } while (false) | |||
384 | << ", ExtraPredicateBytes=" << ExtraPredicateBytesdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes << ", CommonBytes=" << CommonBytes << ", NumPredicatedInstructions=" << NumPredicatedInstructions << ", ExtraPredicateBytes=" << ExtraPredicateBytes << ")\n"; } } while (false) | |||
385 | << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(BranchBytes=" << BranchBytes << ", CommonBytes=" << CommonBytes << ", NumPredicatedInstructions=" << NumPredicatedInstructions << ", ExtraPredicateBytes=" << ExtraPredicateBytes << ")\n"; } } while (false); | |||
386 | return (BranchBytes + CommonBytes) > ExtraPredicateBytes; | |||
387 | } else { | |||
388 | unsigned TCycle = TBBInfo.NonPredSize + TBBInfo.ExtraCost - Dups; | |||
389 | unsigned FCycle = FBBInfo.NonPredSize + FBBInfo.ExtraCost - Dups; | |||
390 | bool Res = TCycle > 0 && FCycle > 0 && | |||
391 | TII->isProfitableToIfCvt( | |||
392 | *TBBInfo.BB, TCycle, TBBInfo.ExtraCost2, *FBBInfo.BB, | |||
393 | FCycle, FBBInfo.ExtraCost2, Prediction); | |||
394 | LLVM_DEBUG(dbgs() << "MeetIfcvtSizeLimit(TCycle=" << TCycledo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(TCycle=" << TCycle << ", FCycle=" << FCycle << ", TExtra=" << TBBInfo.ExtraCost2 << ", FExtra=" << FBBInfo.ExtraCost2 << ") = " << Res << "\n"; } } while (false) | |||
395 | << ", FCycle=" << FCycledo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(TCycle=" << TCycle << ", FCycle=" << FCycle << ", TExtra=" << TBBInfo.ExtraCost2 << ", FExtra=" << FBBInfo.ExtraCost2 << ") = " << Res << "\n"; } } while (false) | |||
396 | << ", TExtra=" << TBBInfo.ExtraCost2 << ", FExtra="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(TCycle=" << TCycle << ", FCycle=" << FCycle << ", TExtra=" << TBBInfo.ExtraCost2 << ", FExtra=" << FBBInfo.ExtraCost2 << ") = " << Res << "\n"; } } while (false) | |||
397 | << FBBInfo.ExtraCost2 << ") = " << Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "MeetIfcvtSizeLimit(TCycle=" << TCycle << ", FCycle=" << FCycle << ", TExtra=" << TBBInfo.ExtraCost2 << ", FExtra=" << FBBInfo.ExtraCost2 << ") = " << Res << "\n"; } } while (false); | |||
398 | return Res; | |||
399 | } | |||
400 | } | |||
401 | ||||
402 | /// Returns true if Block ends without a terminator. | |||
403 | bool blockAlwaysFallThrough(BBInfo &BBI) const { | |||
404 | return BBI.IsBrAnalyzable && BBI.TrueBB == nullptr; | |||
405 | } | |||
406 | ||||
407 | /// Used to sort if-conversion candidates. | |||
408 | static bool IfcvtTokenCmp(const std::unique_ptr<IfcvtToken> &C1, | |||
409 | const std::unique_ptr<IfcvtToken> &C2) { | |||
410 | int Incr1 = (C1->Kind == ICDiamond) | |||
411 | ? -(int)(C1->NumDups + C1->NumDups2) : (int)C1->NumDups; | |||
412 | int Incr2 = (C2->Kind == ICDiamond) | |||
413 | ? -(int)(C2->NumDups + C2->NumDups2) : (int)C2->NumDups; | |||
414 | if (Incr1 > Incr2) | |||
415 | return true; | |||
416 | else if (Incr1 == Incr2) { | |||
417 | // Favors subsumption. | |||
418 | if (!C1->NeedSubsumption && C2->NeedSubsumption) | |||
419 | return true; | |||
420 | else if (C1->NeedSubsumption == C2->NeedSubsumption) { | |||
421 | // Favors diamond over triangle, etc. | |||
422 | if ((unsigned)C1->Kind < (unsigned)C2->Kind) | |||
423 | return true; | |||
424 | else if (C1->Kind == C2->Kind) | |||
425 | return C1->BBI.BB->getNumber() < C2->BBI.BB->getNumber(); | |||
426 | } | |||
427 | } | |||
428 | return false; | |||
429 | } | |||
430 | }; | |||
431 | ||||
432 | } // end anonymous namespace | |||
433 | ||||
434 | char IfConverter::ID = 0; | |||
435 | ||||
436 | char &llvm::IfConverterID = IfConverter::ID; | |||
437 | ||||
438 | INITIALIZE_PASS_BEGIN(IfConverter, DEBUG_TYPE, "If Converter", false, false)static void *initializeIfConverterPassOnce(PassRegistry & Registry) { | |||
439 | INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)initializeMachineBranchProbabilityInfoPass(Registry); | |||
440 | INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)initializeProfileSummaryInfoWrapperPassPass(Registry); | |||
441 | INITIALIZE_PASS_END(IfConverter, DEBUG_TYPE, "If Converter", false, false)PassInfo *PI = new PassInfo( "If Converter", "if-converter", & IfConverter::ID, PassInfo::NormalCtor_t(callDefaultCtor<IfConverter >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeIfConverterPassFlag; void llvm::initializeIfConverterPass(PassRegistry &Registry) { llvm::call_once(InitializeIfConverterPassFlag, initializeIfConverterPassOnce , std::ref(Registry)); } | |||
442 | ||||
443 | bool IfConverter::runOnMachineFunction(MachineFunction &MF) { | |||
444 | if (skipFunction(MF.getFunction()) || (PredicateFtor && !PredicateFtor(MF))) | |||
445 | return false; | |||
446 | ||||
447 | const TargetSubtargetInfo &ST = MF.getSubtarget(); | |||
448 | TLI = ST.getTargetLowering(); | |||
449 | TII = ST.getInstrInfo(); | |||
450 | TRI = ST.getRegisterInfo(); | |||
451 | MBFIWrapper MBFI(getAnalysis<MachineBlockFrequencyInfo>()); | |||
452 | MBPI = &getAnalysis<MachineBranchProbabilityInfo>(); | |||
453 | ProfileSummaryInfo *PSI = | |||
454 | &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); | |||
455 | MRI = &MF.getRegInfo(); | |||
456 | SchedModel.init(&ST); | |||
457 | ||||
458 | if (!TII) return false; | |||
459 | ||||
460 | PreRegAlloc = MRI->isSSA(); | |||
461 | ||||
462 | bool BFChange = false; | |||
463 | if (!PreRegAlloc) { | |||
464 | // Tail merge tend to expose more if-conversion opportunities. | |||
465 | BranchFolder BF(true, false, MBFI, *MBPI, PSI); | |||
466 | BFChange = BF.OptimizeFunction(MF, TII, ST.getRegisterInfo()); | |||
467 | } | |||
468 | ||||
469 | LLVM_DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'" << MF.getName() << "\'"; } } while (false) | |||
470 | << MF.getName() << "\'")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'" << MF.getName() << "\'"; } } while (false); | |||
471 | ||||
472 | if (FnNum < IfCvtFnStart || (IfCvtFnStop != -1 && FnNum > IfCvtFnStop)) { | |||
473 | LLVM_DEBUG(dbgs() << " skipped\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << " skipped\n"; } } while ( false); | |||
474 | return false; | |||
475 | } | |||
476 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "\n"; } } while (false); | |||
477 | ||||
478 | MF.RenumberBlocks(); | |||
479 | BBAnalysis.resize(MF.getNumBlockIDs()); | |||
480 | ||||
481 | std::vector<std::unique_ptr<IfcvtToken>> Tokens; | |||
482 | MadeChange = false; | |||
483 | unsigned NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + | |||
484 | NumTriangleRev + NumTriangleFalse + NumTriangleFRev + NumDiamonds; | |||
485 | while (IfCvtLimit == -1 || (int)NumIfCvts < IfCvtLimit) { | |||
486 | // Do an initial analysis for each basic block and find all the potential | |||
487 | // candidates to perform if-conversion. | |||
488 | bool Change = false; | |||
489 | AnalyzeBlocks(MF, Tokens); | |||
490 | while (!Tokens.empty()) { | |||
491 | std::unique_ptr<IfcvtToken> Token = std::move(Tokens.back()); | |||
492 | Tokens.pop_back(); | |||
493 | BBInfo &BBI = Token->BBI; | |||
494 | IfcvtKind Kind = Token->Kind; | |||
495 | unsigned NumDups = Token->NumDups; | |||
496 | unsigned NumDups2 = Token->NumDups2; | |||
497 | ||||
498 | // If the block has been evicted out of the queue or it has already been | |||
499 | // marked dead (due to it being predicated), then skip it. | |||
500 | if (BBI.IsDone) | |||
501 | BBI.IsEnqueued = false; | |||
502 | if (!BBI.IsEnqueued) | |||
503 | continue; | |||
504 | ||||
505 | BBI.IsEnqueued = false; | |||
506 | ||||
507 | bool RetVal = false; | |||
508 | switch (Kind) { | |||
509 | default: llvm_unreachable("Unexpected!")::llvm::llvm_unreachable_internal("Unexpected!", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 509); | |||
510 | case ICSimple: | |||
511 | case ICSimpleFalse: { | |||
512 | bool isFalse = Kind == ICSimpleFalse; | |||
513 | if ((isFalse && DisableSimpleF) || (!isFalse && DisableSimple)) break; | |||
514 | LLVM_DEBUG(dbgs() << "Ifcvt (Simple"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Simple" << ( Kind == ICSimpleFalse ? " false" : "") << "): " << printMBBReference(*BBI.BB) << " (" << ((Kind == ICSimpleFalse ) ? BBI.FalseBB->getNumber() : BBI.TrueBB->getNumber()) << ") "; } } while (false) | |||
515 | << (Kind == ICSimpleFalse ? " false" : "")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Simple" << ( Kind == ICSimpleFalse ? " false" : "") << "): " << printMBBReference(*BBI.BB) << " (" << ((Kind == ICSimpleFalse ) ? BBI.FalseBB->getNumber() : BBI.TrueBB->getNumber()) << ") "; } } while (false) | |||
516 | << "): " << printMBBReference(*BBI.BB) << " ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Simple" << ( Kind == ICSimpleFalse ? " false" : "") << "): " << printMBBReference(*BBI.BB) << " (" << ((Kind == ICSimpleFalse ) ? BBI.FalseBB->getNumber() : BBI.TrueBB->getNumber()) << ") "; } } while (false) | |||
517 | << ((Kind == ICSimpleFalse) ? BBI.FalseBB->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Simple" << ( Kind == ICSimpleFalse ? " false" : "") << "): " << printMBBReference(*BBI.BB) << " (" << ((Kind == ICSimpleFalse ) ? BBI.FalseBB->getNumber() : BBI.TrueBB->getNumber()) << ") "; } } while (false) | |||
518 | : BBI.TrueBB->getNumber())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Simple" << ( Kind == ICSimpleFalse ? " false" : "") << "): " << printMBBReference(*BBI.BB) << " (" << ((Kind == ICSimpleFalse ) ? BBI.FalseBB->getNumber() : BBI.TrueBB->getNumber()) << ") "; } } while (false) | |||
519 | << ") ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Simple" << ( Kind == ICSimpleFalse ? " false" : "") << "): " << printMBBReference(*BBI.BB) << " (" << ((Kind == ICSimpleFalse ) ? BBI.FalseBB->getNumber() : BBI.TrueBB->getNumber()) << ") "; } } while (false); | |||
520 | RetVal = IfConvertSimple(BBI, Kind); | |||
521 | LLVM_DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << (RetVal ? "succeeded!" : "failed!" ) << "\n"; } } while (false); | |||
522 | if (RetVal) { | |||
523 | if (isFalse) ++NumSimpleFalse; | |||
524 | else ++NumSimple; | |||
525 | } | |||
526 | break; | |||
527 | } | |||
528 | case ICTriangle: | |||
529 | case ICTriangleRev: | |||
530 | case ICTriangleFalse: | |||
531 | case ICTriangleFRev: { | |||
532 | bool isFalse = Kind == ICTriangleFalse; | |||
533 | bool isRev = (Kind == ICTriangleRev || Kind == ICTriangleFRev); | |||
534 | if (DisableTriangle && !isFalse && !isRev) break; | |||
535 | if (DisableTriangleR && !isFalse && isRev) break; | |||
536 | if (DisableTriangleF && isFalse && !isRev) break; | |||
537 | if (DisableTriangleFR && isFalse && isRev) break; | |||
538 | LLVM_DEBUG(dbgs() << "Ifcvt (Triangle")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Triangle"; } } while (false); | |||
539 | if (isFalse) | |||
540 | LLVM_DEBUG(dbgs() << " false")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << " false"; } } while (false ); | |||
541 | if (isRev) | |||
542 | LLVM_DEBUG(dbgs() << " rev")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << " rev"; } } while (false); | |||
543 | LLVM_DEBUG(dbgs() << "): " << printMBBReference(*BBI.BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "): " << printMBBReference (*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB->getNumber() << ") "; } } while (false) | |||
544 | << " (T:" << BBI.TrueBB->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "): " << printMBBReference (*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB->getNumber() << ") "; } } while (false) | |||
545 | << ",F:" << BBI.FalseBB->getNumber() << ") ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "): " << printMBBReference (*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB->getNumber() << ") "; } } while (false); | |||
546 | RetVal = IfConvertTriangle(BBI, Kind); | |||
547 | LLVM_DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << (RetVal ? "succeeded!" : "failed!" ) << "\n"; } } while (false); | |||
548 | if (RetVal) { | |||
549 | if (isFalse) { | |||
550 | if (isRev) ++NumTriangleFRev; | |||
551 | else ++NumTriangleFalse; | |||
552 | } else { | |||
553 | if (isRev) ++NumTriangleRev; | |||
554 | else ++NumTriangle; | |||
555 | } | |||
556 | } | |||
557 | break; | |||
558 | } | |||
559 | case ICDiamond: | |||
560 | if (DisableDiamond) break; | |||
561 | LLVM_DEBUG(dbgs() << "Ifcvt (Diamond): " << printMBBReference(*BBI.BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB ->getNumber() << ",F:" << BBI.FalseBB->getNumber () << ") "; } } while (false) | |||
562 | << " (T:" << BBI.TrueBB->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB ->getNumber() << ",F:" << BBI.FalseBB->getNumber () << ") "; } } while (false) | |||
563 | << ",F:" << BBI.FalseBB->getNumber() << ") ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB ->getNumber() << ",F:" << BBI.FalseBB->getNumber () << ") "; } } while (false); | |||
564 | RetVal = IfConvertDiamond(BBI, Kind, NumDups, NumDups2, | |||
565 | Token->TClobbersPred, | |||
566 | Token->FClobbersPred); | |||
567 | LLVM_DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << (RetVal ? "succeeded!" : "failed!" ) << "\n"; } } while (false); | |||
568 | if (RetVal) ++NumDiamonds; | |||
569 | break; | |||
570 | case ICForkedDiamond: | |||
571 | if (DisableForkedDiamond) break; | |||
572 | LLVM_DEBUG(dbgs() << "Ifcvt (Forked Diamond): "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Forked Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB ->getNumber() << ") "; } } while (false) | |||
573 | << printMBBReference(*BBI.BB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Forked Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB ->getNumber() << ") "; } } while (false) | |||
574 | << " (T:" << BBI.TrueBB->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Forked Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB ->getNumber() << ") "; } } while (false) | |||
575 | << ",F:" << BBI.FalseBB->getNumber() << ") ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << "Ifcvt (Forked Diamond): " << printMBBReference(*BBI.BB) << " (T:" << BBI.TrueBB->getNumber() << ",F:" << BBI.FalseBB ->getNumber() << ") "; } } while (false); | |||
576 | RetVal = IfConvertForkedDiamond(BBI, Kind, NumDups, NumDups2, | |||
577 | Token->TClobbersPred, | |||
578 | Token->FClobbersPred); | |||
579 | LLVM_DEBUG(dbgs() << (RetVal ? "succeeded!" : "failed!") << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("if-converter")) { dbgs() << (RetVal ? "succeeded!" : "failed!" ) << "\n"; } } while (false); | |||
580 | if (RetVal) ++NumForkedDiamonds; | |||
581 | break; | |||
582 | } | |||
583 | ||||
584 | if (RetVal && MRI->tracksLiveness()) | |||
585 | recomputeLivenessFlags(*BBI.BB); | |||
586 | ||||
587 | Change |= RetVal; | |||
588 | ||||
589 | NumIfCvts = NumSimple + NumSimpleFalse + NumTriangle + NumTriangleRev + | |||
590 | NumTriangleFalse + NumTriangleFRev + NumDiamonds; | |||
591 | if (IfCvtLimit != -1 && (int)NumIfCvts >= IfCvtLimit) | |||
592 | break; | |||
593 | } | |||
594 | ||||
595 | if (!Change) | |||
596 | break; | |||
597 | MadeChange |= Change; | |||
598 | } | |||
599 | ||||
600 | Tokens.clear(); | |||
601 | BBAnalysis.clear(); | |||
602 | ||||
603 | if (MadeChange && IfCvtBranchFold) { | |||
604 | BranchFolder BF(false, false, MBFI, *MBPI, PSI); | |||
605 | BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo()); | |||
606 | } | |||
607 | ||||
608 | MadeChange |= BFChange; | |||
609 | return MadeChange; | |||
610 | } | |||
611 | ||||
612 | /// BB has a fallthrough. Find its 'false' successor given its 'true' successor. | |||
613 | static MachineBasicBlock *findFalseBlock(MachineBasicBlock *BB, | |||
614 | MachineBasicBlock *TrueBB) { | |||
615 | for (MachineBasicBlock *SuccBB : BB->successors()) { | |||
616 | if (SuccBB != TrueBB) | |||
617 | return SuccBB; | |||
618 | } | |||
619 | return nullptr; | |||
620 | } | |||
621 | ||||
622 | /// Reverse the condition of the end of the block branch. Swap block's 'true' | |||
623 | /// and 'false' successors. | |||
624 | bool IfConverter::reverseBranchCondition(BBInfo &BBI) const { | |||
625 | DebugLoc dl; // FIXME: this is nowhere | |||
626 | if (!TII->reverseBranchCondition(BBI.BrCond)) { | |||
627 | TII->removeBranch(*BBI.BB); | |||
628 | TII->insertBranch(*BBI.BB, BBI.FalseBB, BBI.TrueBB, BBI.BrCond, dl); | |||
629 | std::swap(BBI.TrueBB, BBI.FalseBB); | |||
630 | return true; | |||
631 | } | |||
632 | return false; | |||
633 | } | |||
634 | ||||
635 | /// Returns the next block in the function blocks ordering. If it is the end, | |||
636 | /// returns NULL. | |||
637 | static inline MachineBasicBlock *getNextBlock(MachineBasicBlock &MBB) { | |||
638 | MachineFunction::iterator I = MBB.getIterator(); | |||
639 | MachineFunction::iterator E = MBB.getParent()->end(); | |||
640 | if (++I == E) | |||
641 | return nullptr; | |||
642 | return &*I; | |||
643 | } | |||
644 | ||||
645 | /// Returns true if the 'true' block (along with its predecessor) forms a valid | |||
646 | /// simple shape for ifcvt. It also returns the number of instructions that the | |||
647 | /// ifcvt would need to duplicate if performed in Dups. | |||
648 | bool IfConverter::ValidSimple(BBInfo &TrueBBI, unsigned &Dups, | |||
649 | BranchProbability Prediction) const { | |||
650 | Dups = 0; | |||
651 | if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone) | |||
652 | return false; | |||
653 | ||||
654 | if (TrueBBI.IsBrAnalyzable) | |||
655 | return false; | |||
656 | ||||
657 | if (TrueBBI.BB->pred_size() > 1) { | |||
658 | if (TrueBBI.CannotBeCopied || | |||
659 | !TII->isProfitableToDupForIfCvt(*TrueBBI.BB, TrueBBI.NonPredSize, | |||
660 | Prediction)) | |||
661 | return false; | |||
662 | Dups = TrueBBI.NonPredSize; | |||
663 | } | |||
664 | ||||
665 | return true; | |||
666 | } | |||
667 | ||||
668 | /// Returns true if the 'true' and 'false' blocks (along with their common | |||
669 | /// predecessor) forms a valid triangle shape for ifcvt. If 'FalseBranch' is | |||
670 | /// true, it checks if 'true' block's false branch branches to the 'false' block | |||
671 | /// rather than the other way around. It also returns the number of instructions | |||
672 | /// that the ifcvt would need to duplicate if performed in 'Dups'. | |||
673 | bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
674 | bool FalseBranch, unsigned &Dups, | |||
675 | BranchProbability Prediction) const { | |||
676 | Dups = 0; | |||
677 | if (TrueBBI.BB == FalseBBI.BB) | |||
678 | return false; | |||
679 | ||||
680 | if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone) | |||
681 | return false; | |||
682 | ||||
683 | if (TrueBBI.BB->pred_size() > 1) { | |||
684 | if (TrueBBI.CannotBeCopied) | |||
685 | return false; | |||
686 | ||||
687 | unsigned Size = TrueBBI.NonPredSize; | |||
688 | if (TrueBBI.IsBrAnalyzable) { | |||
689 | if (TrueBBI.TrueBB && TrueBBI.BrCond.empty()) | |||
690 | // Ends with an unconditional branch. It will be removed. | |||
691 | --Size; | |||
692 | else { | |||
693 | MachineBasicBlock *FExit = FalseBranch | |||
694 | ? TrueBBI.TrueBB : TrueBBI.FalseBB; | |||
695 | if (FExit) | |||
696 | // Require a conditional branch | |||
697 | ++Size; | |||
698 | } | |||
699 | } | |||
700 | if (!TII->isProfitableToDupForIfCvt(*TrueBBI.BB, Size, Prediction)) | |||
701 | return false; | |||
702 | Dups = Size; | |||
703 | } | |||
704 | ||||
705 | MachineBasicBlock *TExit = FalseBranch ? TrueBBI.FalseBB : TrueBBI.TrueBB; | |||
706 | if (!TExit && blockAlwaysFallThrough(TrueBBI)) { | |||
707 | MachineFunction::iterator I = TrueBBI.BB->getIterator(); | |||
708 | if (++I == TrueBBI.BB->getParent()->end()) | |||
709 | return false; | |||
710 | TExit = &*I; | |||
711 | } | |||
712 | return TExit && TExit == FalseBBI.BB; | |||
713 | } | |||
714 | ||||
715 | /// Count duplicated instructions and move the iterators to show where they | |||
716 | /// are. | |||
717 | /// @param TIB True Iterator Begin | |||
718 | /// @param FIB False Iterator Begin | |||
719 | /// These two iterators initially point to the first instruction of the two | |||
720 | /// blocks, and finally point to the first non-shared instruction. | |||
721 | /// @param TIE True Iterator End | |||
722 | /// @param FIE False Iterator End | |||
723 | /// These two iterators initially point to End() for the two blocks() and | |||
724 | /// finally point to the first shared instruction in the tail. | |||
725 | /// Upon return [TIB, TIE), and [FIB, FIE) mark the un-duplicated portions of | |||
726 | /// two blocks. | |||
727 | /// @param Dups1 count of duplicated instructions at the beginning of the 2 | |||
728 | /// blocks. | |||
729 | /// @param Dups2 count of duplicated instructions at the end of the 2 blocks. | |||
730 | /// @param SkipUnconditionalBranches if true, Don't make sure that | |||
731 | /// unconditional branches at the end of the blocks are the same. True is | |||
732 | /// passed when the blocks are analyzable to allow for fallthrough to be | |||
733 | /// handled. | |||
734 | /// @return false if the shared portion prevents if conversion. | |||
735 | bool IfConverter::CountDuplicatedInstructions( | |||
736 | MachineBasicBlock::iterator &TIB, | |||
737 | MachineBasicBlock::iterator &FIB, | |||
738 | MachineBasicBlock::iterator &TIE, | |||
739 | MachineBasicBlock::iterator &FIE, | |||
740 | unsigned &Dups1, unsigned &Dups2, | |||
741 | MachineBasicBlock &TBB, MachineBasicBlock &FBB, | |||
742 | bool SkipUnconditionalBranches) const { | |||
743 | while (TIB != TIE && FIB != FIE) { | |||
744 | // Skip dbg_value instructions. These do not count. | |||
745 | TIB = skipDebugInstructionsForward(TIB, TIE, false); | |||
746 | FIB = skipDebugInstructionsForward(FIB, FIE, false); | |||
747 | if (TIB == TIE || FIB == FIE) | |||
748 | break; | |||
749 | if (!TIB->isIdenticalTo(*FIB)) | |||
750 | break; | |||
751 | // A pred-clobbering instruction in the shared portion prevents | |||
752 | // if-conversion. | |||
753 | std::vector<MachineOperand> PredDefs; | |||
754 | if (TII->ClobbersPredicate(*TIB, PredDefs, false)) | |||
755 | return false; | |||
756 | // If we get all the way to the branch instructions, don't count them. | |||
757 | if (!TIB->isBranch()) | |||
758 | ++Dups1; | |||
| ||||
759 | ++TIB; | |||
760 | ++FIB; | |||
761 | } | |||
762 | ||||
763 | // Check for already containing all of the block. | |||
764 | if (TIB == TIE || FIB == FIE) | |||
765 | return true; | |||
766 | // Now, in preparation for counting duplicate instructions at the ends of the | |||
767 | // blocks, switch to reverse_iterators. Note that getReverse() returns an | |||
768 | // iterator that points to the same instruction, unlike std::reverse_iterator. | |||
769 | // We have to do our own shifting so that we get the same range. | |||
770 | MachineBasicBlock::reverse_iterator RTIE = std::next(TIE.getReverse()); | |||
771 | MachineBasicBlock::reverse_iterator RFIE = std::next(FIE.getReverse()); | |||
772 | const MachineBasicBlock::reverse_iterator RTIB = std::next(TIB.getReverse()); | |||
773 | const MachineBasicBlock::reverse_iterator RFIB = std::next(FIB.getReverse()); | |||
774 | ||||
775 | if (!TBB.succ_empty() || !FBB.succ_empty()) { | |||
776 | if (SkipUnconditionalBranches) { | |||
777 | while (RTIE != RTIB && RTIE->isUnconditionalBranch()) | |||
778 | ++RTIE; | |||
779 | while (RFIE != RFIB && RFIE->isUnconditionalBranch()) | |||
780 | ++RFIE; | |||
781 | } | |||
782 | } | |||
783 | ||||
784 | // Count duplicate instructions at the ends of the blocks. | |||
785 | while (RTIE != RTIB && RFIE != RFIB) { | |||
786 | // Skip dbg_value instructions. These do not count. | |||
787 | // Note that these are reverse iterators going forward. | |||
788 | RTIE = skipDebugInstructionsForward(RTIE, RTIB, false); | |||
789 | RFIE = skipDebugInstructionsForward(RFIE, RFIB, false); | |||
790 | if (RTIE == RTIB || RFIE == RFIB) | |||
791 | break; | |||
792 | if (!RTIE->isIdenticalTo(*RFIE)) | |||
793 | break; | |||
794 | // We have to verify that any branch instructions are the same, and then we | |||
795 | // don't count them toward the # of duplicate instructions. | |||
796 | if (!RTIE->isBranch()) | |||
797 | ++Dups2; | |||
798 | ++RTIE; | |||
799 | ++RFIE; | |||
800 | } | |||
801 | TIE = std::next(RTIE.getReverse()); | |||
802 | FIE = std::next(RFIE.getReverse()); | |||
803 | return true; | |||
804 | } | |||
805 | ||||
806 | /// RescanInstructions - Run ScanInstructions on a pair of blocks. | |||
807 | /// @param TIB - True Iterator Begin, points to first non-shared instruction | |||
808 | /// @param FIB - False Iterator Begin, points to first non-shared instruction | |||
809 | /// @param TIE - True Iterator End, points past last non-shared instruction | |||
810 | /// @param FIE - False Iterator End, points past last non-shared instruction | |||
811 | /// @param TrueBBI - BBInfo to update for the true block. | |||
812 | /// @param FalseBBI - BBInfo to update for the false block. | |||
813 | /// @returns - false if either block cannot be predicated or if both blocks end | |||
814 | /// with a predicate-clobbering instruction. | |||
815 | bool IfConverter::RescanInstructions( | |||
816 | MachineBasicBlock::iterator &TIB, MachineBasicBlock::iterator &FIB, | |||
817 | MachineBasicBlock::iterator &TIE, MachineBasicBlock::iterator &FIE, | |||
818 | BBInfo &TrueBBI, BBInfo &FalseBBI) const { | |||
819 | bool BranchUnpredicable = true; | |||
820 | TrueBBI.IsUnpredicable = FalseBBI.IsUnpredicable = false; | |||
821 | ScanInstructions(TrueBBI, TIB, TIE, BranchUnpredicable); | |||
822 | if (TrueBBI.IsUnpredicable) | |||
823 | return false; | |||
824 | ScanInstructions(FalseBBI, FIB, FIE, BranchUnpredicable); | |||
825 | if (FalseBBI.IsUnpredicable) | |||
826 | return false; | |||
827 | if (TrueBBI.ClobbersPred && FalseBBI.ClobbersPred) | |||
828 | return false; | |||
829 | return true; | |||
830 | } | |||
831 | ||||
832 | #ifndef NDEBUG | |||
833 | static void verifySameBranchInstructions( | |||
834 | MachineBasicBlock *MBB1, | |||
835 | MachineBasicBlock *MBB2) { | |||
836 | const MachineBasicBlock::reverse_iterator B1 = MBB1->rend(); | |||
837 | const MachineBasicBlock::reverse_iterator B2 = MBB2->rend(); | |||
838 | MachineBasicBlock::reverse_iterator E1 = MBB1->rbegin(); | |||
839 | MachineBasicBlock::reverse_iterator E2 = MBB2->rbegin(); | |||
840 | while (E1 != B1 && E2 != B2) { | |||
841 | skipDebugInstructionsForward(E1, B1, false); | |||
842 | skipDebugInstructionsForward(E2, B2, false); | |||
843 | if (E1 == B1 && E2 == B2) | |||
844 | break; | |||
845 | ||||
846 | if (E1 == B1) { | |||
847 | assert(!E2->isBranch() && "Branch mis-match, one block is empty.")(static_cast <bool> (!E2->isBranch() && "Branch mis-match, one block is empty." ) ? void (0) : __assert_fail ("!E2->isBranch() && \"Branch mis-match, one block is empty.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 847, __extension__ __PRETTY_FUNCTION__)); | |||
848 | break; | |||
849 | } | |||
850 | if (E2 == B2) { | |||
851 | assert(!E1->isBranch() && "Branch mis-match, one block is empty.")(static_cast <bool> (!E1->isBranch() && "Branch mis-match, one block is empty." ) ? void (0) : __assert_fail ("!E1->isBranch() && \"Branch mis-match, one block is empty.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 851, __extension__ __PRETTY_FUNCTION__)); | |||
852 | break; | |||
853 | } | |||
854 | ||||
855 | if (E1->isBranch() || E2->isBranch()) | |||
856 | assert(E1->isIdenticalTo(*E2) &&(static_cast <bool> (E1->isIdenticalTo(*E2) && "Branch mis-match, branch instructions don't match.") ? void (0) : __assert_fail ("E1->isIdenticalTo(*E2) && \"Branch mis-match, branch instructions don't match.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 857, __extension__ __PRETTY_FUNCTION__)) | |||
857 | "Branch mis-match, branch instructions don't match.")(static_cast <bool> (E1->isIdenticalTo(*E2) && "Branch mis-match, branch instructions don't match.") ? void (0) : __assert_fail ("E1->isIdenticalTo(*E2) && \"Branch mis-match, branch instructions don't match.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 857, __extension__ __PRETTY_FUNCTION__)); | |||
858 | else | |||
859 | break; | |||
860 | ++E1; | |||
861 | ++E2; | |||
862 | } | |||
863 | } | |||
864 | #endif | |||
865 | ||||
866 | /// ValidForkedDiamond - Returns true if the 'true' and 'false' blocks (along | |||
867 | /// with their common predecessor) form a diamond if a common tail block is | |||
868 | /// extracted. | |||
869 | /// While not strictly a diamond, this pattern would form a diamond if | |||
870 | /// tail-merging had merged the shared tails. | |||
871 | /// EBB | |||
872 | /// _/ \_ | |||
873 | /// | | | |||
874 | /// TBB FBB | |||
875 | /// / \ / \ | |||
876 | /// FalseBB TrueBB FalseBB | |||
877 | /// Currently only handles analyzable branches. | |||
878 | /// Specifically excludes actual diamonds to avoid overlap. | |||
879 | bool IfConverter::ValidForkedDiamond( | |||
880 | BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
881 | unsigned &Dups1, unsigned &Dups2, | |||
882 | BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const { | |||
883 | Dups1 = Dups2 = 0; | |||
884 | if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone || | |||
885 | FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone) | |||
886 | return false; | |||
887 | ||||
888 | if (!TrueBBI.IsBrAnalyzable || !FalseBBI.IsBrAnalyzable) | |||
889 | return false; | |||
890 | // Don't IfConvert blocks that can't be folded into their predecessor. | |||
891 | if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) | |||
892 | return false; | |||
893 | ||||
894 | // This function is specifically looking for conditional tails, as | |||
895 | // unconditional tails are already handled by the standard diamond case. | |||
896 | if (TrueBBI.BrCond.size() == 0 || | |||
897 | FalseBBI.BrCond.size() == 0) | |||
898 | return false; | |||
899 | ||||
900 | MachineBasicBlock *TT = TrueBBI.TrueBB; | |||
901 | MachineBasicBlock *TF = TrueBBI.FalseBB; | |||
902 | MachineBasicBlock *FT = FalseBBI.TrueBB; | |||
903 | MachineBasicBlock *FF = FalseBBI.FalseBB; | |||
904 | ||||
905 | if (!TT) | |||
906 | TT = getNextBlock(*TrueBBI.BB); | |||
907 | if (!TF) | |||
908 | TF = getNextBlock(*TrueBBI.BB); | |||
909 | if (!FT) | |||
910 | FT = getNextBlock(*FalseBBI.BB); | |||
911 | if (!FF) | |||
912 | FF = getNextBlock(*FalseBBI.BB); | |||
913 | ||||
914 | if (!TT || !TF) | |||
915 | return false; | |||
916 | ||||
917 | // Check successors. If they don't match, bail. | |||
918 | if (!((TT == FT && TF == FF) || (TF == FT && TT == FF))) | |||
919 | return false; | |||
920 | ||||
921 | bool FalseReversed = false; | |||
922 | if (TF == FT && TT == FF) { | |||
923 | // If the branches are opposing, but we can't reverse, don't do it. | |||
924 | if (!FalseBBI.IsBrReversible) | |||
925 | return false; | |||
926 | FalseReversed = true; | |||
927 | reverseBranchCondition(FalseBBI); | |||
928 | } | |||
929 | auto UnReverseOnExit = make_scope_exit([&]() { | |||
930 | if (FalseReversed) | |||
931 | reverseBranchCondition(FalseBBI); | |||
932 | }); | |||
933 | ||||
934 | // Count duplicate instructions at the beginning of the true and false blocks. | |||
935 | MachineBasicBlock::iterator TIB = TrueBBI.BB->begin(); | |||
936 | MachineBasicBlock::iterator FIB = FalseBBI.BB->begin(); | |||
937 | MachineBasicBlock::iterator TIE = TrueBBI.BB->end(); | |||
938 | MachineBasicBlock::iterator FIE = FalseBBI.BB->end(); | |||
939 | if(!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2, | |||
940 | *TrueBBI.BB, *FalseBBI.BB, | |||
941 | /* SkipUnconditionalBranches */ true)) | |||
942 | return false; | |||
943 | ||||
944 | TrueBBICalc.BB = TrueBBI.BB; | |||
945 | FalseBBICalc.BB = FalseBBI.BB; | |||
946 | TrueBBICalc.IsBrAnalyzable = TrueBBI.IsBrAnalyzable; | |||
947 | FalseBBICalc.IsBrAnalyzable = FalseBBI.IsBrAnalyzable; | |||
948 | if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc)) | |||
949 | return false; | |||
950 | ||||
951 | // The size is used to decide whether to if-convert, and the shared portions | |||
952 | // are subtracted off. Because of the subtraction, we just use the size that | |||
953 | // was calculated by the original ScanInstructions, as it is correct. | |||
954 | TrueBBICalc.NonPredSize = TrueBBI.NonPredSize; | |||
955 | FalseBBICalc.NonPredSize = FalseBBI.NonPredSize; | |||
956 | return true; | |||
957 | } | |||
958 | ||||
959 | /// ValidDiamond - Returns true if the 'true' and 'false' blocks (along | |||
960 | /// with their common predecessor) forms a valid diamond shape for ifcvt. | |||
961 | bool IfConverter::ValidDiamond( | |||
962 | BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
963 | unsigned &Dups1, unsigned &Dups2, | |||
964 | BBInfo &TrueBBICalc, BBInfo &FalseBBICalc) const { | |||
965 | Dups1 = Dups2 = 0; | |||
966 | if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone || | |||
967 | FalseBBI.IsBeingAnalyzed || FalseBBI.IsDone) | |||
968 | return false; | |||
969 | ||||
970 | // If the True and False BBs are equal we're dealing with a degenerate case | |||
971 | // that we don't treat as a diamond. | |||
972 | if (TrueBBI.BB == FalseBBI.BB) | |||
973 | return false; | |||
974 | ||||
975 | MachineBasicBlock *TT = TrueBBI.TrueBB; | |||
976 | MachineBasicBlock *FT = FalseBBI.TrueBB; | |||
977 | ||||
978 | if (!TT && blockAlwaysFallThrough(TrueBBI)) | |||
979 | TT = getNextBlock(*TrueBBI.BB); | |||
980 | if (!FT && blockAlwaysFallThrough(FalseBBI)) | |||
981 | FT = getNextBlock(*FalseBBI.BB); | |||
982 | if (TT != FT) | |||
983 | return false; | |||
984 | if (!TT && (TrueBBI.IsBrAnalyzable || FalseBBI.IsBrAnalyzable)) | |||
985 | return false; | |||
986 | if (TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) | |||
987 | return false; | |||
988 | ||||
989 | // FIXME: Allow true block to have an early exit? | |||
990 | if (TrueBBI.FalseBB || FalseBBI.FalseBB) | |||
991 | return false; | |||
992 | ||||
993 | // Count duplicate instructions at the beginning and end of the true and | |||
994 | // false blocks. | |||
995 | // Skip unconditional branches only if we are considering an analyzable | |||
996 | // diamond. Otherwise the branches must be the same. | |||
997 | bool SkipUnconditionalBranches = | |||
998 | TrueBBI.IsBrAnalyzable && FalseBBI.IsBrAnalyzable; | |||
999 | MachineBasicBlock::iterator TIB = TrueBBI.BB->begin(); | |||
1000 | MachineBasicBlock::iterator FIB = FalseBBI.BB->begin(); | |||
1001 | MachineBasicBlock::iterator TIE = TrueBBI.BB->end(); | |||
1002 | MachineBasicBlock::iterator FIE = FalseBBI.BB->end(); | |||
1003 | if(!CountDuplicatedInstructions(TIB, FIB, TIE, FIE, Dups1, Dups2, | |||
1004 | *TrueBBI.BB, *FalseBBI.BB, | |||
1005 | SkipUnconditionalBranches)) | |||
1006 | return false; | |||
1007 | ||||
1008 | TrueBBICalc.BB = TrueBBI.BB; | |||
1009 | FalseBBICalc.BB = FalseBBI.BB; | |||
1010 | TrueBBICalc.IsBrAnalyzable = TrueBBI.IsBrAnalyzable; | |||
1011 | FalseBBICalc.IsBrAnalyzable = FalseBBI.IsBrAnalyzable; | |||
1012 | if (!RescanInstructions(TIB, FIB, TIE, FIE, TrueBBICalc, FalseBBICalc)) | |||
1013 | return false; | |||
1014 | // The size is used to decide whether to if-convert, and the shared portions | |||
1015 | // are subtracted off. Because of the subtraction, we just use the size that | |||
1016 | // was calculated by the original ScanInstructions, as it is correct. | |||
1017 | TrueBBICalc.NonPredSize = TrueBBI.NonPredSize; | |||
1018 | FalseBBICalc.NonPredSize = FalseBBI.NonPredSize; | |||
1019 | return true; | |||
1020 | } | |||
1021 | ||||
1022 | /// AnalyzeBranches - Look at the branches at the end of a block to determine if | |||
1023 | /// the block is predicable. | |||
1024 | void IfConverter::AnalyzeBranches(BBInfo &BBI) { | |||
1025 | if (BBI.IsDone) | |||
1026 | return; | |||
1027 | ||||
1028 | BBI.TrueBB = BBI.FalseBB = nullptr; | |||
1029 | BBI.BrCond.clear(); | |||
1030 | BBI.IsBrAnalyzable = | |||
1031 | !TII->analyzeBranch(*BBI.BB, BBI.TrueBB, BBI.FalseBB, BBI.BrCond); | |||
1032 | if (!BBI.IsBrAnalyzable) { | |||
1033 | BBI.TrueBB = nullptr; | |||
1034 | BBI.FalseBB = nullptr; | |||
1035 | BBI.BrCond.clear(); | |||
1036 | } | |||
1037 | ||||
1038 | SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end()); | |||
1039 | BBI.IsBrReversible = (RevCond.size() == 0) || | |||
1040 | !TII->reverseBranchCondition(RevCond); | |||
1041 | BBI.HasFallThrough = BBI.IsBrAnalyzable && BBI.FalseBB == nullptr; | |||
1042 | ||||
1043 | if (BBI.BrCond.size()) { | |||
1044 | // No false branch. This BB must end with a conditional branch and a | |||
1045 | // fallthrough. | |||
1046 | if (!BBI.FalseBB) | |||
1047 | BBI.FalseBB = findFalseBlock(BBI.BB, BBI.TrueBB); | |||
1048 | if (!BBI.FalseBB) { | |||
1049 | // Malformed bcc? True and false blocks are the same? | |||
1050 | BBI.IsUnpredicable = true; | |||
1051 | } | |||
1052 | } | |||
1053 | } | |||
1054 | ||||
1055 | /// ScanInstructions - Scan all the instructions in the block to determine if | |||
1056 | /// the block is predicable. In most cases, that means all the instructions | |||
1057 | /// in the block are isPredicable(). Also checks if the block contains any | |||
1058 | /// instruction which can clobber a predicate (e.g. condition code register). | |||
1059 | /// If so, the block is not predicable unless it's the last instruction. | |||
1060 | void IfConverter::ScanInstructions(BBInfo &BBI, | |||
1061 | MachineBasicBlock::iterator &Begin, | |||
1062 | MachineBasicBlock::iterator &End, | |||
1063 | bool BranchUnpredicable) const { | |||
1064 | if (BBI.IsDone || BBI.IsUnpredicable) | |||
1065 | return; | |||
1066 | ||||
1067 | bool AlreadyPredicated = !BBI.Predicate.empty(); | |||
1068 | ||||
1069 | BBI.NonPredSize = 0; | |||
1070 | BBI.ExtraCost = 0; | |||
1071 | BBI.ExtraCost2 = 0; | |||
1072 | BBI.ClobbersPred = false; | |||
1073 | for (MachineInstr &MI : make_range(Begin, End)) { | |||
1074 | if (MI.isDebugInstr()) | |||
1075 | continue; | |||
1076 | ||||
1077 | // It's unsafe to duplicate convergent instructions in this context, so set | |||
1078 | // BBI.CannotBeCopied to true if MI is convergent. To see why, consider the | |||
1079 | // following CFG, which is subject to our "simple" transformation. | |||
1080 | // | |||
1081 | // BB0 // if (c1) goto BB1; else goto BB2; | |||
1082 | // / \ | |||
1083 | // BB1 | | |||
1084 | // | BB2 // if (c2) goto TBB; else goto FBB; | |||
1085 | // | / | | |||
1086 | // | / | | |||
1087 | // TBB | | |||
1088 | // | | | |||
1089 | // | FBB | |||
1090 | // | | |||
1091 | // exit | |||
1092 | // | |||
1093 | // Suppose we want to move TBB's contents up into BB1 and BB2 (in BB1 they'd | |||
1094 | // be unconditional, and in BB2, they'd be predicated upon c2), and suppose | |||
1095 | // TBB contains a convergent instruction. This is safe iff doing so does | |||
1096 | // not add a control-flow dependency to the convergent instruction -- i.e., | |||
1097 | // it's safe iff the set of control flows that leads us to the convergent | |||
1098 | // instruction does not get smaller after the transformation. | |||
1099 | // | |||
1100 | // Originally we executed TBB if c1 || c2. After the transformation, there | |||
1101 | // are two copies of TBB's instructions. We get to the first if c1, and we | |||
1102 | // get to the second if !c1 && c2. | |||
1103 | // | |||
1104 | // There are clearly fewer ways to satisfy the condition "c1" than | |||
1105 | // "c1 || c2". Since we've shrunk the set of control flows which lead to | |||
1106 | // our convergent instruction, the transformation is unsafe. | |||
1107 | if (MI.isNotDuplicable() || MI.isConvergent()) | |||
1108 | BBI.CannotBeCopied = true; | |||
1109 | ||||
1110 | bool isPredicated = TII->isPredicated(MI); | |||
1111 | bool isCondBr = BBI.IsBrAnalyzable && MI.isConditionalBranch(); | |||
1112 | ||||
1113 | if (BranchUnpredicable && MI.isBranch()) { | |||
1114 | BBI.IsUnpredicable = true; | |||
1115 | return; | |||
1116 | } | |||
1117 | ||||
1118 | // A conditional branch is not predicable, but it may be eliminated. | |||
1119 | if (isCondBr) | |||
1120 | continue; | |||
1121 | ||||
1122 | if (!isPredicated) { | |||
1123 | BBI.NonPredSize++; | |||
1124 | unsigned ExtraPredCost = TII->getPredicationCost(MI); | |||
1125 | unsigned NumCycles = SchedModel.computeInstrLatency(&MI, false); | |||
1126 | if (NumCycles > 1) | |||
1127 | BBI.ExtraCost += NumCycles-1; | |||
1128 | BBI.ExtraCost2 += ExtraPredCost; | |||
1129 | } else if (!AlreadyPredicated) { | |||
1130 | // FIXME: This instruction is already predicated before the | |||
1131 | // if-conversion pass. It's probably something like a conditional move. | |||
1132 | // Mark this block unpredicable for now. | |||
1133 | BBI.IsUnpredicable = true; | |||
1134 | return; | |||
1135 | } | |||
1136 | ||||
1137 | if (BBI.ClobbersPred && !isPredicated) { | |||
1138 | // Predicate modification instruction should end the block (except for | |||
1139 | // already predicated instructions and end of block branches). | |||
1140 | // Predicate may have been modified, the subsequent (currently) | |||
1141 | // unpredicated instructions cannot be correctly predicated. | |||
1142 | BBI.IsUnpredicable = true; | |||
1143 | return; | |||
1144 | } | |||
1145 | ||||
1146 | // FIXME: Make use of PredDefs? e.g. ADDC, SUBC sets predicates but are | |||
1147 | // still potentially predicable. | |||
1148 | std::vector<MachineOperand> PredDefs; | |||
1149 | if (TII->ClobbersPredicate(MI, PredDefs, true)) | |||
1150 | BBI.ClobbersPred = true; | |||
1151 | ||||
1152 | if (!TII->isPredicable(MI)) { | |||
1153 | BBI.IsUnpredicable = true; | |||
1154 | return; | |||
1155 | } | |||
1156 | } | |||
1157 | } | |||
1158 | ||||
1159 | /// Determine if the block is a suitable candidate to be predicated by the | |||
1160 | /// specified predicate. | |||
1161 | /// @param BBI BBInfo for the block to check | |||
1162 | /// @param Pred Predicate array for the branch that leads to BBI | |||
1163 | /// @param isTriangle true if the Analysis is for a triangle | |||
1164 | /// @param RevBranch true if Reverse(Pred) leads to BBI (e.g. BBI is the false | |||
1165 | /// case | |||
1166 | /// @param hasCommonTail true if BBI shares a tail with a sibling block that | |||
1167 | /// contains any instruction that would make the block unpredicable. | |||
1168 | bool IfConverter::FeasibilityAnalysis(BBInfo &BBI, | |||
1169 | SmallVectorImpl<MachineOperand> &Pred, | |||
1170 | bool isTriangle, bool RevBranch, | |||
1171 | bool hasCommonTail) { | |||
1172 | // If the block is dead or unpredicable, then it cannot be predicated. | |||
1173 | // Two blocks may share a common unpredicable tail, but this doesn't prevent | |||
1174 | // them from being if-converted. The non-shared portion is assumed to have | |||
1175 | // been checked | |||
1176 | if (BBI.IsDone || (BBI.IsUnpredicable && !hasCommonTail)) | |||
1177 | return false; | |||
1178 | ||||
1179 | // If it is already predicated but we couldn't analyze its terminator, the | |||
1180 | // latter might fallthrough, but we can't determine where to. | |||
1181 | // Conservatively avoid if-converting again. | |||
1182 | if (BBI.Predicate.size() && !BBI.IsBrAnalyzable) | |||
1183 | return false; | |||
1184 | ||||
1185 | // If it is already predicated, check if the new predicate subsumes | |||
1186 | // its predicate. | |||
1187 | if (BBI.Predicate.size() && !TII->SubsumesPredicate(Pred, BBI.Predicate)) | |||
1188 | return false; | |||
1189 | ||||
1190 | if (!hasCommonTail && BBI.BrCond.size()) { | |||
1191 | if (!isTriangle) | |||
1192 | return false; | |||
1193 | ||||
1194 | // Test predicate subsumption. | |||
1195 | SmallVector<MachineOperand, 4> RevPred(Pred.begin(), Pred.end()); | |||
1196 | SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end()); | |||
1197 | if (RevBranch) { | |||
1198 | if (TII->reverseBranchCondition(Cond)) | |||
1199 | return false; | |||
1200 | } | |||
1201 | if (TII->reverseBranchCondition(RevPred) || | |||
1202 | !TII->SubsumesPredicate(Cond, RevPred)) | |||
1203 | return false; | |||
1204 | } | |||
1205 | ||||
1206 | return true; | |||
1207 | } | |||
1208 | ||||
1209 | /// Analyze the structure of the sub-CFG starting from the specified block. | |||
1210 | /// Record its successors and whether it looks like an if-conversion candidate. | |||
1211 | void IfConverter::AnalyzeBlock( | |||
1212 | MachineBasicBlock &MBB, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) { | |||
1213 | struct BBState { | |||
1214 | BBState(MachineBasicBlock &MBB) : MBB(&MBB), SuccsAnalyzed(false) {} | |||
1215 | MachineBasicBlock *MBB; | |||
1216 | ||||
1217 | /// This flag is true if MBB's successors have been analyzed. | |||
1218 | bool SuccsAnalyzed; | |||
1219 | }; | |||
1220 | ||||
1221 | // Push MBB to the stack. | |||
1222 | SmallVector<BBState, 16> BBStack(1, MBB); | |||
1223 | ||||
1224 | while (!BBStack.empty()) { | |||
1225 | BBState &State = BBStack.back(); | |||
1226 | MachineBasicBlock *BB = State.MBB; | |||
1227 | BBInfo &BBI = BBAnalysis[BB->getNumber()]; | |||
1228 | ||||
1229 | if (!State.SuccsAnalyzed) { | |||
1230 | if (BBI.IsAnalyzed || BBI.IsBeingAnalyzed) { | |||
1231 | BBStack.pop_back(); | |||
1232 | continue; | |||
1233 | } | |||
1234 | ||||
1235 | BBI.BB = BB; | |||
1236 | BBI.IsBeingAnalyzed = true; | |||
1237 | ||||
1238 | AnalyzeBranches(BBI); | |||
1239 | MachineBasicBlock::iterator Begin = BBI.BB->begin(); | |||
1240 | MachineBasicBlock::iterator End = BBI.BB->end(); | |||
1241 | ScanInstructions(BBI, Begin, End); | |||
1242 | ||||
1243 | // Unanalyzable or ends with fallthrough or unconditional branch, or if is | |||
1244 | // not considered for ifcvt anymore. | |||
1245 | if (!BBI.IsBrAnalyzable || BBI.BrCond.empty() || BBI.IsDone) { | |||
1246 | BBI.IsBeingAnalyzed = false; | |||
1247 | BBI.IsAnalyzed = true; | |||
1248 | BBStack.pop_back(); | |||
1249 | continue; | |||
1250 | } | |||
1251 | ||||
1252 | // Do not ifcvt if either path is a back edge to the entry block. | |||
1253 | if (BBI.TrueBB == BB || BBI.FalseBB == BB) { | |||
1254 | BBI.IsBeingAnalyzed = false; | |||
1255 | BBI.IsAnalyzed = true; | |||
1256 | BBStack.pop_back(); | |||
1257 | continue; | |||
1258 | } | |||
1259 | ||||
1260 | // Do not ifcvt if true and false fallthrough blocks are the same. | |||
1261 | if (!BBI.FalseBB) { | |||
1262 | BBI.IsBeingAnalyzed = false; | |||
1263 | BBI.IsAnalyzed = true; | |||
1264 | BBStack.pop_back(); | |||
1265 | continue; | |||
1266 | } | |||
1267 | ||||
1268 | // Push the False and True blocks to the stack. | |||
1269 | State.SuccsAnalyzed = true; | |||
1270 | BBStack.push_back(*BBI.FalseBB); | |||
1271 | BBStack.push_back(*BBI.TrueBB); | |||
1272 | continue; | |||
1273 | } | |||
1274 | ||||
1275 | BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; | |||
1276 | BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; | |||
1277 | ||||
1278 | if (TrueBBI.IsDone && FalseBBI.IsDone) { | |||
1279 | BBI.IsBeingAnalyzed = false; | |||
1280 | BBI.IsAnalyzed = true; | |||
1281 | BBStack.pop_back(); | |||
1282 | continue; | |||
1283 | } | |||
1284 | ||||
1285 | SmallVector<MachineOperand, 4> | |||
1286 | RevCond(BBI.BrCond.begin(), BBI.BrCond.end()); | |||
1287 | bool CanRevCond = !TII->reverseBranchCondition(RevCond); | |||
1288 | ||||
1289 | unsigned Dups = 0; | |||
1290 | unsigned Dups2 = 0; | |||
1291 | bool TNeedSub = !TrueBBI.Predicate.empty(); | |||
1292 | bool FNeedSub = !FalseBBI.Predicate.empty(); | |||
1293 | bool Enqueued = false; | |||
1294 | ||||
1295 | BranchProbability Prediction = MBPI->getEdgeProbability(BB, TrueBBI.BB); | |||
1296 | ||||
1297 | if (CanRevCond) { | |||
1298 | BBInfo TrueBBICalc, FalseBBICalc; | |||
1299 | auto feasibleDiamond = [&](bool Forked) { | |||
1300 | bool MeetsSize = MeetIfcvtSizeLimit(TrueBBICalc, FalseBBICalc, *BB, | |||
1301 | Dups + Dups2, Prediction, Forked); | |||
1302 | bool TrueFeasible = FeasibilityAnalysis(TrueBBI, BBI.BrCond, | |||
1303 | /* IsTriangle */ false, /* RevCond */ false, | |||
1304 | /* hasCommonTail */ true); | |||
1305 | bool FalseFeasible = FeasibilityAnalysis(FalseBBI, RevCond, | |||
1306 | /* IsTriangle */ false, /* RevCond */ false, | |||
1307 | /* hasCommonTail */ true); | |||
1308 | return MeetsSize && TrueFeasible && FalseFeasible; | |||
1309 | }; | |||
1310 | ||||
1311 | if (ValidDiamond(TrueBBI, FalseBBI, Dups, Dups2, | |||
1312 | TrueBBICalc, FalseBBICalc)) { | |||
1313 | if (feasibleDiamond(false)) { | |||
1314 | // Diamond: | |||
1315 | // EBB | |||
1316 | // / \_ | |||
1317 | // | | | |||
1318 | // TBB FBB | |||
1319 | // \ / | |||
1320 | // TailBB | |||
1321 | // Note TailBB can be empty. | |||
1322 | Tokens.push_back(std::make_unique<IfcvtToken>( | |||
1323 | BBI, ICDiamond, TNeedSub | FNeedSub, Dups, Dups2, | |||
1324 | (bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred)); | |||
1325 | Enqueued = true; | |||
1326 | } | |||
1327 | } else if (ValidForkedDiamond(TrueBBI, FalseBBI, Dups, Dups2, | |||
1328 | TrueBBICalc, FalseBBICalc)) { | |||
1329 | if (feasibleDiamond(true)) { | |||
1330 | // ForkedDiamond: | |||
1331 | // if TBB and FBB have a common tail that includes their conditional | |||
1332 | // branch instructions, then we can If Convert this pattern. | |||
1333 | // EBB | |||
1334 | // _/ \_ | |||
1335 | // | | | |||
1336 | // TBB FBB | |||
1337 | // / \ / \ | |||
1338 | // FalseBB TrueBB FalseBB | |||
1339 | // | |||
1340 | Tokens.push_back(std::make_unique<IfcvtToken>( | |||
1341 | BBI, ICForkedDiamond, TNeedSub | FNeedSub, Dups, Dups2, | |||
1342 | (bool) TrueBBICalc.ClobbersPred, (bool) FalseBBICalc.ClobbersPred)); | |||
1343 | Enqueued = true; | |||
1344 | } | |||
1345 | } | |||
1346 | } | |||
1347 | ||||
1348 | if (ValidTriangle(TrueBBI, FalseBBI, false, Dups, Prediction) && | |||
1349 | MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost, | |||
1350 | TrueBBI.ExtraCost2, Prediction) && | |||
1351 | FeasibilityAnalysis(TrueBBI, BBI.BrCond, true)) { | |||
1352 | // Triangle: | |||
1353 | // EBB | |||
1354 | // | \_ | |||
1355 | // | | | |||
1356 | // | TBB | |||
1357 | // | / | |||
1358 | // FBB | |||
1359 | Tokens.push_back( | |||
1360 | std::make_unique<IfcvtToken>(BBI, ICTriangle, TNeedSub, Dups)); | |||
1361 | Enqueued = true; | |||
1362 | } | |||
1363 | ||||
1364 | if (ValidTriangle(TrueBBI, FalseBBI, true, Dups, Prediction) && | |||
1365 | MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost, | |||
1366 | TrueBBI.ExtraCost2, Prediction) && | |||
1367 | FeasibilityAnalysis(TrueBBI, BBI.BrCond, true, true)) { | |||
1368 | Tokens.push_back( | |||
1369 | std::make_unique<IfcvtToken>(BBI, ICTriangleRev, TNeedSub, Dups)); | |||
1370 | Enqueued = true; | |||
1371 | } | |||
1372 | ||||
1373 | if (ValidSimple(TrueBBI, Dups, Prediction) && | |||
1374 | MeetIfcvtSizeLimit(*TrueBBI.BB, TrueBBI.NonPredSize + TrueBBI.ExtraCost, | |||
1375 | TrueBBI.ExtraCost2, Prediction) && | |||
1376 | FeasibilityAnalysis(TrueBBI, BBI.BrCond)) { | |||
1377 | // Simple (split, no rejoin): | |||
1378 | // EBB | |||
1379 | // | \_ | |||
1380 | // | | | |||
1381 | // | TBB---> exit | |||
1382 | // | | |||
1383 | // FBB | |||
1384 | Tokens.push_back( | |||
1385 | std::make_unique<IfcvtToken>(BBI, ICSimple, TNeedSub, Dups)); | |||
1386 | Enqueued = true; | |||
1387 | } | |||
1388 | ||||
1389 | if (CanRevCond) { | |||
1390 | // Try the other path... | |||
1391 | if (ValidTriangle(FalseBBI, TrueBBI, false, Dups, | |||
1392 | Prediction.getCompl()) && | |||
1393 | MeetIfcvtSizeLimit(*FalseBBI.BB, | |||
1394 | FalseBBI.NonPredSize + FalseBBI.ExtraCost, | |||
1395 | FalseBBI.ExtraCost2, Prediction.getCompl()) && | |||
1396 | FeasibilityAnalysis(FalseBBI, RevCond, true)) { | |||
1397 | Tokens.push_back(std::make_unique<IfcvtToken>(BBI, ICTriangleFalse, | |||
1398 | FNeedSub, Dups)); | |||
1399 | Enqueued = true; | |||
1400 | } | |||
1401 | ||||
1402 | if (ValidTriangle(FalseBBI, TrueBBI, true, Dups, | |||
1403 | Prediction.getCompl()) && | |||
1404 | MeetIfcvtSizeLimit(*FalseBBI.BB, | |||
1405 | FalseBBI.NonPredSize + FalseBBI.ExtraCost, | |||
1406 | FalseBBI.ExtraCost2, Prediction.getCompl()) && | |||
1407 | FeasibilityAnalysis(FalseBBI, RevCond, true, true)) { | |||
1408 | Tokens.push_back( | |||
1409 | std::make_unique<IfcvtToken>(BBI, ICTriangleFRev, FNeedSub, Dups)); | |||
1410 | Enqueued = true; | |||
1411 | } | |||
1412 | ||||
1413 | if (ValidSimple(FalseBBI, Dups, Prediction.getCompl()) && | |||
1414 | MeetIfcvtSizeLimit(*FalseBBI.BB, | |||
1415 | FalseBBI.NonPredSize + FalseBBI.ExtraCost, | |||
1416 | FalseBBI.ExtraCost2, Prediction.getCompl()) && | |||
1417 | FeasibilityAnalysis(FalseBBI, RevCond)) { | |||
1418 | Tokens.push_back( | |||
1419 | std::make_unique<IfcvtToken>(BBI, ICSimpleFalse, FNeedSub, Dups)); | |||
1420 | Enqueued = true; | |||
1421 | } | |||
1422 | } | |||
1423 | ||||
1424 | BBI.IsEnqueued = Enqueued; | |||
1425 | BBI.IsBeingAnalyzed = false; | |||
1426 | BBI.IsAnalyzed = true; | |||
1427 | BBStack.pop_back(); | |||
1428 | } | |||
1429 | } | |||
1430 | ||||
1431 | /// Analyze all blocks and find entries for all if-conversion candidates. | |||
1432 | void IfConverter::AnalyzeBlocks( | |||
1433 | MachineFunction &MF, std::vector<std::unique_ptr<IfcvtToken>> &Tokens) { | |||
1434 | for (MachineBasicBlock &MBB : MF) | |||
1435 | AnalyzeBlock(MBB, Tokens); | |||
1436 | ||||
1437 | // Sort to favor more complex ifcvt scheme. | |||
1438 | llvm::stable_sort(Tokens, IfcvtTokenCmp); | |||
1439 | } | |||
1440 | ||||
1441 | /// Returns true either if ToMBB is the next block after MBB or that all the | |||
1442 | /// intervening blocks are empty (given MBB can fall through to its next block). | |||
1443 | static bool canFallThroughTo(MachineBasicBlock &MBB, MachineBasicBlock &ToMBB) { | |||
1444 | MachineFunction::iterator PI = MBB.getIterator(); | |||
1445 | MachineFunction::iterator I = std::next(PI); | |||
1446 | MachineFunction::iterator TI = ToMBB.getIterator(); | |||
1447 | MachineFunction::iterator E = MBB.getParent()->end(); | |||
1448 | while (I != TI) { | |||
1449 | // Check isSuccessor to avoid case where the next block is empty, but | |||
1450 | // it's not a successor. | |||
1451 | if (I == E || !I->empty() || !PI->isSuccessor(&*I)) | |||
1452 | return false; | |||
1453 | PI = I++; | |||
1454 | } | |||
1455 | // Finally see if the last I is indeed a successor to PI. | |||
1456 | return PI->isSuccessor(&*I); | |||
1457 | } | |||
1458 | ||||
1459 | /// Invalidate predecessor BB info so it would be re-analyzed to determine if it | |||
1460 | /// can be if-converted. If predecessor is already enqueued, dequeue it! | |||
1461 | void IfConverter::InvalidatePreds(MachineBasicBlock &MBB) { | |||
1462 | for (const MachineBasicBlock *Predecessor : MBB.predecessors()) { | |||
1463 | BBInfo &PBBI = BBAnalysis[Predecessor->getNumber()]; | |||
1464 | if (PBBI.IsDone || PBBI.BB == &MBB) | |||
1465 | continue; | |||
1466 | PBBI.IsAnalyzed = false; | |||
1467 | PBBI.IsEnqueued = false; | |||
1468 | } | |||
1469 | } | |||
1470 | ||||
1471 | /// Inserts an unconditional branch from \p MBB to \p ToMBB. | |||
1472 | static void InsertUncondBranch(MachineBasicBlock &MBB, MachineBasicBlock &ToMBB, | |||
1473 | const TargetInstrInfo *TII) { | |||
1474 | DebugLoc dl; // FIXME: this is nowhere | |||
1475 | SmallVector<MachineOperand, 0> NoCond; | |||
1476 | TII->insertBranch(MBB, &ToMBB, nullptr, NoCond, dl); | |||
1477 | } | |||
1478 | ||||
1479 | /// Behaves like LiveRegUnits::StepForward() but also adds implicit uses to all | |||
1480 | /// values defined in MI which are also live/used by MI. | |||
1481 | static void UpdatePredRedefs(MachineInstr &MI, LivePhysRegs &Redefs) { | |||
1482 | const TargetRegisterInfo *TRI = MI.getMF()->getSubtarget().getRegisterInfo(); | |||
1483 | ||||
1484 | // Before stepping forward past MI, remember which regs were live | |||
1485 | // before MI. This is needed to set the Undef flag only when reg is | |||
1486 | // dead. | |||
1487 | SparseSet<MCPhysReg, identity<MCPhysReg>> LiveBeforeMI; | |||
1488 | LiveBeforeMI.setUniverse(TRI->getNumRegs()); | |||
1489 | for (unsigned Reg : Redefs) | |||
1490 | LiveBeforeMI.insert(Reg); | |||
1491 | ||||
1492 | SmallVector<std::pair<MCPhysReg, const MachineOperand*>, 4> Clobbers; | |||
1493 | Redefs.stepForward(MI, Clobbers); | |||
1494 | ||||
1495 | // Now add the implicit uses for each of the clobbered values. | |||
1496 | for (auto Clobber : Clobbers) { | |||
1497 | // FIXME: Const cast here is nasty, but better than making StepForward | |||
1498 | // take a mutable instruction instead of const. | |||
1499 | unsigned Reg = Clobber.first; | |||
1500 | MachineOperand &Op = const_cast<MachineOperand&>(*Clobber.second); | |||
1501 | MachineInstr *OpMI = Op.getParent(); | |||
1502 | MachineInstrBuilder MIB(*OpMI->getMF(), OpMI); | |||
1503 | if (Op.isRegMask()) { | |||
1504 | // First handle regmasks. They clobber any entries in the mask which | |||
1505 | // means that we need a def for those registers. | |||
1506 | if (LiveBeforeMI.count(Reg)) | |||
1507 | MIB.addReg(Reg, RegState::Implicit); | |||
1508 | ||||
1509 | // We also need to add an implicit def of this register for the later | |||
1510 | // use to read from. | |||
1511 | // For the register allocator to have allocated a register clobbered | |||
1512 | // by the call which is used later, it must be the case that | |||
1513 | // the call doesn't return. | |||
1514 | MIB.addReg(Reg, RegState::Implicit | RegState::Define); | |||
1515 | continue; | |||
1516 | } | |||
1517 | if (LiveBeforeMI.count(Reg)) | |||
1518 | MIB.addReg(Reg, RegState::Implicit); | |||
1519 | else { | |||
1520 | bool HasLiveSubReg = false; | |||
1521 | for (MCSubRegIterator S(Reg, TRI); S.isValid(); ++S) { | |||
1522 | if (!LiveBeforeMI.count(*S)) | |||
1523 | continue; | |||
1524 | HasLiveSubReg = true; | |||
1525 | break; | |||
1526 | } | |||
1527 | if (HasLiveSubReg) | |||
1528 | MIB.addReg(Reg, RegState::Implicit); | |||
1529 | } | |||
1530 | } | |||
1531 | } | |||
1532 | ||||
1533 | /// If convert a simple (split, no rejoin) sub-CFG. | |||
1534 | bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind Kind) { | |||
1535 | BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; | |||
1536 | BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; | |||
1537 | BBInfo *CvtBBI = &TrueBBI; | |||
1538 | BBInfo *NextBBI = &FalseBBI; | |||
1539 | ||||
1540 | SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end()); | |||
1541 | if (Kind == ICSimpleFalse) | |||
1542 | std::swap(CvtBBI, NextBBI); | |||
1543 | ||||
1544 | MachineBasicBlock &CvtMBB = *CvtBBI->BB; | |||
1545 | MachineBasicBlock &NextMBB = *NextBBI->BB; | |||
1546 | if (CvtBBI->IsDone || | |||
1547 | (CvtBBI->CannotBeCopied && CvtMBB.pred_size() > 1)) { | |||
1548 | // Something has changed. It's no longer safe to predicate this block. | |||
1549 | BBI.IsAnalyzed = false; | |||
1550 | CvtBBI->IsAnalyzed = false; | |||
1551 | return false; | |||
1552 | } | |||
1553 | ||||
1554 | if (CvtMBB.hasAddressTaken()) | |||
1555 | // Conservatively abort if-conversion if BB's address is taken. | |||
1556 | return false; | |||
1557 | ||||
1558 | if (Kind == ICSimpleFalse) | |||
1559 | if (TII->reverseBranchCondition(Cond)) | |||
1560 | llvm_unreachable("Unable to reverse branch condition!")::llvm::llvm_unreachable_internal("Unable to reverse branch condition!" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1560); | |||
1561 | ||||
1562 | Redefs.init(*TRI); | |||
1563 | ||||
1564 | if (MRI->tracksLiveness()) { | |||
1565 | // Initialize liveins to the first BB. These are potentially redefined by | |||
1566 | // predicated instructions. | |||
1567 | Redefs.addLiveIns(CvtMBB); | |||
1568 | Redefs.addLiveIns(NextMBB); | |||
1569 | } | |||
1570 | ||||
1571 | // Remove the branches from the entry so we can add the contents of the true | |||
1572 | // block to it. | |||
1573 | BBI.NonPredSize -= TII->removeBranch(*BBI.BB); | |||
1574 | ||||
1575 | if (CvtMBB.pred_size() > 1) { | |||
1576 | // Copy instructions in the true block, predicate them, and add them to | |||
1577 | // the entry block. | |||
1578 | CopyAndPredicateBlock(BBI, *CvtBBI, Cond); | |||
1579 | ||||
1580 | // Keep the CFG updated. | |||
1581 | BBI.BB->removeSuccessor(&CvtMBB, true); | |||
1582 | } else { | |||
1583 | // Predicate the instructions in the true block. | |||
1584 | PredicateBlock(*CvtBBI, CvtMBB.end(), Cond); | |||
1585 | ||||
1586 | // Merge converted block into entry block. The BB to Cvt edge is removed | |||
1587 | // by MergeBlocks. | |||
1588 | MergeBlocks(BBI, *CvtBBI); | |||
1589 | } | |||
1590 | ||||
1591 | bool IterIfcvt = true; | |||
1592 | if (!canFallThroughTo(*BBI.BB, NextMBB)) { | |||
1593 | InsertUncondBranch(*BBI.BB, NextMBB, TII); | |||
1594 | BBI.HasFallThrough = false; | |||
1595 | // Now ifcvt'd block will look like this: | |||
1596 | // BB: | |||
1597 | // ... | |||
1598 | // t, f = cmp | |||
1599 | // if t op | |||
1600 | // b BBf | |||
1601 | // | |||
1602 | // We cannot further ifcvt this block because the unconditional branch | |||
1603 | // will have to be predicated on the new condition, that will not be | |||
1604 | // available if cmp executes. | |||
1605 | IterIfcvt = false; | |||
1606 | } | |||
1607 | ||||
1608 | // Update block info. BB can be iteratively if-converted. | |||
1609 | if (!IterIfcvt) | |||
1610 | BBI.IsDone = true; | |||
1611 | InvalidatePreds(*BBI.BB); | |||
1612 | CvtBBI->IsDone = true; | |||
1613 | ||||
1614 | // FIXME: Must maintain LiveIns. | |||
1615 | return true; | |||
1616 | } | |||
1617 | ||||
1618 | /// If convert a triangle sub-CFG. | |||
1619 | bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) { | |||
1620 | BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; | |||
1621 | BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; | |||
1622 | BBInfo *CvtBBI = &TrueBBI; | |||
1623 | BBInfo *NextBBI = &FalseBBI; | |||
1624 | DebugLoc dl; // FIXME: this is nowhere | |||
1625 | ||||
1626 | SmallVector<MachineOperand, 4> Cond(BBI.BrCond.begin(), BBI.BrCond.end()); | |||
1627 | if (Kind == ICTriangleFalse || Kind == ICTriangleFRev) | |||
1628 | std::swap(CvtBBI, NextBBI); | |||
1629 | ||||
1630 | MachineBasicBlock &CvtMBB = *CvtBBI->BB; | |||
1631 | MachineBasicBlock &NextMBB = *NextBBI->BB; | |||
1632 | if (CvtBBI->IsDone || | |||
1633 | (CvtBBI->CannotBeCopied && CvtMBB.pred_size() > 1)) { | |||
1634 | // Something has changed. It's no longer safe to predicate this block. | |||
1635 | BBI.IsAnalyzed = false; | |||
1636 | CvtBBI->IsAnalyzed = false; | |||
1637 | return false; | |||
1638 | } | |||
1639 | ||||
1640 | if (CvtMBB.hasAddressTaken()) | |||
1641 | // Conservatively abort if-conversion if BB's address is taken. | |||
1642 | return false; | |||
1643 | ||||
1644 | if (Kind == ICTriangleFalse || Kind == ICTriangleFRev) | |||
1645 | if (TII->reverseBranchCondition(Cond)) | |||
1646 | llvm_unreachable("Unable to reverse branch condition!")::llvm::llvm_unreachable_internal("Unable to reverse branch condition!" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1646); | |||
1647 | ||||
1648 | if (Kind == ICTriangleRev || Kind == ICTriangleFRev) { | |||
1649 | if (reverseBranchCondition(*CvtBBI)) { | |||
1650 | // BB has been changed, modify its predecessors (except for this | |||
1651 | // one) so they don't get ifcvt'ed based on bad intel. | |||
1652 | for (MachineBasicBlock *PBB : CvtMBB.predecessors()) { | |||
1653 | if (PBB == BBI.BB) | |||
1654 | continue; | |||
1655 | BBInfo &PBBI = BBAnalysis[PBB->getNumber()]; | |||
1656 | if (PBBI.IsEnqueued) { | |||
1657 | PBBI.IsAnalyzed = false; | |||
1658 | PBBI.IsEnqueued = false; | |||
1659 | } | |||
1660 | } | |||
1661 | } | |||
1662 | } | |||
1663 | ||||
1664 | // Initialize liveins to the first BB. These are potentially redefined by | |||
1665 | // predicated instructions. | |||
1666 | Redefs.init(*TRI); | |||
1667 | if (MRI->tracksLiveness()) { | |||
1668 | Redefs.addLiveIns(CvtMBB); | |||
1669 | Redefs.addLiveIns(NextMBB); | |||
1670 | } | |||
1671 | ||||
1672 | bool HasEarlyExit = CvtBBI->FalseBB != nullptr; | |||
1673 | BranchProbability CvtNext, CvtFalse, BBNext, BBCvt; | |||
1674 | ||||
1675 | if (HasEarlyExit) { | |||
1676 | // Get probabilities before modifying CvtMBB and BBI.BB. | |||
1677 | CvtNext = MBPI->getEdgeProbability(&CvtMBB, &NextMBB); | |||
1678 | CvtFalse = MBPI->getEdgeProbability(&CvtMBB, CvtBBI->FalseBB); | |||
1679 | BBNext = MBPI->getEdgeProbability(BBI.BB, &NextMBB); | |||
1680 | BBCvt = MBPI->getEdgeProbability(BBI.BB, &CvtMBB); | |||
1681 | } | |||
1682 | ||||
1683 | // Remove the branches from the entry so we can add the contents of the true | |||
1684 | // block to it. | |||
1685 | BBI.NonPredSize -= TII->removeBranch(*BBI.BB); | |||
1686 | ||||
1687 | if (CvtMBB.pred_size() > 1) { | |||
1688 | // Copy instructions in the true block, predicate them, and add them to | |||
1689 | // the entry block. | |||
1690 | CopyAndPredicateBlock(BBI, *CvtBBI, Cond, true); | |||
1691 | } else { | |||
1692 | // Predicate the 'true' block after removing its branch. | |||
1693 | CvtBBI->NonPredSize -= TII->removeBranch(CvtMBB); | |||
1694 | PredicateBlock(*CvtBBI, CvtMBB.end(), Cond); | |||
1695 | ||||
1696 | // Now merge the entry of the triangle with the true block. | |||
1697 | MergeBlocks(BBI, *CvtBBI, false); | |||
1698 | } | |||
1699 | ||||
1700 | // Keep the CFG updated. | |||
1701 | BBI.BB->removeSuccessor(&CvtMBB, true); | |||
1702 | ||||
1703 | // If 'true' block has a 'false' successor, add an exit branch to it. | |||
1704 | if (HasEarlyExit) { | |||
1705 | SmallVector<MachineOperand, 4> RevCond(CvtBBI->BrCond.begin(), | |||
1706 | CvtBBI->BrCond.end()); | |||
1707 | if (TII->reverseBranchCondition(RevCond)) | |||
1708 | llvm_unreachable("Unable to reverse branch condition!")::llvm::llvm_unreachable_internal("Unable to reverse branch condition!" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1708); | |||
1709 | ||||
1710 | // Update the edge probability for both CvtBBI->FalseBB and NextBBI. | |||
1711 | // NewNext = New_Prob(BBI.BB, NextMBB) = | |||
1712 | // Prob(BBI.BB, NextMBB) + | |||
1713 | // Prob(BBI.BB, CvtMBB) * Prob(CvtMBB, NextMBB) | |||
1714 | // NewFalse = New_Prob(BBI.BB, CvtBBI->FalseBB) = | |||
1715 | // Prob(BBI.BB, CvtMBB) * Prob(CvtMBB, CvtBBI->FalseBB) | |||
1716 | auto NewTrueBB = getNextBlock(*BBI.BB); | |||
1717 | auto NewNext = BBNext + BBCvt * CvtNext; | |||
1718 | auto NewTrueBBIter = find(BBI.BB->successors(), NewTrueBB); | |||
1719 | if (NewTrueBBIter != BBI.BB->succ_end()) | |||
1720 | BBI.BB->setSuccProbability(NewTrueBBIter, NewNext); | |||
1721 | ||||
1722 | auto NewFalse = BBCvt * CvtFalse; | |||
1723 | TII->insertBranch(*BBI.BB, CvtBBI->FalseBB, nullptr, RevCond, dl); | |||
1724 | BBI.BB->addSuccessor(CvtBBI->FalseBB, NewFalse); | |||
1725 | } | |||
1726 | ||||
1727 | // Merge in the 'false' block if the 'false' block has no other | |||
1728 | // predecessors. Otherwise, add an unconditional branch to 'false'. | |||
1729 | bool FalseBBDead = false; | |||
1730 | bool IterIfcvt = true; | |||
1731 | bool isFallThrough = canFallThroughTo(*BBI.BB, NextMBB); | |||
1732 | if (!isFallThrough) { | |||
1733 | // Only merge them if the true block does not fallthrough to the false | |||
1734 | // block. By not merging them, we make it possible to iteratively | |||
1735 | // ifcvt the blocks. | |||
1736 | if (!HasEarlyExit && | |||
1737 | NextMBB.pred_size() == 1 && !NextBBI->HasFallThrough && | |||
1738 | !NextMBB.hasAddressTaken()) { | |||
1739 | MergeBlocks(BBI, *NextBBI); | |||
1740 | FalseBBDead = true; | |||
1741 | } else { | |||
1742 | InsertUncondBranch(*BBI.BB, NextMBB, TII); | |||
1743 | BBI.HasFallThrough = false; | |||
1744 | } | |||
1745 | // Mixed predicated and unpredicated code. This cannot be iteratively | |||
1746 | // predicated. | |||
1747 | IterIfcvt = false; | |||
1748 | } | |||
1749 | ||||
1750 | // Update block info. BB can be iteratively if-converted. | |||
1751 | if (!IterIfcvt) | |||
1752 | BBI.IsDone = true; | |||
1753 | InvalidatePreds(*BBI.BB); | |||
1754 | CvtBBI->IsDone = true; | |||
1755 | if (FalseBBDead) | |||
1756 | NextBBI->IsDone = true; | |||
1757 | ||||
1758 | // FIXME: Must maintain LiveIns. | |||
1759 | return true; | |||
1760 | } | |||
1761 | ||||
1762 | /// Common code shared between diamond conversions. | |||
1763 | /// \p BBI, \p TrueBBI, and \p FalseBBI form the diamond shape. | |||
1764 | /// \p NumDups1 - number of shared instructions at the beginning of \p TrueBBI | |||
1765 | /// and FalseBBI | |||
1766 | /// \p NumDups2 - number of shared instructions at the end of \p TrueBBI | |||
1767 | /// and \p FalseBBI | |||
1768 | /// \p RemoveBranch - Remove the common branch of the two blocks before | |||
1769 | /// predicating. Only false for unanalyzable fallthrough | |||
1770 | /// cases. The caller will replace the branch if necessary. | |||
1771 | /// \p MergeAddEdges - Add successor edges when merging blocks. Only false for | |||
1772 | /// unanalyzable fallthrough | |||
1773 | bool IfConverter::IfConvertDiamondCommon( | |||
1774 | BBInfo &BBI, BBInfo &TrueBBI, BBInfo &FalseBBI, | |||
1775 | unsigned NumDups1, unsigned NumDups2, | |||
1776 | bool TClobbersPred, bool FClobbersPred, | |||
1777 | bool RemoveBranch, bool MergeAddEdges) { | |||
1778 | ||||
1779 | if (TrueBBI.IsDone || FalseBBI.IsDone || | |||
1780 | TrueBBI.BB->pred_size() > 1 || FalseBBI.BB->pred_size() > 1) { | |||
1781 | // Something has changed. It's no longer safe to predicate these blocks. | |||
1782 | BBI.IsAnalyzed = false; | |||
1783 | TrueBBI.IsAnalyzed = false; | |||
1784 | FalseBBI.IsAnalyzed = false; | |||
1785 | return false; | |||
1786 | } | |||
1787 | ||||
1788 | if (TrueBBI.BB->hasAddressTaken() || FalseBBI.BB->hasAddressTaken()) | |||
1789 | // Conservatively abort if-conversion if either BB has its address taken. | |||
1790 | return false; | |||
1791 | ||||
1792 | // Put the predicated instructions from the 'true' block before the | |||
1793 | // instructions from the 'false' block, unless the true block would clobber | |||
1794 | // the predicate, in which case, do the opposite. | |||
1795 | BBInfo *BBI1 = &TrueBBI; | |||
1796 | BBInfo *BBI2 = &FalseBBI; | |||
1797 | SmallVector<MachineOperand, 4> RevCond(BBI.BrCond.begin(), BBI.BrCond.end()); | |||
1798 | if (TII->reverseBranchCondition(RevCond)) | |||
1799 | llvm_unreachable("Unable to reverse branch condition!")::llvm::llvm_unreachable_internal("Unable to reverse branch condition!" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1799); | |||
1800 | SmallVector<MachineOperand, 4> *Cond1 = &BBI.BrCond; | |||
1801 | SmallVector<MachineOperand, 4> *Cond2 = &RevCond; | |||
1802 | ||||
1803 | // Figure out the more profitable ordering. | |||
1804 | bool DoSwap = false; | |||
1805 | if (TClobbersPred && !FClobbersPred) | |||
1806 | DoSwap = true; | |||
1807 | else if (!TClobbersPred && !FClobbersPred) { | |||
1808 | if (TrueBBI.NonPredSize > FalseBBI.NonPredSize) | |||
1809 | DoSwap = true; | |||
1810 | } else if (TClobbersPred && FClobbersPred) | |||
1811 | llvm_unreachable("Predicate info cannot be clobbered by both sides.")::llvm::llvm_unreachable_internal("Predicate info cannot be clobbered by both sides." , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1811); | |||
1812 | if (DoSwap) { | |||
1813 | std::swap(BBI1, BBI2); | |||
1814 | std::swap(Cond1, Cond2); | |||
1815 | } | |||
1816 | ||||
1817 | // Remove the conditional branch from entry to the blocks. | |||
1818 | BBI.NonPredSize -= TII->removeBranch(*BBI.BB); | |||
1819 | ||||
1820 | MachineBasicBlock &MBB1 = *BBI1->BB; | |||
1821 | MachineBasicBlock &MBB2 = *BBI2->BB; | |||
1822 | ||||
1823 | // Initialize the Redefs: | |||
1824 | // - BB2 live-in regs need implicit uses before being redefined by BB1 | |||
1825 | // instructions. | |||
1826 | // - BB1 live-out regs need implicit uses before being redefined by BB2 | |||
1827 | // instructions. We start with BB1 live-ins so we have the live-out regs | |||
1828 | // after tracking the BB1 instructions. | |||
1829 | Redefs.init(*TRI); | |||
1830 | if (MRI->tracksLiveness()) { | |||
1831 | Redefs.addLiveIns(MBB1); | |||
1832 | Redefs.addLiveIns(MBB2); | |||
1833 | } | |||
1834 | ||||
1835 | // Remove the duplicated instructions at the beginnings of both paths. | |||
1836 | // Skip dbg_value instructions. | |||
1837 | MachineBasicBlock::iterator DI1 = MBB1.getFirstNonDebugInstr(false); | |||
1838 | MachineBasicBlock::iterator DI2 = MBB2.getFirstNonDebugInstr(false); | |||
1839 | BBI1->NonPredSize -= NumDups1; | |||
1840 | BBI2->NonPredSize -= NumDups1; | |||
1841 | ||||
1842 | // Skip past the dups on each side separately since there may be | |||
1843 | // differing dbg_value entries. NumDups1 can include a "return" | |||
1844 | // instruction, if it's not marked as "branch". | |||
1845 | for (unsigned i = 0; i < NumDups1; ++DI1) { | |||
1846 | if (DI1 == MBB1.end()) | |||
1847 | break; | |||
1848 | if (!DI1->isDebugInstr()) | |||
1849 | ++i; | |||
1850 | } | |||
1851 | while (NumDups1 != 0) { | |||
1852 | // Since this instruction is going to be deleted, update call | |||
1853 | // site info state if the instruction is call instruction. | |||
1854 | if (DI2->shouldUpdateCallSiteInfo()) | |||
1855 | MBB2.getParent()->eraseCallSiteInfo(&*DI2); | |||
1856 | ||||
1857 | ++DI2; | |||
1858 | if (DI2 == MBB2.end()) | |||
1859 | break; | |||
1860 | if (!DI2->isDebugInstr()) | |||
1861 | --NumDups1; | |||
1862 | } | |||
1863 | ||||
1864 | if (MRI->tracksLiveness()) { | |||
1865 | for (const MachineInstr &MI : make_range(MBB1.begin(), DI1)) { | |||
1866 | SmallVector<std::pair<MCPhysReg, const MachineOperand*>, 4> Dummy; | |||
1867 | Redefs.stepForward(MI, Dummy); | |||
1868 | } | |||
1869 | } | |||
1870 | ||||
1871 | BBI.BB->splice(BBI.BB->end(), &MBB1, MBB1.begin(), DI1); | |||
1872 | MBB2.erase(MBB2.begin(), DI2); | |||
1873 | ||||
1874 | // The branches have been checked to match, so it is safe to remove the | |||
1875 | // branch in BB1 and rely on the copy in BB2. The complication is that | |||
1876 | // the blocks may end with a return instruction, which may or may not | |||
1877 | // be marked as "branch". If it's not, then it could be included in | |||
1878 | // "dups1", leaving the blocks potentially empty after moving the common | |||
1879 | // duplicates. | |||
1880 | #ifndef NDEBUG | |||
1881 | // Unanalyzable branches must match exactly. Check that now. | |||
1882 | if (!BBI1->IsBrAnalyzable) | |||
1883 | verifySameBranchInstructions(&MBB1, &MBB2); | |||
1884 | #endif | |||
1885 | // Remove duplicated instructions from the tail of MBB1: any branch | |||
1886 | // instructions, and the common instructions counted by NumDups2. | |||
1887 | DI1 = MBB1.end(); | |||
1888 | while (DI1 != MBB1.begin()) { | |||
1889 | MachineBasicBlock::iterator Prev = std::prev(DI1); | |||
1890 | if (!Prev->isBranch() && !Prev->isDebugInstr()) | |||
1891 | break; | |||
1892 | DI1 = Prev; | |||
1893 | } | |||
1894 | for (unsigned i = 0; i != NumDups2; ) { | |||
1895 | // NumDups2 only counted non-dbg_value instructions, so this won't | |||
1896 | // run off the head of the list. | |||
1897 | assert(DI1 != MBB1.begin())(static_cast <bool> (DI1 != MBB1.begin()) ? void (0) : __assert_fail ("DI1 != MBB1.begin()", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1897, __extension__ __PRETTY_FUNCTION__)); | |||
1898 | ||||
1899 | --DI1; | |||
1900 | ||||
1901 | // Since this instruction is going to be deleted, update call | |||
1902 | // site info state if the instruction is call instruction. | |||
1903 | if (DI1->shouldUpdateCallSiteInfo()) | |||
1904 | MBB1.getParent()->eraseCallSiteInfo(&*DI1); | |||
1905 | ||||
1906 | // skip dbg_value instructions | |||
1907 | if (!DI1->isDebugInstr()) | |||
1908 | ++i; | |||
1909 | } | |||
1910 | MBB1.erase(DI1, MBB1.end()); | |||
1911 | ||||
1912 | DI2 = BBI2->BB->end(); | |||
1913 | // The branches have been checked to match. Skip over the branch in the false | |||
1914 | // block so that we don't try to predicate it. | |||
1915 | if (RemoveBranch) | |||
1916 | BBI2->NonPredSize -= TII->removeBranch(*BBI2->BB); | |||
1917 | else { | |||
1918 | // Make DI2 point to the end of the range where the common "tail" | |||
1919 | // instructions could be found. | |||
1920 | while (DI2 != MBB2.begin()) { | |||
1921 | MachineBasicBlock::iterator Prev = std::prev(DI2); | |||
1922 | if (!Prev->isBranch() && !Prev->isDebugInstr()) | |||
1923 | break; | |||
1924 | DI2 = Prev; | |||
1925 | } | |||
1926 | } | |||
1927 | while (NumDups2 != 0) { | |||
1928 | // NumDups2 only counted non-dbg_value instructions, so this won't | |||
1929 | // run off the head of the list. | |||
1930 | assert(DI2 != MBB2.begin())(static_cast <bool> (DI2 != MBB2.begin()) ? void (0) : __assert_fail ("DI2 != MBB2.begin()", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 1930, __extension__ __PRETTY_FUNCTION__)); | |||
1931 | --DI2; | |||
1932 | // skip dbg_value instructions | |||
1933 | if (!DI2->isDebugInstr()) | |||
1934 | --NumDups2; | |||
1935 | } | |||
1936 | ||||
1937 | // Remember which registers would later be defined by the false block. | |||
1938 | // This allows us not to predicate instructions in the true block that would | |||
1939 | // later be re-defined. That is, rather than | |||
1940 | // subeq r0, r1, #1 | |||
1941 | // addne r0, r1, #1 | |||
1942 | // generate: | |||
1943 | // sub r0, r1, #1 | |||
1944 | // addne r0, r1, #1 | |||
1945 | SmallSet<MCPhysReg, 4> RedefsByFalse; | |||
1946 | SmallSet<MCPhysReg, 4> ExtUses; | |||
1947 | if (TII->isProfitableToUnpredicate(MBB1, MBB2)) { | |||
1948 | for (const MachineInstr &FI : make_range(MBB2.begin(), DI2)) { | |||
1949 | if (FI.isDebugInstr()) | |||
1950 | continue; | |||
1951 | SmallVector<MCPhysReg, 4> Defs; | |||
1952 | for (const MachineOperand &MO : FI.operands()) { | |||
1953 | if (!MO.isReg()) | |||
1954 | continue; | |||
1955 | Register Reg = MO.getReg(); | |||
1956 | if (!Reg) | |||
1957 | continue; | |||
1958 | if (MO.isDef()) { | |||
1959 | Defs.push_back(Reg); | |||
1960 | } else if (!RedefsByFalse.count(Reg)) { | |||
1961 | // These are defined before ctrl flow reach the 'false' instructions. | |||
1962 | // They cannot be modified by the 'true' instructions. | |||
1963 | for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); | |||
1964 | SubRegs.isValid(); ++SubRegs) | |||
1965 | ExtUses.insert(*SubRegs); | |||
1966 | } | |||
1967 | } | |||
1968 | ||||
1969 | for (MCPhysReg Reg : Defs) { | |||
1970 | if (!ExtUses.count(Reg)) { | |||
1971 | for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); | |||
1972 | SubRegs.isValid(); ++SubRegs) | |||
1973 | RedefsByFalse.insert(*SubRegs); | |||
1974 | } | |||
1975 | } | |||
1976 | } | |||
1977 | } | |||
1978 | ||||
1979 | // Predicate the 'true' block. | |||
1980 | PredicateBlock(*BBI1, MBB1.end(), *Cond1, &RedefsByFalse); | |||
1981 | ||||
1982 | // After predicating BBI1, if there is a predicated terminator in BBI1 and | |||
1983 | // a non-predicated in BBI2, then we don't want to predicate the one from | |||
1984 | // BBI2. The reason is that if we merged these blocks, we would end up with | |||
1985 | // two predicated terminators in the same block. | |||
1986 | // Also, if the branches in MBB1 and MBB2 were non-analyzable, then don't | |||
1987 | // predicate them either. They were checked to be identical, and so the | |||
1988 | // same branch would happen regardless of which path was taken. | |||
1989 | if (!MBB2.empty() && (DI2 == MBB2.end())) { | |||
1990 | MachineBasicBlock::iterator BBI1T = MBB1.getFirstTerminator(); | |||
1991 | MachineBasicBlock::iterator BBI2T = MBB2.getFirstTerminator(); | |||
1992 | bool BB1Predicated = BBI1T != MBB1.end() && TII->isPredicated(*BBI1T); | |||
1993 | bool BB2NonPredicated = BBI2T != MBB2.end() && !TII->isPredicated(*BBI2T); | |||
1994 | if (BB2NonPredicated && (BB1Predicated || !BBI2->IsBrAnalyzable)) | |||
1995 | --DI2; | |||
1996 | } | |||
1997 | ||||
1998 | // Predicate the 'false' block. | |||
1999 | PredicateBlock(*BBI2, DI2, *Cond2); | |||
2000 | ||||
2001 | // Merge the true block into the entry of the diamond. | |||
2002 | MergeBlocks(BBI, *BBI1, MergeAddEdges); | |||
2003 | MergeBlocks(BBI, *BBI2, MergeAddEdges); | |||
2004 | return true; | |||
2005 | } | |||
2006 | ||||
2007 | /// If convert an almost-diamond sub-CFG where the true | |||
2008 | /// and false blocks share a common tail. | |||
2009 | bool IfConverter::IfConvertForkedDiamond( | |||
2010 | BBInfo &BBI, IfcvtKind Kind, | |||
2011 | unsigned NumDups1, unsigned NumDups2, | |||
2012 | bool TClobbersPred, bool FClobbersPred) { | |||
2013 | BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; | |||
2014 | BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; | |||
2015 | ||||
2016 | // Save the debug location for later. | |||
2017 | DebugLoc dl; | |||
2018 | MachineBasicBlock::iterator TIE = TrueBBI.BB->getFirstTerminator(); | |||
2019 | if (TIE != TrueBBI.BB->end()) | |||
2020 | dl = TIE->getDebugLoc(); | |||
2021 | // Removing branches from both blocks is safe, because we have already | |||
2022 | // determined that both blocks have the same branch instructions. The branch | |||
2023 | // will be added back at the end, unpredicated. | |||
2024 | if (!IfConvertDiamondCommon( | |||
2025 | BBI, TrueBBI, FalseBBI, | |||
2026 | NumDups1, NumDups2, | |||
2027 | TClobbersPred, FClobbersPred, | |||
2028 | /* RemoveBranch */ true, /* MergeAddEdges */ true)) | |||
2029 | return false; | |||
2030 | ||||
2031 | // Add back the branch. | |||
2032 | // Debug location saved above when removing the branch from BBI2 | |||
2033 | TII->insertBranch(*BBI.BB, TrueBBI.TrueBB, TrueBBI.FalseBB, | |||
2034 | TrueBBI.BrCond, dl); | |||
2035 | ||||
2036 | // Update block info. | |||
2037 | BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true; | |||
2038 | InvalidatePreds(*BBI.BB); | |||
2039 | ||||
2040 | // FIXME: Must maintain LiveIns. | |||
2041 | return true; | |||
2042 | } | |||
2043 | ||||
2044 | /// If convert a diamond sub-CFG. | |||
2045 | bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind, | |||
2046 | unsigned NumDups1, unsigned NumDups2, | |||
2047 | bool TClobbersPred, bool FClobbersPred) { | |||
2048 | BBInfo &TrueBBI = BBAnalysis[BBI.TrueBB->getNumber()]; | |||
2049 | BBInfo &FalseBBI = BBAnalysis[BBI.FalseBB->getNumber()]; | |||
2050 | MachineBasicBlock *TailBB = TrueBBI.TrueBB; | |||
2051 | ||||
2052 | // True block must fall through or end with an unanalyzable terminator. | |||
2053 | if (!TailBB) { | |||
2054 | if (blockAlwaysFallThrough(TrueBBI)) | |||
2055 | TailBB = FalseBBI.TrueBB; | |||
2056 | assert((TailBB || !TrueBBI.IsBrAnalyzable) && "Unexpected!")(static_cast <bool> ((TailBB || !TrueBBI.IsBrAnalyzable ) && "Unexpected!") ? void (0) : __assert_fail ("(TailBB || !TrueBBI.IsBrAnalyzable) && \"Unexpected!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 2056, __extension__ __PRETTY_FUNCTION__)); | |||
2057 | } | |||
2058 | ||||
2059 | if (!IfConvertDiamondCommon( | |||
2060 | BBI, TrueBBI, FalseBBI, | |||
2061 | NumDups1, NumDups2, | |||
2062 | TClobbersPred, FClobbersPred, | |||
2063 | /* RemoveBranch */ TrueBBI.IsBrAnalyzable, | |||
2064 | /* MergeAddEdges */ TailBB == nullptr)) | |||
2065 | return false; | |||
2066 | ||||
2067 | // If the if-converted block falls through or unconditionally branches into | |||
2068 | // the tail block, and the tail block does not have other predecessors, then | |||
2069 | // fold the tail block in as well. Otherwise, unless it falls through to the | |||
2070 | // tail, add a unconditional branch to it. | |||
2071 | if (TailBB) { | |||
2072 | // We need to remove the edges to the true and false blocks manually since | |||
2073 | // we didn't let IfConvertDiamondCommon update the CFG. | |||
2074 | BBI.BB->removeSuccessor(TrueBBI.BB); | |||
2075 | BBI.BB->removeSuccessor(FalseBBI.BB, true); | |||
2076 | ||||
2077 | BBInfo &TailBBI = BBAnalysis[TailBB->getNumber()]; | |||
2078 | bool CanMergeTail = !TailBBI.HasFallThrough && | |||
2079 | !TailBBI.BB->hasAddressTaken(); | |||
2080 | // The if-converted block can still have a predicated terminator | |||
2081 | // (e.g. a predicated return). If that is the case, we cannot merge | |||
2082 | // it with the tail block. | |||
2083 | MachineBasicBlock::const_iterator TI = BBI.BB->getFirstTerminator(); | |||
2084 | if (TI != BBI.BB->end() && TII->isPredicated(*TI)) | |||
2085 | CanMergeTail = false; | |||
2086 | // There may still be a fall-through edge from BBI1 or BBI2 to TailBB; | |||
2087 | // check if there are any other predecessors besides those. | |||
2088 | unsigned NumPreds = TailBB->pred_size(); | |||
2089 | if (NumPreds > 1) | |||
2090 | CanMergeTail = false; | |||
2091 | else if (NumPreds == 1 && CanMergeTail) { | |||
2092 | MachineBasicBlock::pred_iterator PI = TailBB->pred_begin(); | |||
2093 | if (*PI != TrueBBI.BB && *PI != FalseBBI.BB) | |||
2094 | CanMergeTail = false; | |||
2095 | } | |||
2096 | if (CanMergeTail) { | |||
2097 | MergeBlocks(BBI, TailBBI); | |||
2098 | TailBBI.IsDone = true; | |||
2099 | } else { | |||
2100 | BBI.BB->addSuccessor(TailBB, BranchProbability::getOne()); | |||
2101 | InsertUncondBranch(*BBI.BB, *TailBB, TII); | |||
2102 | BBI.HasFallThrough = false; | |||
2103 | } | |||
2104 | } | |||
2105 | ||||
2106 | // Update block info. | |||
2107 | BBI.IsDone = TrueBBI.IsDone = FalseBBI.IsDone = true; | |||
2108 | InvalidatePreds(*BBI.BB); | |||
2109 | ||||
2110 | // FIXME: Must maintain LiveIns. | |||
2111 | return true; | |||
2112 | } | |||
2113 | ||||
2114 | static bool MaySpeculate(const MachineInstr &MI, | |||
2115 | SmallSet<MCPhysReg, 4> &LaterRedefs) { | |||
2116 | bool SawStore = true; | |||
2117 | if (!MI.isSafeToMove(nullptr, SawStore)) | |||
2118 | return false; | |||
2119 | ||||
2120 | for (const MachineOperand &MO : MI.operands()) { | |||
2121 | if (!MO.isReg()) | |||
2122 | continue; | |||
2123 | Register Reg = MO.getReg(); | |||
2124 | if (!Reg) | |||
2125 | continue; | |||
2126 | if (MO.isDef() && !LaterRedefs.count(Reg)) | |||
2127 | return false; | |||
2128 | } | |||
2129 | ||||
2130 | return true; | |||
2131 | } | |||
2132 | ||||
2133 | /// Predicate instructions from the start of the block to the specified end with | |||
2134 | /// the specified condition. | |||
2135 | void IfConverter::PredicateBlock(BBInfo &BBI, | |||
2136 | MachineBasicBlock::iterator E, | |||
2137 | SmallVectorImpl<MachineOperand> &Cond, | |||
2138 | SmallSet<MCPhysReg, 4> *LaterRedefs) { | |||
2139 | bool AnyUnpred = false; | |||
2140 | bool MaySpec = LaterRedefs != nullptr; | |||
2141 | for (MachineInstr &I : make_range(BBI.BB->begin(), E)) { | |||
2142 | if (I.isDebugInstr() || TII->isPredicated(I)) | |||
2143 | continue; | |||
2144 | // It may be possible not to predicate an instruction if it's the 'true' | |||
2145 | // side of a diamond and the 'false' side may re-define the instruction's | |||
2146 | // defs. | |||
2147 | if (MaySpec && MaySpeculate(I, *LaterRedefs)) { | |||
2148 | AnyUnpred = true; | |||
2149 | continue; | |||
2150 | } | |||
2151 | // If any instruction is predicated, then every instruction after it must | |||
2152 | // be predicated. | |||
2153 | MaySpec = false; | |||
2154 | if (!TII->PredicateInstruction(I, Cond)) { | |||
2155 | #ifndef NDEBUG | |||
2156 | dbgs() << "Unable to predicate " << I << "!\n"; | |||
2157 | #endif | |||
2158 | llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 2158); | |||
2159 | } | |||
2160 | ||||
2161 | // If the predicated instruction now redefines a register as the result of | |||
2162 | // if-conversion, add an implicit kill. | |||
2163 | UpdatePredRedefs(I, Redefs); | |||
2164 | } | |||
2165 | ||||
2166 | BBI.Predicate.append(Cond.begin(), Cond.end()); | |||
2167 | ||||
2168 | BBI.IsAnalyzed = false; | |||
2169 | BBI.NonPredSize = 0; | |||
2170 | ||||
2171 | ++NumIfConvBBs; | |||
2172 | if (AnyUnpred) | |||
2173 | ++NumUnpred; | |||
2174 | } | |||
2175 | ||||
2176 | /// Copy and predicate instructions from source BB to the destination block. | |||
2177 | /// Skip end of block branches if IgnoreBr is true. | |||
2178 | void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, | |||
2179 | SmallVectorImpl<MachineOperand> &Cond, | |||
2180 | bool IgnoreBr) { | |||
2181 | MachineFunction &MF = *ToBBI.BB->getParent(); | |||
2182 | ||||
2183 | MachineBasicBlock &FromMBB = *FromBBI.BB; | |||
2184 | for (MachineInstr &I : FromMBB) { | |||
2185 | // Do not copy the end of the block branches. | |||
2186 | if (IgnoreBr && I.isBranch()) | |||
2187 | break; | |||
2188 | ||||
2189 | MachineInstr *MI = MF.CloneMachineInstr(&I); | |||
2190 | // Make a copy of the call site info. | |||
2191 | if (I.isCandidateForCallSiteEntry()) | |||
2192 | MF.copyCallSiteInfo(&I, MI); | |||
2193 | ||||
2194 | ToBBI.BB->insert(ToBBI.BB->end(), MI); | |||
2195 | ToBBI.NonPredSize++; | |||
2196 | unsigned ExtraPredCost = TII->getPredicationCost(I); | |||
2197 | unsigned NumCycles = SchedModel.computeInstrLatency(&I, false); | |||
2198 | if (NumCycles > 1) | |||
2199 | ToBBI.ExtraCost += NumCycles-1; | |||
2200 | ToBBI.ExtraCost2 += ExtraPredCost; | |||
2201 | ||||
2202 | if (!TII->isPredicated(I) && !MI->isDebugInstr()) { | |||
2203 | if (!TII->PredicateInstruction(*MI, Cond)) { | |||
2204 | #ifndef NDEBUG | |||
2205 | dbgs() << "Unable to predicate " << I << "!\n"; | |||
2206 | #endif | |||
2207 | llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 2207); | |||
2208 | } | |||
2209 | } | |||
2210 | ||||
2211 | // If the predicated instruction now redefines a register as the result of | |||
2212 | // if-conversion, add an implicit kill. | |||
2213 | UpdatePredRedefs(*MI, Redefs); | |||
2214 | } | |||
2215 | ||||
2216 | if (!IgnoreBr) { | |||
2217 | std::vector<MachineBasicBlock *> Succs(FromMBB.succ_begin(), | |||
2218 | FromMBB.succ_end()); | |||
2219 | MachineBasicBlock *NBB = getNextBlock(FromMBB); | |||
2220 | MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr; | |||
2221 | ||||
2222 | for (MachineBasicBlock *Succ : Succs) { | |||
2223 | // Fallthrough edge can't be transferred. | |||
2224 | if (Succ == FallThrough) | |||
2225 | continue; | |||
2226 | ToBBI.BB->addSuccessor(Succ); | |||
2227 | } | |||
2228 | } | |||
2229 | ||||
2230 | ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end()); | |||
2231 | ToBBI.Predicate.append(Cond.begin(), Cond.end()); | |||
2232 | ||||
2233 | ToBBI.ClobbersPred |= FromBBI.ClobbersPred; | |||
2234 | ToBBI.IsAnalyzed = false; | |||
2235 | ||||
2236 | ++NumDupBBs; | |||
2237 | } | |||
2238 | ||||
2239 | /// Move all instructions from FromBB to the end of ToBB. This will leave | |||
2240 | /// FromBB as an empty block, so remove all of its successor edges and move it | |||
2241 | /// to the end of the function. If AddEdges is true, i.e., when FromBBI's | |||
2242 | /// branch is being moved, add those successor edges to ToBBI and remove the old | |||
2243 | /// edge from ToBBI to FromBBI. | |||
2244 | void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) { | |||
2245 | MachineBasicBlock &FromMBB = *FromBBI.BB; | |||
2246 | assert(!FromMBB.hasAddressTaken() &&(static_cast <bool> (!FromMBB.hasAddressTaken() && "Removing a BB whose address is taken!") ? void (0) : __assert_fail ("!FromMBB.hasAddressTaken() && \"Removing a BB whose address is taken!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 2247, __extension__ __PRETTY_FUNCTION__)) | |||
2247 | "Removing a BB whose address is taken!")(static_cast <bool> (!FromMBB.hasAddressTaken() && "Removing a BB whose address is taken!") ? void (0) : __assert_fail ("!FromMBB.hasAddressTaken() && \"Removing a BB whose address is taken!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/lib/CodeGen/IfConversion.cpp" , 2247, __extension__ __PRETTY_FUNCTION__)); | |||
2248 | ||||
2249 | // In case FromMBB contains terminators (e.g. return instruction), | |||
2250 | // first move the non-terminator instructions, then the terminators. | |||
2251 | MachineBasicBlock::iterator FromTI = FromMBB.getFirstTerminator(); | |||
2252 | MachineBasicBlock::iterator ToTI = ToBBI.BB->getFirstTerminator(); | |||
2253 | ToBBI.BB->splice(ToTI, &FromMBB, FromMBB.begin(), FromTI); | |||
2254 | ||||
2255 | // If FromBB has non-predicated terminator we should copy it at the end. | |||
2256 | if (FromTI != FromMBB.end() && !TII->isPredicated(*FromTI)) | |||
2257 | ToTI = ToBBI.BB->end(); | |||
2258 | ToBBI.BB->splice(ToTI, &FromMBB, FromTI, FromMBB.end()); | |||
2259 | ||||
2260 | // Force normalizing the successors' probabilities of ToBBI.BB to convert all | |||
2261 | // unknown probabilities into known ones. | |||
2262 | // FIXME: This usage is too tricky and in the future we would like to | |||
2263 | // eliminate all unknown probabilities in MBB. | |||
2264 | if (ToBBI.IsBrAnalyzable) | |||
2265 | ToBBI.BB->normalizeSuccProbs(); | |||
2266 | ||||
2267 | SmallVector<MachineBasicBlock *, 4> FromSuccs(FromMBB.successors()); | |||
2268 | MachineBasicBlock *NBB = getNextBlock(FromMBB); | |||
2269 | MachineBasicBlock *FallThrough = FromBBI.HasFallThrough ? NBB : nullptr; | |||
2270 | // The edge probability from ToBBI.BB to FromMBB, which is only needed when | |||
2271 | // AddEdges is true and FromMBB is a successor of ToBBI.BB. | |||
2272 | auto To2FromProb = BranchProbability::getZero(); | |||
2273 | if (AddEdges && ToBBI.BB->isSuccessor(&FromMBB)) { | |||
2274 | // Remove the old edge but remember the edge probability so we can calculate | |||
2275 | // the correct weights on the new edges being added further down. | |||
2276 | To2FromProb = MBPI->getEdgeProbability(ToBBI.BB, &FromMBB); | |||
2277 | ToBBI.BB->removeSuccessor(&FromMBB); | |||
2278 | } | |||
2279 | ||||
2280 | for (MachineBasicBlock *Succ : FromSuccs) { | |||
2281 | // Fallthrough edge can't be transferred. | |||
2282 | if (Succ == FallThrough) { | |||
2283 | FromMBB.removeSuccessor(Succ); | |||
2284 | continue; | |||
2285 | } | |||
2286 | ||||
2287 | auto NewProb = BranchProbability::getZero(); | |||
2288 | if (AddEdges) { | |||
2289 | // Calculate the edge probability for the edge from ToBBI.BB to Succ, | |||
2290 | // which is a portion of the edge probability from FromMBB to Succ. The | |||
2291 | // portion ratio is the edge probability from ToBBI.BB to FromMBB (if | |||
2292 | // FromBBI is a successor of ToBBI.BB. See comment below for exception). | |||
2293 | NewProb = MBPI->getEdgeProbability(&FromMBB, Succ); | |||
2294 | ||||
2295 | // To2FromProb is 0 when FromMBB is not a successor of ToBBI.BB. This | |||
2296 | // only happens when if-converting a diamond CFG and FromMBB is the | |||
2297 | // tail BB. In this case FromMBB post-dominates ToBBI.BB and hence we | |||
2298 | // could just use the probabilities on FromMBB's out-edges when adding | |||
2299 | // new successors. | |||
2300 | if (!To2FromProb.isZero()) | |||
2301 | NewProb *= To2FromProb; | |||
2302 | } | |||
2303 | ||||
2304 | FromMBB.removeSuccessor(Succ); | |||
2305 | ||||
2306 | if (AddEdges) { | |||
2307 | // If the edge from ToBBI.BB to Succ already exists, update the | |||
2308 | // probability of this edge by adding NewProb to it. An example is shown | |||
2309 | // below, in which A is ToBBI.BB and B is FromMBB. In this case we | |||
2310 | // don't have to set C as A's successor as it already is. We only need to | |||
2311 | // update the edge probability on A->C. Note that B will not be | |||
2312 | // immediately removed from A's successors. It is possible that B->D is | |||
2313 | // not removed either if D is a fallthrough of B. Later the edge A->D | |||
2314 | // (generated here) and B->D will be combined into one edge. To maintain | |||
2315 | // correct edge probability of this combined edge, we need to set the edge | |||
2316 | // probability of A->B to zero, which is already done above. The edge | |||
2317 | // probability on A->D is calculated by scaling the original probability | |||
2318 | // on A->B by the probability of B->D. | |||
2319 | // | |||
2320 | // Before ifcvt: After ifcvt (assume B->D is kept): | |||
2321 | // | |||
2322 | // A A | |||
2323 | // /| /|\ | |||
2324 | // / B / B| | |||
2325 | // | /| | || | |||
2326 | // |/ | | |/ | |||
2327 | // C D C D | |||
2328 | // | |||
2329 | if (ToBBI.BB->isSuccessor(Succ)) | |||
2330 | ToBBI.BB->setSuccProbability( | |||
2331 | find(ToBBI.BB->successors(), Succ), | |||
2332 | MBPI->getEdgeProbability(ToBBI.BB, Succ) + NewProb); | |||
2333 | else | |||
2334 | ToBBI.BB->addSuccessor(Succ, NewProb); | |||
2335 | } | |||
2336 | } | |||
2337 | ||||
2338 | // Move the now empty FromMBB out of the way to the end of the function so | |||
2339 | // it doesn't interfere with fallthrough checks done by canFallThroughTo(). | |||
2340 | MachineBasicBlock *Last = &*FromMBB.getParent()->rbegin(); | |||
2341 | if (Last != &FromMBB) | |||
2342 | FromMBB.moveAfter(Last); | |||
2343 | ||||
2344 | // Normalize the probabilities of ToBBI.BB's successors with all adjustment | |||
2345 | // we've done above. | |||
2346 | if (ToBBI.IsBrAnalyzable && FromBBI.IsBrAnalyzable) | |||
2347 | ToBBI.BB->normalizeSuccProbs(); | |||
2348 | ||||
2349 | ToBBI.Predicate.append(FromBBI.Predicate.begin(), FromBBI.Predicate.end()); | |||
2350 | FromBBI.Predicate.clear(); | |||
2351 | ||||
2352 | ToBBI.NonPredSize += FromBBI.NonPredSize; | |||
2353 | ToBBI.ExtraCost += FromBBI.ExtraCost; | |||
2354 | ToBBI.ExtraCost2 += FromBBI.ExtraCost2; | |||
2355 | FromBBI.NonPredSize = 0; | |||
2356 | FromBBI.ExtraCost = 0; | |||
2357 | FromBBI.ExtraCost2 = 0; | |||
2358 | ||||
2359 | ToBBI.ClobbersPred |= FromBBI.ClobbersPred; | |||
2360 | ToBBI.HasFallThrough = FromBBI.HasFallThrough; | |||
2361 | ToBBI.IsAnalyzed = false; | |||
2362 | FromBBI.IsAnalyzed = false; | |||
2363 | } | |||
2364 | ||||
2365 | FunctionPass * | |||
2366 | llvm::createIfConverter(std::function<bool(const MachineFunction &)> Ftor) { | |||
2367 | return new IfConverter(std::move(Ftor)); | |||
2368 | } |
1 | //===- llvm/CodeGen/MachineInstrBundleIterator.h ----------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Defines an iterator class that bundles MachineInstr. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #ifndef LLVM_CODEGEN_MACHINEINSTRBUNDLEITERATOR_H |
14 | #define LLVM_CODEGEN_MACHINEINSTRBUNDLEITERATOR_H |
15 | |
16 | #include "llvm/ADT/ilist.h" |
17 | #include "llvm/ADT/simple_ilist.h" |
18 | #include <cassert> |
19 | #include <iterator> |
20 | #include <type_traits> |
21 | |
22 | namespace llvm { |
23 | |
24 | template <class T, bool IsReverse> struct MachineInstrBundleIteratorTraits; |
25 | template <class T> struct MachineInstrBundleIteratorTraits<T, false> { |
26 | using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>; |
27 | using instr_iterator = typename list_type::iterator; |
28 | using nonconst_instr_iterator = typename list_type::iterator; |
29 | using const_instr_iterator = typename list_type::const_iterator; |
30 | }; |
31 | template <class T> struct MachineInstrBundleIteratorTraits<T, true> { |
32 | using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>; |
33 | using instr_iterator = typename list_type::reverse_iterator; |
34 | using nonconst_instr_iterator = typename list_type::reverse_iterator; |
35 | using const_instr_iterator = typename list_type::const_reverse_iterator; |
36 | }; |
37 | template <class T> struct MachineInstrBundleIteratorTraits<const T, false> { |
38 | using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>; |
39 | using instr_iterator = typename list_type::const_iterator; |
40 | using nonconst_instr_iterator = typename list_type::iterator; |
41 | using const_instr_iterator = typename list_type::const_iterator; |
42 | }; |
43 | template <class T> struct MachineInstrBundleIteratorTraits<const T, true> { |
44 | using list_type = simple_ilist<T, ilist_sentinel_tracking<true>>; |
45 | using instr_iterator = typename list_type::const_reverse_iterator; |
46 | using nonconst_instr_iterator = typename list_type::reverse_iterator; |
47 | using const_instr_iterator = typename list_type::const_reverse_iterator; |
48 | }; |
49 | |
50 | template <bool IsReverse> struct MachineInstrBundleIteratorHelper; |
51 | template <> struct MachineInstrBundleIteratorHelper<false> { |
52 | /// Get the beginning of the current bundle. |
53 | template <class Iterator> static Iterator getBundleBegin(Iterator I) { |
54 | if (!I.isEnd()) |
55 | while (I->isBundledWithPred()) |
56 | --I; |
57 | return I; |
58 | } |
59 | |
60 | /// Get the final node of the current bundle. |
61 | template <class Iterator> static Iterator getBundleFinal(Iterator I) { |
62 | if (!I.isEnd()) |
63 | while (I->isBundledWithSucc()) |
64 | ++I; |
65 | return I; |
66 | } |
67 | |
68 | /// Increment forward ilist iterator. |
69 | template <class Iterator> static void increment(Iterator &I) { |
70 | I = std::next(getBundleFinal(I)); |
71 | } |
72 | |
73 | /// Decrement forward ilist iterator. |
74 | template <class Iterator> static void decrement(Iterator &I) { |
75 | I = getBundleBegin(std::prev(I)); |
76 | } |
77 | }; |
78 | |
79 | template <> struct MachineInstrBundleIteratorHelper<true> { |
80 | /// Get the beginning of the current bundle. |
81 | template <class Iterator> static Iterator getBundleBegin(Iterator I) { |
82 | return MachineInstrBundleIteratorHelper<false>::getBundleBegin( |
83 | I.getReverse()) |
84 | .getReverse(); |
85 | } |
86 | |
87 | /// Get the final node of the current bundle. |
88 | template <class Iterator> static Iterator getBundleFinal(Iterator I) { |
89 | return MachineInstrBundleIteratorHelper<false>::getBundleFinal( |
90 | I.getReverse()) |
91 | .getReverse(); |
92 | } |
93 | |
94 | /// Increment reverse ilist iterator. |
95 | template <class Iterator> static void increment(Iterator &I) { |
96 | I = getBundleBegin(std::next(I)); |
97 | } |
98 | |
99 | /// Decrement reverse ilist iterator. |
100 | template <class Iterator> static void decrement(Iterator &I) { |
101 | I = std::prev(getBundleFinal(I)); |
102 | } |
103 | }; |
104 | |
105 | /// MachineBasicBlock iterator that automatically skips over MIs that are |
106 | /// inside bundles (i.e. walk top level MIs only). |
107 | template <typename Ty, bool IsReverse = false> |
108 | class MachineInstrBundleIterator : MachineInstrBundleIteratorHelper<IsReverse> { |
109 | using Traits = MachineInstrBundleIteratorTraits<Ty, IsReverse>; |
110 | using instr_iterator = typename Traits::instr_iterator; |
111 | |
112 | instr_iterator MII; |
113 | |
114 | public: |
115 | using value_type = typename instr_iterator::value_type; |
116 | using difference_type = typename instr_iterator::difference_type; |
117 | using pointer = typename instr_iterator::pointer; |
118 | using reference = typename instr_iterator::reference; |
119 | using const_pointer = typename instr_iterator::const_pointer; |
120 | using const_reference = typename instr_iterator::const_reference; |
121 | using iterator_category = std::bidirectional_iterator_tag; |
122 | |
123 | private: |
124 | using nonconst_instr_iterator = typename Traits::nonconst_instr_iterator; |
125 | using const_instr_iterator = typename Traits::const_instr_iterator; |
126 | using nonconst_iterator = |
127 | MachineInstrBundleIterator<typename nonconst_instr_iterator::value_type, |
128 | IsReverse>; |
129 | using reverse_iterator = MachineInstrBundleIterator<Ty, !IsReverse>; |
130 | |
131 | public: |
132 | MachineInstrBundleIterator(instr_iterator MI) : MII(MI) { |
133 | assert((!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) &&(static_cast <bool> ((!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) && "It's not legal to initialize MachineInstrBundleIterator with a " "bundled MI") ? void (0) : __assert_fail ("(!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) && \"It's not legal to initialize MachineInstrBundleIterator with a \" \"bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 135, __extension__ __PRETTY_FUNCTION__)) |
134 | "It's not legal to initialize MachineInstrBundleIterator with a "(static_cast <bool> ((!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) && "It's not legal to initialize MachineInstrBundleIterator with a " "bundled MI") ? void (0) : __assert_fail ("(!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) && \"It's not legal to initialize MachineInstrBundleIterator with a \" \"bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 135, __extension__ __PRETTY_FUNCTION__)) |
135 | "bundled MI")(static_cast <bool> ((!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) && "It's not legal to initialize MachineInstrBundleIterator with a " "bundled MI") ? void (0) : __assert_fail ("(!MI.getNodePtr() || MI.isEnd() || !MI->isBundledWithPred()) && \"It's not legal to initialize MachineInstrBundleIterator with a \" \"bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 135, __extension__ __PRETTY_FUNCTION__)); |
136 | } |
137 | |
138 | MachineInstrBundleIterator(reference MI) : MII(MI) { |
139 | assert(!MI.isBundledWithPred() && "It's not legal to initialize "(static_cast <bool> (!MI.isBundledWithPred() && "It's not legal to initialize " "MachineInstrBundleIterator with a " "bundled MI") ? void (0) : __assert_fail ("!MI.isBundledWithPred() && \"It's not legal to initialize \" \"MachineInstrBundleIterator with a \" \"bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 141, __extension__ __PRETTY_FUNCTION__)) |
140 | "MachineInstrBundleIterator with a "(static_cast <bool> (!MI.isBundledWithPred() && "It's not legal to initialize " "MachineInstrBundleIterator with a " "bundled MI") ? void (0) : __assert_fail ("!MI.isBundledWithPred() && \"It's not legal to initialize \" \"MachineInstrBundleIterator with a \" \"bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 141, __extension__ __PRETTY_FUNCTION__)) |
141 | "bundled MI")(static_cast <bool> (!MI.isBundledWithPred() && "It's not legal to initialize " "MachineInstrBundleIterator with a " "bundled MI") ? void (0) : __assert_fail ("!MI.isBundledWithPred() && \"It's not legal to initialize \" \"MachineInstrBundleIterator with a \" \"bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 141, __extension__ __PRETTY_FUNCTION__)); |
142 | } |
143 | |
144 | MachineInstrBundleIterator(pointer MI) : MII(MI) { |
145 | // FIXME: This conversion should be explicit. |
146 | assert((!MI || !MI->isBundledWithPred()) && "It's not legal to initialize "(static_cast <bool> ((!MI || !MI->isBundledWithPred( )) && "It's not legal to initialize " "MachineInstrBundleIterator " "with a bundled MI") ? void (0) : __assert_fail ("(!MI || !MI->isBundledWithPred()) && \"It's not legal to initialize \" \"MachineInstrBundleIterator \" \"with a bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 148, __extension__ __PRETTY_FUNCTION__)) |
147 | "MachineInstrBundleIterator "(static_cast <bool> ((!MI || !MI->isBundledWithPred( )) && "It's not legal to initialize " "MachineInstrBundleIterator " "with a bundled MI") ? void (0) : __assert_fail ("(!MI || !MI->isBundledWithPred()) && \"It's not legal to initialize \" \"MachineInstrBundleIterator \" \"with a bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 148, __extension__ __PRETTY_FUNCTION__)) |
148 | "with a bundled MI")(static_cast <bool> ((!MI || !MI->isBundledWithPred( )) && "It's not legal to initialize " "MachineInstrBundleIterator " "with a bundled MI") ? void (0) : __assert_fail ("(!MI || !MI->isBundledWithPred()) && \"It's not legal to initialize \" \"MachineInstrBundleIterator \" \"with a bundled MI\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h" , 148, __extension__ __PRETTY_FUNCTION__)); |
149 | } |
150 | |
151 | // Template allows conversion from const to nonconst. |
152 | template <class OtherTy> |
153 | MachineInstrBundleIterator( |
154 | const MachineInstrBundleIterator<OtherTy, IsReverse> &I, |
155 | std::enable_if_t<std::is_convertible<OtherTy *, Ty *>::value, void *> = |
156 | nullptr) |
157 | : MII(I.getInstrIterator()) {} |
158 | |
159 | MachineInstrBundleIterator() : MII(nullptr) {} |
160 | |
161 | /// Explicit conversion between forward/reverse iterators. |
162 | /// |
163 | /// Translate between forward and reverse iterators without changing range |
164 | /// boundaries. The resulting iterator will dereference (and have a handle) |
165 | /// to the previous node, which is somewhat unexpected; but converting the |
166 | /// two endpoints in a range will give the same range in reverse. |
167 | /// |
168 | /// This matches std::reverse_iterator conversions. |
169 | explicit MachineInstrBundleIterator( |
170 | const MachineInstrBundleIterator<Ty, !IsReverse> &I) |
171 | : MachineInstrBundleIterator(++I.getReverse()) {} |
172 | |
173 | /// Get the bundle iterator for the given instruction's bundle. |
174 | static MachineInstrBundleIterator getAtBundleBegin(instr_iterator MI) { |
175 | return MachineInstrBundleIteratorHelper<IsReverse>::getBundleBegin(MI); |
176 | } |
177 | |
178 | reference operator*() const { return *MII; } |
179 | pointer operator->() const { return &operator*(); } |
180 | |
181 | /// Check for null. |
182 | bool isValid() const { return MII.getNodePtr(); } |
183 | |
184 | friend bool operator==(const MachineInstrBundleIterator &L, |
185 | const MachineInstrBundleIterator &R) { |
186 | return L.MII == R.MII; |
187 | } |
188 | friend bool operator==(const MachineInstrBundleIterator &L, |
189 | const const_instr_iterator &R) { |
190 | return L.MII == R; // Avoid assertion about validity of R. |
191 | } |
192 | friend bool operator==(const const_instr_iterator &L, |
193 | const MachineInstrBundleIterator &R) { |
194 | return L == R.MII; // Avoid assertion about validity of L. |
195 | } |
196 | friend bool operator==(const MachineInstrBundleIterator &L, |
197 | const nonconst_instr_iterator &R) { |
198 | return L.MII == R; // Avoid assertion about validity of R. |
199 | } |
200 | friend bool operator==(const nonconst_instr_iterator &L, |
201 | const MachineInstrBundleIterator &R) { |
202 | return L == R.MII; // Avoid assertion about validity of L. |
203 | } |
204 | friend bool operator==(const MachineInstrBundleIterator &L, const_pointer R) { |
205 | return L == const_instr_iterator(R); // Avoid assertion about validity of R. |
206 | } |
207 | friend bool operator==(const_pointer L, const MachineInstrBundleIterator &R) { |
208 | return const_instr_iterator(L) == R; // Avoid assertion about validity of L. |
209 | } |
210 | friend bool operator==(const MachineInstrBundleIterator &L, |
211 | const_reference R) { |
212 | return L == &R; // Avoid assertion about validity of R. |
213 | } |
214 | friend bool operator==(const_reference L, |
215 | const MachineInstrBundleIterator &R) { |
216 | return &L == R; // Avoid assertion about validity of L. |
217 | } |
218 | |
219 | friend bool operator!=(const MachineInstrBundleIterator &L, |
220 | const MachineInstrBundleIterator &R) { |
221 | return !(L == R); |
222 | } |
223 | friend bool operator!=(const MachineInstrBundleIterator &L, |
224 | const const_instr_iterator &R) { |
225 | return !(L == R); |
226 | } |
227 | friend bool operator!=(const const_instr_iterator &L, |
228 | const MachineInstrBundleIterator &R) { |
229 | return !(L == R); |
230 | } |
231 | friend bool operator!=(const MachineInstrBundleIterator &L, |
232 | const nonconst_instr_iterator &R) { |
233 | return !(L == R); |
234 | } |
235 | friend bool operator!=(const nonconst_instr_iterator &L, |
236 | const MachineInstrBundleIterator &R) { |
237 | return !(L == R); |
238 | } |
239 | friend bool operator!=(const MachineInstrBundleIterator &L, const_pointer R) { |
240 | return !(L == R); |
241 | } |
242 | friend bool operator!=(const_pointer L, const MachineInstrBundleIterator &R) { |
243 | return !(L == R); |
244 | } |
245 | friend bool operator!=(const MachineInstrBundleIterator &L, |
246 | const_reference R) { |
247 | return !(L == R); |
248 | } |
249 | friend bool operator!=(const_reference L, |
250 | const MachineInstrBundleIterator &R) { |
251 | return !(L == R); |
252 | } |
253 | |
254 | // Increment and decrement operators... |
255 | MachineInstrBundleIterator &operator--() { |
256 | this->decrement(MII); |
257 | return *this; |
258 | } |
259 | MachineInstrBundleIterator &operator++() { |
260 | this->increment(MII); |
261 | return *this; |
262 | } |
263 | MachineInstrBundleIterator operator--(int) { |
264 | MachineInstrBundleIterator Temp = *this; |
265 | --*this; |
266 | return Temp; |
267 | } |
268 | MachineInstrBundleIterator operator++(int) { |
269 | MachineInstrBundleIterator Temp = *this; |
270 | ++*this; |
271 | return Temp; |
272 | } |
273 | |
274 | instr_iterator getInstrIterator() const { return MII; } |
275 | |
276 | nonconst_iterator getNonConstIterator() const { return MII.getNonConst(); } |
277 | |
278 | /// Get a reverse iterator to the same node. |
279 | /// |
280 | /// Gives a reverse iterator that will dereference (and have a handle) to the |
281 | /// same node. Converting the endpoint iterators in a range will give a |
282 | /// different range; for range operations, use the explicit conversions. |
283 | reverse_iterator getReverse() const { return MII.getReverse(); } |
284 | }; |
285 | |
286 | } // end namespace llvm |
287 | |
288 | #endif // LLVM_CODEGEN_MACHINEINSTRBUNDLEITERATOR_H |
1 | //===- llvm/ADT/ilist_iterator.h - Intrusive List Iterator ------*- C++ -*-===// | ||||||||||||||||
2 | // | ||||||||||||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||||||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||||||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||||||||||
6 | // | ||||||||||||||||
7 | //===----------------------------------------------------------------------===// | ||||||||||||||||
8 | |||||||||||||||||
9 | #ifndef LLVM_ADT_ILIST_ITERATOR_H | ||||||||||||||||
10 | #define LLVM_ADT_ILIST_ITERATOR_H | ||||||||||||||||
11 | |||||||||||||||||
12 | #include "llvm/ADT/ilist_node.h" | ||||||||||||||||
13 | #include <cassert> | ||||||||||||||||
14 | #include <cstddef> | ||||||||||||||||
15 | #include <iterator> | ||||||||||||||||
16 | #include <type_traits> | ||||||||||||||||
17 | |||||||||||||||||
18 | namespace llvm { | ||||||||||||||||
19 | |||||||||||||||||
20 | namespace ilist_detail { | ||||||||||||||||
21 | |||||||||||||||||
22 | /// Find const-correct node types. | ||||||||||||||||
23 | template <class OptionsT, bool IsConst> struct IteratorTraits; | ||||||||||||||||
24 | template <class OptionsT> struct IteratorTraits<OptionsT, false> { | ||||||||||||||||
25 | using value_type = typename OptionsT::value_type; | ||||||||||||||||
26 | using pointer = typename OptionsT::pointer; | ||||||||||||||||
27 | using reference = typename OptionsT::reference; | ||||||||||||||||
28 | using node_pointer = ilist_node_impl<OptionsT> *; | ||||||||||||||||
29 | using node_reference = ilist_node_impl<OptionsT> &; | ||||||||||||||||
30 | }; | ||||||||||||||||
31 | template <class OptionsT> struct IteratorTraits<OptionsT, true> { | ||||||||||||||||
32 | using value_type = const typename OptionsT::value_type; | ||||||||||||||||
33 | using pointer = typename OptionsT::const_pointer; | ||||||||||||||||
34 | using reference = typename OptionsT::const_reference; | ||||||||||||||||
35 | using node_pointer = const ilist_node_impl<OptionsT> *; | ||||||||||||||||
36 | using node_reference = const ilist_node_impl<OptionsT> &; | ||||||||||||||||
37 | }; | ||||||||||||||||
38 | |||||||||||||||||
39 | template <bool IsReverse> struct IteratorHelper; | ||||||||||||||||
40 | template <> struct IteratorHelper<false> : ilist_detail::NodeAccess { | ||||||||||||||||
41 | using Access = ilist_detail::NodeAccess; | ||||||||||||||||
42 | |||||||||||||||||
43 | template <class T> static void increment(T *&I) { I = Access::getNext(*I); } | ||||||||||||||||
44 | template <class T> static void decrement(T *&I) { I = Access::getPrev(*I); } | ||||||||||||||||
45 | }; | ||||||||||||||||
46 | template <> struct IteratorHelper<true> : ilist_detail::NodeAccess { | ||||||||||||||||
47 | using Access = ilist_detail::NodeAccess; | ||||||||||||||||
48 | |||||||||||||||||
49 | template <class T> static void increment(T *&I) { I = Access::getPrev(*I); } | ||||||||||||||||
50 | template <class T> static void decrement(T *&I) { I = Access::getNext(*I); } | ||||||||||||||||
51 | }; | ||||||||||||||||
52 | |||||||||||||||||
53 | } // end namespace ilist_detail | ||||||||||||||||
54 | |||||||||||||||||
55 | /// Iterator for intrusive lists based on ilist_node. | ||||||||||||||||
56 | template <class OptionsT, bool IsReverse, bool IsConst> | ||||||||||||||||
57 | class ilist_iterator : ilist_detail::SpecificNodeAccess<OptionsT> { | ||||||||||||||||
58 | friend ilist_iterator<OptionsT, IsReverse, !IsConst>; | ||||||||||||||||
59 | friend ilist_iterator<OptionsT, !IsReverse, IsConst>; | ||||||||||||||||
60 | friend ilist_iterator<OptionsT, !IsReverse, !IsConst>; | ||||||||||||||||
61 | |||||||||||||||||
62 | using Traits = ilist_detail::IteratorTraits<OptionsT, IsConst>; | ||||||||||||||||
63 | using Access = ilist_detail::SpecificNodeAccess<OptionsT>; | ||||||||||||||||
64 | |||||||||||||||||
65 | public: | ||||||||||||||||
66 | using value_type = typename Traits::value_type; | ||||||||||||||||
67 | using pointer = typename Traits::pointer; | ||||||||||||||||
68 | using reference = typename Traits::reference; | ||||||||||||||||
69 | using difference_type = ptrdiff_t; | ||||||||||||||||
70 | using iterator_category = std::bidirectional_iterator_tag; | ||||||||||||||||
71 | using const_pointer = typename OptionsT::const_pointer; | ||||||||||||||||
72 | using const_reference = typename OptionsT::const_reference; | ||||||||||||||||
73 | |||||||||||||||||
74 | private: | ||||||||||||||||
75 | using node_pointer = typename Traits::node_pointer; | ||||||||||||||||
76 | using node_reference = typename Traits::node_reference; | ||||||||||||||||
77 | |||||||||||||||||
78 | node_pointer NodePtr = nullptr; | ||||||||||||||||
79 | |||||||||||||||||
80 | public: | ||||||||||||||||
81 | /// Create from an ilist_node. | ||||||||||||||||
82 | explicit ilist_iterator(node_reference N) : NodePtr(&N) {} | ||||||||||||||||
83 | |||||||||||||||||
84 | explicit ilist_iterator(pointer NP) : NodePtr(Access::getNodePtr(NP)) {} | ||||||||||||||||
85 | explicit ilist_iterator(reference NR) : NodePtr(Access::getNodePtr(&NR)) {} | ||||||||||||||||
86 | ilist_iterator() = default; | ||||||||||||||||
87 | |||||||||||||||||
88 | // This is templated so that we can allow constructing a const iterator from | ||||||||||||||||
89 | // a nonconst iterator... | ||||||||||||||||
90 | template <bool RHSIsConst> | ||||||||||||||||
91 | ilist_iterator(const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS, | ||||||||||||||||
92 | std::enable_if_t<IsConst || !RHSIsConst, void *> = nullptr) | ||||||||||||||||
93 | : NodePtr(RHS.NodePtr) {} | ||||||||||||||||
94 | |||||||||||||||||
95 | // This is templated so that we can allow assigning to a const iterator from | ||||||||||||||||
96 | // a nonconst iterator... | ||||||||||||||||
97 | template <bool RHSIsConst> | ||||||||||||||||
98 | std::enable_if_t<IsConst || !RHSIsConst, ilist_iterator &> | ||||||||||||||||
99 | operator=(const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS) { | ||||||||||||||||
100 | NodePtr = RHS.NodePtr; | ||||||||||||||||
101 | return *this; | ||||||||||||||||
102 | } | ||||||||||||||||
103 | |||||||||||||||||
104 | /// Explicit conversion between forward/reverse iterators. | ||||||||||||||||
105 | /// | ||||||||||||||||
106 | /// Translate between forward and reverse iterators without changing range | ||||||||||||||||
107 | /// boundaries. The resulting iterator will dereference (and have a handle) | ||||||||||||||||
108 | /// to the previous node, which is somewhat unexpected; but converting the | ||||||||||||||||
109 | /// two endpoints in a range will give the same range in reverse. | ||||||||||||||||
110 | /// | ||||||||||||||||
111 | /// This matches std::reverse_iterator conversions. | ||||||||||||||||
112 | explicit ilist_iterator( | ||||||||||||||||
113 | const ilist_iterator<OptionsT, !IsReverse, IsConst> &RHS) | ||||||||||||||||
114 | : ilist_iterator(++RHS.getReverse()) {} | ||||||||||||||||
115 | |||||||||||||||||
116 | /// Get a reverse iterator to the same node. | ||||||||||||||||
117 | /// | ||||||||||||||||
118 | /// Gives a reverse iterator that will dereference (and have a handle) to the | ||||||||||||||||
119 | /// same node. Converting the endpoint iterators in a range will give a | ||||||||||||||||
120 | /// different range; for range operations, use the explicit conversions. | ||||||||||||||||
121 | ilist_iterator<OptionsT, !IsReverse, IsConst> getReverse() const { | ||||||||||||||||
122 | if (NodePtr) | ||||||||||||||||
123 | return ilist_iterator<OptionsT, !IsReverse, IsConst>(*NodePtr); | ||||||||||||||||
124 | return ilist_iterator<OptionsT, !IsReverse, IsConst>(); | ||||||||||||||||
125 | } | ||||||||||||||||
126 | |||||||||||||||||
127 | /// Const-cast. | ||||||||||||||||
128 | ilist_iterator<OptionsT, IsReverse, false> getNonConst() const { | ||||||||||||||||
129 | if (NodePtr) | ||||||||||||||||
130 | return ilist_iterator<OptionsT, IsReverse, false>( | ||||||||||||||||
131 | const_cast<typename ilist_iterator<OptionsT, IsReverse, | ||||||||||||||||
132 | false>::node_reference>(*NodePtr)); | ||||||||||||||||
133 | return ilist_iterator<OptionsT, IsReverse, false>(); | ||||||||||||||||
134 | } | ||||||||||||||||
135 | |||||||||||||||||
136 | // Accessors... | ||||||||||||||||
137 | reference operator*() const { | ||||||||||||||||
138 | assert(!NodePtr->isKnownSentinel())(static_cast <bool> (!NodePtr->isKnownSentinel()) ? void (0) : __assert_fail ("!NodePtr->isKnownSentinel()", "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/ADT/ilist_iterator.h" , 138, __extension__ __PRETTY_FUNCTION__)); | ||||||||||||||||
139 | return *Access::getValuePtr(NodePtr); | ||||||||||||||||
140 | } | ||||||||||||||||
141 | pointer operator->() const { return &operator*(); } | ||||||||||||||||
142 | |||||||||||||||||
143 | // Comparison operators | ||||||||||||||||
144 | friend bool operator==(const ilist_iterator &LHS, const ilist_iterator &RHS) { | ||||||||||||||||
145 | return LHS.NodePtr
| ||||||||||||||||
146 | } | ||||||||||||||||
147 | friend bool operator!=(const ilist_iterator &LHS, const ilist_iterator &RHS) { | ||||||||||||||||
148 | return LHS.NodePtr != RHS.NodePtr; | ||||||||||||||||
149 | } | ||||||||||||||||
150 | |||||||||||||||||
151 | // Increment and decrement operators... | ||||||||||||||||
152 | ilist_iterator &operator--() { | ||||||||||||||||
153 | NodePtr = IsReverse ? NodePtr->getNext() : NodePtr->getPrev(); | ||||||||||||||||
154 | return *this; | ||||||||||||||||
155 | } | ||||||||||||||||
156 | ilist_iterator &operator++() { | ||||||||||||||||
157 | NodePtr = IsReverse ? NodePtr->getPrev() : NodePtr->getNext(); | ||||||||||||||||
158 | return *this; | ||||||||||||||||
159 | } | ||||||||||||||||
160 | ilist_iterator operator--(int) { | ||||||||||||||||
161 | ilist_iterator tmp = *this; | ||||||||||||||||
162 | --*this; | ||||||||||||||||
163 | return tmp; | ||||||||||||||||
164 | } | ||||||||||||||||
165 | ilist_iterator operator++(int) { | ||||||||||||||||
166 | ilist_iterator tmp = *this; | ||||||||||||||||
167 | ++*this; | ||||||||||||||||
168 | return tmp; | ||||||||||||||||
169 | } | ||||||||||||||||
170 | |||||||||||||||||
171 | /// Get the underlying ilist_node. | ||||||||||||||||
172 | node_pointer getNodePtr() const { return static_cast<node_pointer>(NodePtr); } | ||||||||||||||||
173 | |||||||||||||||||
174 | /// Check for end. Only valid if ilist_sentinel_tracking<true>. | ||||||||||||||||
175 | bool isEnd() const { return NodePtr ? NodePtr->isSentinel() : false; } | ||||||||||||||||
176 | }; | ||||||||||||||||
177 | |||||||||||||||||
178 | template <typename From> struct simplify_type; | ||||||||||||||||
179 | |||||||||||||||||
180 | /// Allow ilist_iterators to convert into pointers to a node automatically when | ||||||||||||||||
181 | /// used by the dyn_cast, cast, isa mechanisms... | ||||||||||||||||
182 | /// | ||||||||||||||||
183 | /// FIXME: remove this, since there is no implicit conversion to NodeTy. | ||||||||||||||||
184 | template <class OptionsT, bool IsConst> | ||||||||||||||||
185 | struct simplify_type<ilist_iterator<OptionsT, false, IsConst>> { | ||||||||||||||||
186 | using iterator = ilist_iterator<OptionsT, false, IsConst>; | ||||||||||||||||
187 | using SimpleType = typename iterator::pointer; | ||||||||||||||||
188 | |||||||||||||||||
189 | static SimpleType getSimplifiedValue(const iterator &Node) { return &*Node; } | ||||||||||||||||
190 | }; | ||||||||||||||||
191 | template <class OptionsT, bool IsConst> | ||||||||||||||||
192 | struct simplify_type<const ilist_iterator<OptionsT, false, IsConst>> | ||||||||||||||||
193 | : simplify_type<ilist_iterator<OptionsT, false, IsConst>> {}; | ||||||||||||||||
194 | |||||||||||||||||
195 | } // end namespace llvm | ||||||||||||||||
196 | |||||||||||||||||
197 | #endif // LLVM_ADT_ILIST_ITERATOR_H |
1 | //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===// | ||||||||
2 | // | ||||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
6 | // | ||||||||
7 | //===----------------------------------------------------------------------===// | ||||||||
8 | // | ||||||||
9 | // This file contains the declaration of the MachineInstr class, which is the | ||||||||
10 | // basic representation for all target dependent machine instructions used by | ||||||||
11 | // the back end. | ||||||||
12 | // | ||||||||
13 | //===----------------------------------------------------------------------===// | ||||||||
14 | |||||||||
15 | #ifndef LLVM_CODEGEN_MACHINEINSTR_H | ||||||||
16 | #define LLVM_CODEGEN_MACHINEINSTR_H | ||||||||
17 | |||||||||
18 | #include "llvm/ADT/DenseMapInfo.h" | ||||||||
19 | #include "llvm/ADT/PointerSumType.h" | ||||||||
20 | #include "llvm/ADT/SmallSet.h" | ||||||||
21 | #include "llvm/ADT/ilist.h" | ||||||||
22 | #include "llvm/ADT/ilist_node.h" | ||||||||
23 | #include "llvm/ADT/iterator_range.h" | ||||||||
24 | #include "llvm/CodeGen/MachineMemOperand.h" | ||||||||
25 | #include "llvm/CodeGen/MachineOperand.h" | ||||||||
26 | #include "llvm/CodeGen/TargetOpcodes.h" | ||||||||
27 | #include "llvm/IR/DebugLoc.h" | ||||||||
28 | #include "llvm/IR/InlineAsm.h" | ||||||||
29 | #include "llvm/IR/PseudoProbe.h" | ||||||||
30 | #include "llvm/MC/MCInstrDesc.h" | ||||||||
31 | #include "llvm/MC/MCSymbol.h" | ||||||||
32 | #include "llvm/Support/ArrayRecycler.h" | ||||||||
33 | #include "llvm/Support/TrailingObjects.h" | ||||||||
34 | #include <algorithm> | ||||||||
35 | #include <cassert> | ||||||||
36 | #include <cstdint> | ||||||||
37 | #include <utility> | ||||||||
38 | |||||||||
39 | namespace llvm { | ||||||||
40 | |||||||||
41 | class AAResults; | ||||||||
42 | template <typename T> class ArrayRef; | ||||||||
43 | class DIExpression; | ||||||||
44 | class DILocalVariable; | ||||||||
45 | class MachineBasicBlock; | ||||||||
46 | class MachineFunction; | ||||||||
47 | class MachineRegisterInfo; | ||||||||
48 | class ModuleSlotTracker; | ||||||||
49 | class raw_ostream; | ||||||||
50 | template <typename T> class SmallVectorImpl; | ||||||||
51 | class SmallBitVector; | ||||||||
52 | class StringRef; | ||||||||
53 | class TargetInstrInfo; | ||||||||
54 | class TargetRegisterClass; | ||||||||
55 | class TargetRegisterInfo; | ||||||||
56 | |||||||||
57 | //===----------------------------------------------------------------------===// | ||||||||
58 | /// Representation of each machine instruction. | ||||||||
59 | /// | ||||||||
60 | /// This class isn't a POD type, but it must have a trivial destructor. When a | ||||||||
61 | /// MachineFunction is deleted, all the contained MachineInstrs are deallocated | ||||||||
62 | /// without having their destructor called. | ||||||||
63 | /// | ||||||||
64 | class MachineInstr | ||||||||
65 | : public ilist_node_with_parent<MachineInstr, MachineBasicBlock, | ||||||||
66 | ilist_sentinel_tracking<true>> { | ||||||||
67 | public: | ||||||||
68 | using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator; | ||||||||
69 | |||||||||
70 | /// Flags to specify different kinds of comments to output in | ||||||||
71 | /// assembly code. These flags carry semantic information not | ||||||||
72 | /// otherwise easily derivable from the IR text. | ||||||||
73 | /// | ||||||||
74 | enum CommentFlag { | ||||||||
75 | ReloadReuse = 0x1, // higher bits are reserved for target dep comments. | ||||||||
76 | NoSchedComment = 0x2, | ||||||||
77 | TAsmComments = 0x4 // Target Asm comments should start from this value. | ||||||||
78 | }; | ||||||||
79 | |||||||||
80 | enum MIFlag { | ||||||||
81 | NoFlags = 0, | ||||||||
82 | FrameSetup = 1 << 0, // Instruction is used as a part of | ||||||||
83 | // function frame setup code. | ||||||||
84 | FrameDestroy = 1 << 1, // Instruction is used as a part of | ||||||||
85 | // function frame destruction code. | ||||||||
86 | BundledPred = 1 << 2, // Instruction has bundled predecessors. | ||||||||
87 | BundledSucc = 1 << 3, // Instruction has bundled successors. | ||||||||
88 | FmNoNans = 1 << 4, // Instruction does not support Fast | ||||||||
89 | // math nan values. | ||||||||
90 | FmNoInfs = 1 << 5, // Instruction does not support Fast | ||||||||
91 | // math infinity values. | ||||||||
92 | FmNsz = 1 << 6, // Instruction is not required to retain | ||||||||
93 | // signed zero values. | ||||||||
94 | FmArcp = 1 << 7, // Instruction supports Fast math | ||||||||
95 | // reciprocal approximations. | ||||||||
96 | FmContract = 1 << 8, // Instruction supports Fast math | ||||||||
97 | // contraction operations like fma. | ||||||||
98 | FmAfn = 1 << 9, // Instruction may map to Fast math | ||||||||
99 | // instrinsic approximation. | ||||||||
100 | FmReassoc = 1 << 10, // Instruction supports Fast math | ||||||||
101 | // reassociation of operand order. | ||||||||
102 | NoUWrap = 1 << 11, // Instruction supports binary operator | ||||||||
103 | // no unsigned wrap. | ||||||||
104 | NoSWrap = 1 << 12, // Instruction supports binary operator | ||||||||
105 | // no signed wrap. | ||||||||
106 | IsExact = 1 << 13, // Instruction supports division is | ||||||||
107 | // known to be exact. | ||||||||
108 | NoFPExcept = 1 << 14, // Instruction does not raise | ||||||||
109 | // floatint-point exceptions. | ||||||||
110 | NoMerge = 1 << 15, // Passes that drop source location info | ||||||||
111 | // (e.g. branch folding) should skip | ||||||||
112 | // this instruction. | ||||||||
113 | }; | ||||||||
114 | |||||||||
115 | private: | ||||||||
116 | const MCInstrDesc *MCID; // Instruction descriptor. | ||||||||
117 | MachineBasicBlock *Parent = nullptr; // Pointer to the owning basic block. | ||||||||
118 | |||||||||
119 | // Operands are allocated by an ArrayRecycler. | ||||||||
120 | MachineOperand *Operands = nullptr; // Pointer to the first operand. | ||||||||
121 | unsigned NumOperands = 0; // Number of operands on instruction. | ||||||||
122 | |||||||||
123 | uint16_t Flags = 0; // Various bits of additional | ||||||||
124 | // information about machine | ||||||||
125 | // instruction. | ||||||||
126 | |||||||||
127 | uint8_t AsmPrinterFlags = 0; // Various bits of information used by | ||||||||
128 | // the AsmPrinter to emit helpful | ||||||||
129 | // comments. This is *not* semantic | ||||||||
130 | // information. Do not use this for | ||||||||
131 | // anything other than to convey comment | ||||||||
132 | // information to AsmPrinter. | ||||||||
133 | |||||||||
134 | // OperandCapacity has uint8_t size, so it should be next to AsmPrinterFlags | ||||||||
135 | // to properly pack. | ||||||||
136 | using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity; | ||||||||
137 | OperandCapacity CapOperands; // Capacity of the Operands array. | ||||||||
138 | |||||||||
139 | /// Internal implementation detail class that provides out-of-line storage for | ||||||||
140 | /// extra info used by the machine instruction when this info cannot be stored | ||||||||
141 | /// in-line within the instruction itself. | ||||||||
142 | /// | ||||||||
143 | /// This has to be defined eagerly due to the implementation constraints of | ||||||||
144 | /// `PointerSumType` where it is used. | ||||||||
145 | class ExtraInfo final | ||||||||
146 | : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> { | ||||||||
147 | public: | ||||||||
148 | static ExtraInfo *create(BumpPtrAllocator &Allocator, | ||||||||
149 | ArrayRef<MachineMemOperand *> MMOs, | ||||||||
150 | MCSymbol *PreInstrSymbol = nullptr, | ||||||||
151 | MCSymbol *PostInstrSymbol = nullptr, | ||||||||
152 | MDNode *HeapAllocMarker = nullptr) { | ||||||||
153 | bool HasPreInstrSymbol = PreInstrSymbol != nullptr; | ||||||||
154 | bool HasPostInstrSymbol = PostInstrSymbol != nullptr; | ||||||||
155 | bool HasHeapAllocMarker = HeapAllocMarker != nullptr; | ||||||||
156 | auto *Result = new (Allocator.Allocate( | ||||||||
157 | totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>( | ||||||||
158 | MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, | ||||||||
159 | HasHeapAllocMarker), | ||||||||
160 | alignof(ExtraInfo))) | ||||||||
161 | ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, | ||||||||
162 | HasHeapAllocMarker); | ||||||||
163 | |||||||||
164 | // Copy the actual data into the trailing objects. | ||||||||
165 | std::copy(MMOs.begin(), MMOs.end(), | ||||||||
166 | Result->getTrailingObjects<MachineMemOperand *>()); | ||||||||
167 | |||||||||
168 | if (HasPreInstrSymbol) | ||||||||
169 | Result->getTrailingObjects<MCSymbol *>()[0] = PreInstrSymbol; | ||||||||
170 | if (HasPostInstrSymbol) | ||||||||
171 | Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] = | ||||||||
172 | PostInstrSymbol; | ||||||||
173 | if (HasHeapAllocMarker) | ||||||||
174 | Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker; | ||||||||
175 | |||||||||
176 | return Result; | ||||||||
177 | } | ||||||||
178 | |||||||||
179 | ArrayRef<MachineMemOperand *> getMMOs() const { | ||||||||
180 | return makeArrayRef(getTrailingObjects<MachineMemOperand *>(), NumMMOs); | ||||||||
181 | } | ||||||||
182 | |||||||||
183 | MCSymbol *getPreInstrSymbol() const { | ||||||||
184 | return HasPreInstrSymbol ? getTrailingObjects<MCSymbol *>()[0] : nullptr; | ||||||||
185 | } | ||||||||
186 | |||||||||
187 | MCSymbol *getPostInstrSymbol() const { | ||||||||
188 | return HasPostInstrSymbol | ||||||||
189 | ? getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] | ||||||||
190 | : nullptr; | ||||||||
191 | } | ||||||||
192 | |||||||||
193 | MDNode *getHeapAllocMarker() const { | ||||||||
194 | return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr; | ||||||||
195 | } | ||||||||
196 | |||||||||
197 | private: | ||||||||
198 | friend TrailingObjects; | ||||||||
199 | |||||||||
200 | // Description of the extra info, used to interpret the actual optional | ||||||||
201 | // data appended. | ||||||||
202 | // | ||||||||
203 | // Note that this is not terribly space optimized. This leaves a great deal | ||||||||
204 | // of flexibility to fit more in here later. | ||||||||
205 | const int NumMMOs; | ||||||||
206 | const bool HasPreInstrSymbol; | ||||||||
207 | const bool HasPostInstrSymbol; | ||||||||
208 | const bool HasHeapAllocMarker; | ||||||||
209 | |||||||||
210 | // Implement the `TrailingObjects` internal API. | ||||||||
211 | size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const { | ||||||||
212 | return NumMMOs; | ||||||||
213 | } | ||||||||
214 | size_t numTrailingObjects(OverloadToken<MCSymbol *>) const { | ||||||||
215 | return HasPreInstrSymbol + HasPostInstrSymbol; | ||||||||
216 | } | ||||||||
217 | size_t numTrailingObjects(OverloadToken<MDNode *>) const { | ||||||||
218 | return HasHeapAllocMarker; | ||||||||
219 | } | ||||||||
220 | |||||||||
221 | // Just a boring constructor to allow us to initialize the sizes. Always use | ||||||||
222 | // the `create` routine above. | ||||||||
223 | ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, | ||||||||
224 | bool HasHeapAllocMarker) | ||||||||
225 | : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), | ||||||||
226 | HasPostInstrSymbol(HasPostInstrSymbol), | ||||||||
227 | HasHeapAllocMarker(HasHeapAllocMarker) {} | ||||||||
228 | }; | ||||||||
229 | |||||||||
230 | /// Enumeration of the kinds of inline extra info available. It is important | ||||||||
231 | /// that the `MachineMemOperand` inline kind has a tag value of zero to make | ||||||||
232 | /// it accessible as an `ArrayRef`. | ||||||||
233 | enum ExtraInfoInlineKinds { | ||||||||
234 | EIIK_MMO = 0, | ||||||||
235 | EIIK_PreInstrSymbol, | ||||||||
236 | EIIK_PostInstrSymbol, | ||||||||
237 | EIIK_OutOfLine | ||||||||
238 | }; | ||||||||
239 | |||||||||
240 | // We store extra information about the instruction here. The common case is | ||||||||
241 | // expected to be nothing or a single pointer (typically a MMO or a symbol). | ||||||||
242 | // We work to optimize this common case by storing it inline here rather than | ||||||||
243 | // requiring a separate allocation, but we fall back to an allocation when | ||||||||
244 | // multiple pointers are needed. | ||||||||
245 | PointerSumType<ExtraInfoInlineKinds, | ||||||||
246 | PointerSumTypeMember<EIIK_MMO, MachineMemOperand *>, | ||||||||
247 | PointerSumTypeMember<EIIK_PreInstrSymbol, MCSymbol *>, | ||||||||
248 | PointerSumTypeMember<EIIK_PostInstrSymbol, MCSymbol *>, | ||||||||
249 | PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>> | ||||||||
250 | Info; | ||||||||
251 | |||||||||
252 | DebugLoc debugLoc; // Source line information. | ||||||||
253 | |||||||||
254 | /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values | ||||||||
255 | /// defined by this instruction. | ||||||||
256 | unsigned DebugInstrNum; | ||||||||
257 | |||||||||
258 | // Intrusive list support | ||||||||
259 | friend struct ilist_traits<MachineInstr>; | ||||||||
260 | friend struct ilist_callback_traits<MachineBasicBlock>; | ||||||||
261 | void setParent(MachineBasicBlock *P) { Parent = P; } | ||||||||
262 | |||||||||
263 | /// This constructor creates a copy of the given | ||||||||
264 | /// MachineInstr in the given MachineFunction. | ||||||||
265 | MachineInstr(MachineFunction &, const MachineInstr &); | ||||||||
266 | |||||||||
267 | /// This constructor create a MachineInstr and add the implicit operands. | ||||||||
268 | /// It reserves space for number of operands specified by | ||||||||
269 | /// MCInstrDesc. An explicit DebugLoc is supplied. | ||||||||
270 | MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl, | ||||||||
271 | bool NoImp = false); | ||||||||
272 | |||||||||
273 | // MachineInstrs are pool-allocated and owned by MachineFunction. | ||||||||
274 | friend class MachineFunction; | ||||||||
275 | |||||||||
276 | void | ||||||||
277 | dumprImpl(const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth, | ||||||||
278 | SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const; | ||||||||
279 | |||||||||
280 | public: | ||||||||
281 | MachineInstr(const MachineInstr &) = delete; | ||||||||
282 | MachineInstr &operator=(const MachineInstr &) = delete; | ||||||||
283 | // Use MachineFunction::DeleteMachineInstr() instead. | ||||||||
284 | ~MachineInstr() = delete; | ||||||||
285 | |||||||||
286 | const MachineBasicBlock* getParent() const { return Parent; } | ||||||||
287 | MachineBasicBlock* getParent() { return Parent; } | ||||||||
288 | |||||||||
289 | /// Move the instruction before \p MovePos. | ||||||||
290 | void moveBefore(MachineInstr *MovePos); | ||||||||
291 | |||||||||
292 | /// Return the function that contains the basic block that this instruction | ||||||||
293 | /// belongs to. | ||||||||
294 | /// | ||||||||
295 | /// Note: this is undefined behaviour if the instruction does not have a | ||||||||
296 | /// parent. | ||||||||
297 | const MachineFunction *getMF() const; | ||||||||
298 | MachineFunction *getMF() { | ||||||||
299 | return const_cast<MachineFunction *>( | ||||||||
300 | static_cast<const MachineInstr *>(this)->getMF()); | ||||||||
301 | } | ||||||||
302 | |||||||||
303 | /// Return the asm printer flags bitvector. | ||||||||
304 | uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } | ||||||||
305 | |||||||||
306 | /// Clear the AsmPrinter bitvector. | ||||||||
307 | void clearAsmPrinterFlags() { AsmPrinterFlags = 0; } | ||||||||
308 | |||||||||
309 | /// Return whether an AsmPrinter flag is set. | ||||||||
310 | bool getAsmPrinterFlag(CommentFlag Flag) const { | ||||||||
311 | return AsmPrinterFlags & Flag; | ||||||||
312 | } | ||||||||
313 | |||||||||
314 | /// Set a flag for the AsmPrinter. | ||||||||
315 | void setAsmPrinterFlag(uint8_t Flag) { | ||||||||
316 | AsmPrinterFlags |= Flag; | ||||||||
317 | } | ||||||||
318 | |||||||||
319 | /// Clear specific AsmPrinter flags. | ||||||||
320 | void clearAsmPrinterFlag(CommentFlag Flag) { | ||||||||
321 | AsmPrinterFlags &= ~Flag; | ||||||||
322 | } | ||||||||
323 | |||||||||
324 | /// Return the MI flags bitvector. | ||||||||
325 | uint16_t getFlags() const { | ||||||||
326 | return Flags; | ||||||||
327 | } | ||||||||
328 | |||||||||
329 | /// Return whether an MI flag is set. | ||||||||
330 | bool getFlag(MIFlag Flag) const { | ||||||||
331 | return Flags & Flag; | ||||||||
332 | } | ||||||||
333 | |||||||||
334 | /// Set a MI flag. | ||||||||
335 | void setFlag(MIFlag Flag) { | ||||||||
336 | Flags |= (uint16_t)Flag; | ||||||||
337 | } | ||||||||
338 | |||||||||
339 | void setFlags(unsigned flags) { | ||||||||
340 | // Filter out the automatically maintained flags. | ||||||||
341 | unsigned Mask = BundledPred | BundledSucc; | ||||||||
342 | Flags = (Flags & Mask) | (flags & ~Mask); | ||||||||
343 | } | ||||||||
344 | |||||||||
345 | /// clearFlag - Clear a MI flag. | ||||||||
346 | void clearFlag(MIFlag Flag) { | ||||||||
347 | Flags &= ~((uint16_t)Flag); | ||||||||
348 | } | ||||||||
349 | |||||||||
350 | /// Return true if MI is in a bundle (but not the first MI in a bundle). | ||||||||
351 | /// | ||||||||
352 | /// A bundle looks like this before it's finalized: | ||||||||
353 | /// ---------------- | ||||||||
354 | /// | MI | | ||||||||
355 | /// ---------------- | ||||||||
356 | /// | | ||||||||
357 | /// ---------------- | ||||||||
358 | /// | MI * | | ||||||||
359 | /// ---------------- | ||||||||
360 | /// | | ||||||||
361 | /// ---------------- | ||||||||
362 | /// | MI * | | ||||||||
363 | /// ---------------- | ||||||||
364 | /// In this case, the first MI starts a bundle but is not inside a bundle, the | ||||||||
365 | /// next 2 MIs are considered "inside" the bundle. | ||||||||
366 | /// | ||||||||
367 | /// After a bundle is finalized, it looks like this: | ||||||||
368 | /// ---------------- | ||||||||
369 | /// | Bundle | | ||||||||
370 | /// ---------------- | ||||||||
371 | /// | | ||||||||
372 | /// ---------------- | ||||||||
373 | /// | MI * | | ||||||||
374 | /// ---------------- | ||||||||
375 | /// | | ||||||||
376 | /// ---------------- | ||||||||
377 | /// | MI * | | ||||||||
378 | /// ---------------- | ||||||||
379 | /// | | ||||||||
380 | /// ---------------- | ||||||||
381 | /// | MI * | | ||||||||
382 | /// ---------------- | ||||||||
383 | /// The first instruction has the special opcode "BUNDLE". It's not "inside" | ||||||||
384 | /// a bundle, but the next three MIs are. | ||||||||
385 | bool isInsideBundle() const { | ||||||||
386 | return getFlag(BundledPred); | ||||||||
387 | } | ||||||||
388 | |||||||||
389 | /// Return true if this instruction part of a bundle. This is true | ||||||||
390 | /// if either itself or its following instruction is marked "InsideBundle". | ||||||||
391 | bool isBundled() const { | ||||||||
392 | return isBundledWithPred() || isBundledWithSucc(); | ||||||||
393 | } | ||||||||
394 | |||||||||
395 | /// Return true if this instruction is part of a bundle, and it is not the | ||||||||
396 | /// first instruction in the bundle. | ||||||||
397 | bool isBundledWithPred() const { return getFlag(BundledPred); } | ||||||||
398 | |||||||||
399 | /// Return true if this instruction is part of a bundle, and it is not the | ||||||||
400 | /// last instruction in the bundle. | ||||||||
401 | bool isBundledWithSucc() const { return getFlag(BundledSucc); } | ||||||||
402 | |||||||||
403 | /// Bundle this instruction with its predecessor. This can be an unbundled | ||||||||
404 | /// instruction, or it can be the first instruction in a bundle. | ||||||||
405 | void bundleWithPred(); | ||||||||
406 | |||||||||
407 | /// Bundle this instruction with its successor. This can be an unbundled | ||||||||
408 | /// instruction, or it can be the last instruction in a bundle. | ||||||||
409 | void bundleWithSucc(); | ||||||||
410 | |||||||||
411 | /// Break bundle above this instruction. | ||||||||
412 | void unbundleFromPred(); | ||||||||
413 | |||||||||
414 | /// Break bundle below this instruction. | ||||||||
415 | void unbundleFromSucc(); | ||||||||
416 | |||||||||
417 | /// Returns the debug location id of this MachineInstr. | ||||||||
418 | const DebugLoc &getDebugLoc() const { return debugLoc; } | ||||||||
419 | |||||||||
420 | /// Return the operand containing the offset to be used if this DBG_VALUE | ||||||||
421 | /// instruction is indirect; will be an invalid register if this value is | ||||||||
422 | /// not indirect, and an immediate with value 0 otherwise. | ||||||||
423 | const MachineOperand &getDebugOffset() const { | ||||||||
424 | assert(isNonListDebugValue() && "not a DBG_VALUE")(static_cast <bool> (isNonListDebugValue() && "not a DBG_VALUE" ) ? void (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 424, __extension__ __PRETTY_FUNCTION__)); | ||||||||
425 | return getOperand(1); | ||||||||
426 | } | ||||||||
427 | MachineOperand &getDebugOffset() { | ||||||||
428 | assert(isNonListDebugValue() && "not a DBG_VALUE")(static_cast <bool> (isNonListDebugValue() && "not a DBG_VALUE" ) ? void (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 428, __extension__ __PRETTY_FUNCTION__)); | ||||||||
429 | return getOperand(1); | ||||||||
430 | } | ||||||||
431 | |||||||||
432 | /// Return the operand for the debug variable referenced by | ||||||||
433 | /// this DBG_VALUE instruction. | ||||||||
434 | const MachineOperand &getDebugVariableOp() const; | ||||||||
435 | MachineOperand &getDebugVariableOp(); | ||||||||
436 | |||||||||
437 | /// Return the debug variable referenced by | ||||||||
438 | /// this DBG_VALUE instruction. | ||||||||
439 | const DILocalVariable *getDebugVariable() const; | ||||||||
440 | |||||||||
441 | /// Return the operand for the complex address expression referenced by | ||||||||
442 | /// this DBG_VALUE instruction. | ||||||||
443 | const MachineOperand &getDebugExpressionOp() const; | ||||||||
444 | MachineOperand &getDebugExpressionOp(); | ||||||||
445 | |||||||||
446 | /// Return the complex address expression referenced by | ||||||||
447 | /// this DBG_VALUE instruction. | ||||||||
448 | const DIExpression *getDebugExpression() const; | ||||||||
449 | |||||||||
450 | /// Return the debug label referenced by | ||||||||
451 | /// this DBG_LABEL instruction. | ||||||||
452 | const DILabel *getDebugLabel() const; | ||||||||
453 | |||||||||
454 | /// Fetch the instruction number of this MachineInstr. If it does not have | ||||||||
455 | /// one already, a new and unique number will be assigned. | ||||||||
456 | unsigned getDebugInstrNum(); | ||||||||
457 | |||||||||
458 | /// Examine the instruction number of this MachineInstr. May be zero if | ||||||||
459 | /// it hasn't been assigned a number yet. | ||||||||
460 | unsigned peekDebugInstrNum() const { return DebugInstrNum; } | ||||||||
461 | |||||||||
462 | /// Set instruction number of this MachineInstr. Avoid using unless you're | ||||||||
463 | /// deserializing this information. | ||||||||
464 | void setDebugInstrNum(unsigned Num) { DebugInstrNum = Num; } | ||||||||
465 | |||||||||
466 | /// Emit an error referring to the source location of this instruction. | ||||||||
467 | /// This should only be used for inline assembly that is somehow | ||||||||
468 | /// impossible to compile. Other errors should have been handled much | ||||||||
469 | /// earlier. | ||||||||
470 | /// | ||||||||
471 | /// If this method returns, the caller should try to recover from the error. | ||||||||
472 | void emitError(StringRef Msg) const; | ||||||||
473 | |||||||||
474 | /// Returns the target instruction descriptor of this MachineInstr. | ||||||||
475 | const MCInstrDesc &getDesc() const { return *MCID; } | ||||||||
476 | |||||||||
477 | /// Returns the opcode of this MachineInstr. | ||||||||
478 | unsigned getOpcode() const { return MCID->Opcode; } | ||||||||
479 | |||||||||
480 | /// Retuns the total number of operands. | ||||||||
481 | unsigned getNumOperands() const { return NumOperands; } | ||||||||
482 | |||||||||
483 | /// Returns the total number of operands which are debug locations. | ||||||||
484 | unsigned getNumDebugOperands() const { | ||||||||
485 | return std::distance(debug_operands().begin(), debug_operands().end()); | ||||||||
486 | } | ||||||||
487 | |||||||||
488 | const MachineOperand& getOperand(unsigned i) const { | ||||||||
489 | assert(i < getNumOperands() && "getOperand() out of range!")(static_cast <bool> (i < getNumOperands() && "getOperand() out of range!") ? void (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 489, __extension__ __PRETTY_FUNCTION__)); | ||||||||
490 | return Operands[i]; | ||||||||
491 | } | ||||||||
492 | MachineOperand& getOperand(unsigned i) { | ||||||||
493 | assert(i < getNumOperands() && "getOperand() out of range!")(static_cast <bool> (i < getNumOperands() && "getOperand() out of range!") ? void (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 493, __extension__ __PRETTY_FUNCTION__)); | ||||||||
494 | return Operands[i]; | ||||||||
495 | } | ||||||||
496 | |||||||||
497 | MachineOperand &getDebugOperand(unsigned Index) { | ||||||||
498 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")(static_cast <bool> (Index < getNumDebugOperands() && "getDebugOperand() out of range!") ? void (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 498, __extension__ __PRETTY_FUNCTION__)); | ||||||||
499 | return *(debug_operands().begin() + Index); | ||||||||
500 | } | ||||||||
501 | const MachineOperand &getDebugOperand(unsigned Index) const { | ||||||||
502 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")(static_cast <bool> (Index < getNumDebugOperands() && "getDebugOperand() out of range!") ? void (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 502, __extension__ __PRETTY_FUNCTION__)); | ||||||||
503 | return *(debug_operands().begin() + Index); | ||||||||
504 | } | ||||||||
505 | |||||||||
506 | SmallSet<Register, 4> getUsedDebugRegs() const { | ||||||||
507 | assert(isDebugValue() && "not a DBG_VALUE*")(static_cast <bool> (isDebugValue() && "not a DBG_VALUE*" ) ? void (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE*\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 507, __extension__ __PRETTY_FUNCTION__)); | ||||||||
508 | SmallSet<Register, 4> UsedRegs; | ||||||||
509 | for (auto MO : debug_operands()) | ||||||||
510 | if (MO.isReg() && MO.getReg()) | ||||||||
511 | UsedRegs.insert(MO.getReg()); | ||||||||
512 | return UsedRegs; | ||||||||
513 | } | ||||||||
514 | |||||||||
515 | /// Returns whether this debug value has at least one debug operand with the | ||||||||
516 | /// register \p Reg. | ||||||||
517 | bool hasDebugOperandForReg(Register Reg) const { | ||||||||
518 | return any_of(debug_operands(), [Reg](const MachineOperand &Op) { | ||||||||
519 | return Op.isReg() && Op.getReg() == Reg; | ||||||||
520 | }); | ||||||||
521 | } | ||||||||
522 | |||||||||
523 | /// Returns a range of all of the operands that correspond to a debug use of | ||||||||
524 | /// \p Reg. | ||||||||
525 | template <typename Operand, typename Instruction> | ||||||||
526 | static iterator_range< | ||||||||
527 | filter_iterator<Operand *, std::function<bool(Operand &Op)>>> | ||||||||
528 | getDebugOperandsForReg(Instruction *MI, Register Reg) { | ||||||||
529 | std::function<bool(Operand & Op)> OpUsesReg( | ||||||||
530 | [Reg](Operand &Op) { return Op.isReg() && Op.getReg() == Reg; }); | ||||||||
531 | return make_filter_range(MI->debug_operands(), OpUsesReg); | ||||||||
532 | } | ||||||||
533 | iterator_range<filter_iterator<const MachineOperand *, | ||||||||
534 | std::function<bool(const MachineOperand &Op)>>> | ||||||||
535 | getDebugOperandsForReg(Register Reg) const { | ||||||||
536 | return MachineInstr::getDebugOperandsForReg<const MachineOperand, | ||||||||
537 | const MachineInstr>(this, Reg); | ||||||||
538 | } | ||||||||
539 | iterator_range<filter_iterator<MachineOperand *, | ||||||||
540 | std::function<bool(MachineOperand &Op)>>> | ||||||||
541 | getDebugOperandsForReg(Register Reg) { | ||||||||
542 | return MachineInstr::getDebugOperandsForReg<MachineOperand, MachineInstr>( | ||||||||
543 | this, Reg); | ||||||||
544 | } | ||||||||
545 | |||||||||
546 | bool isDebugOperand(const MachineOperand *Op) const { | ||||||||
547 | return Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()); | ||||||||
548 | } | ||||||||
549 | |||||||||
550 | unsigned getDebugOperandIndex(const MachineOperand *Op) const { | ||||||||
551 | assert(isDebugOperand(Op) && "Expected a debug operand.")(static_cast <bool> (isDebugOperand(Op) && "Expected a debug operand." ) ? void (0) : __assert_fail ("isDebugOperand(Op) && \"Expected a debug operand.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 551, __extension__ __PRETTY_FUNCTION__)); | ||||||||
552 | return std::distance(adl_begin(debug_operands()), Op); | ||||||||
553 | } | ||||||||
554 | |||||||||
555 | /// Returns the total number of definitions. | ||||||||
556 | unsigned getNumDefs() const { | ||||||||
557 | return getNumExplicitDefs() + MCID->getNumImplicitDefs(); | ||||||||
558 | } | ||||||||
559 | |||||||||
560 | /// Returns true if the instruction has implicit definition. | ||||||||
561 | bool hasImplicitDef() const { | ||||||||
562 | for (unsigned I = getNumExplicitOperands(), E = getNumOperands(); | ||||||||
563 | I != E; ++I) { | ||||||||
564 | const MachineOperand &MO = getOperand(I); | ||||||||
565 | if (MO.isDef() && MO.isImplicit()) | ||||||||
566 | return true; | ||||||||
567 | } | ||||||||
568 | return false; | ||||||||
569 | } | ||||||||
570 | |||||||||
571 | /// Returns the implicit operands number. | ||||||||
572 | unsigned getNumImplicitOperands() const { | ||||||||
573 | return getNumOperands() - getNumExplicitOperands(); | ||||||||
574 | } | ||||||||
575 | |||||||||
576 | /// Return true if operand \p OpIdx is a subregister index. | ||||||||
577 | bool isOperandSubregIdx(unsigned OpIdx) const { | ||||||||
578 | assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate &&(static_cast <bool> (getOperand(OpIdx).getType() == MachineOperand ::MO_Immediate && "Expected MO_Immediate operand type." ) ? void (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 579, __extension__ __PRETTY_FUNCTION__)) | ||||||||
579 | "Expected MO_Immediate operand type.")(static_cast <bool> (getOperand(OpIdx).getType() == MachineOperand ::MO_Immediate && "Expected MO_Immediate operand type." ) ? void (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 579, __extension__ __PRETTY_FUNCTION__)); | ||||||||
580 | if (isExtractSubreg() && OpIdx == 2) | ||||||||
581 | return true; | ||||||||
582 | if (isInsertSubreg() && OpIdx == 3) | ||||||||
583 | return true; | ||||||||
584 | if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0) | ||||||||
585 | return true; | ||||||||
586 | if (isSubregToReg() && OpIdx == 3) | ||||||||
587 | return true; | ||||||||
588 | return false; | ||||||||
589 | } | ||||||||
590 | |||||||||
591 | /// Returns the number of non-implicit operands. | ||||||||
592 | unsigned getNumExplicitOperands() const; | ||||||||
593 | |||||||||
594 | /// Returns the number of non-implicit definitions. | ||||||||
595 | unsigned getNumExplicitDefs() const; | ||||||||
596 | |||||||||
597 | /// iterator/begin/end - Iterate over all operands of a machine instruction. | ||||||||
598 | using mop_iterator = MachineOperand *; | ||||||||
599 | using const_mop_iterator = const MachineOperand *; | ||||||||
600 | |||||||||
601 | mop_iterator operands_begin() { return Operands; } | ||||||||
602 | mop_iterator operands_end() { return Operands + NumOperands; } | ||||||||
603 | |||||||||
604 | const_mop_iterator operands_begin() const { return Operands; } | ||||||||
605 | const_mop_iterator operands_end() const { return Operands + NumOperands; } | ||||||||
606 | |||||||||
607 | iterator_range<mop_iterator> operands() { | ||||||||
608 | return make_range(operands_begin(), operands_end()); | ||||||||
609 | } | ||||||||
610 | iterator_range<const_mop_iterator> operands() const { | ||||||||
611 | return make_range(operands_begin(), operands_end()); | ||||||||
612 | } | ||||||||
613 | iterator_range<mop_iterator> explicit_operands() { | ||||||||
614 | return make_range(operands_begin(), | ||||||||
615 | operands_begin() + getNumExplicitOperands()); | ||||||||
616 | } | ||||||||
617 | iterator_range<const_mop_iterator> explicit_operands() const { | ||||||||
618 | return make_range(operands_begin(), | ||||||||
619 | operands_begin() + getNumExplicitOperands()); | ||||||||
620 | } | ||||||||
621 | iterator_range<mop_iterator> implicit_operands() { | ||||||||
622 | return make_range(explicit_operands().end(), operands_end()); | ||||||||
623 | } | ||||||||
624 | iterator_range<const_mop_iterator> implicit_operands() const { | ||||||||
625 | return make_range(explicit_operands().end(), operands_end()); | ||||||||
626 | } | ||||||||
627 | /// Returns a range over all operands that are used to determine the variable | ||||||||
628 | /// location for this DBG_VALUE instruction. | ||||||||
629 | iterator_range<mop_iterator> debug_operands() { | ||||||||
630 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 630, __extension__ __PRETTY_FUNCTION__)); | ||||||||
631 | return isDebugValueList() | ||||||||
632 | ? make_range(operands_begin() + 2, operands_end()) | ||||||||
633 | : make_range(operands_begin(), operands_begin() + 1); | ||||||||
634 | } | ||||||||
635 | /// \copydoc debug_operands() | ||||||||
636 | iterator_range<const_mop_iterator> debug_operands() const { | ||||||||
637 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 637, __extension__ __PRETTY_FUNCTION__)); | ||||||||
638 | return isDebugValueList() | ||||||||
639 | ? make_range(operands_begin() + 2, operands_end()) | ||||||||
640 | : make_range(operands_begin(), operands_begin() + 1); | ||||||||
641 | } | ||||||||
642 | /// Returns a range over all explicit operands that are register definitions. | ||||||||
643 | /// Implicit definition are not included! | ||||||||
644 | iterator_range<mop_iterator> defs() { | ||||||||
645 | return make_range(operands_begin(), | ||||||||
646 | operands_begin() + getNumExplicitDefs()); | ||||||||
647 | } | ||||||||
648 | /// \copydoc defs() | ||||||||
649 | iterator_range<const_mop_iterator> defs() const { | ||||||||
650 | return make_range(operands_begin(), | ||||||||
651 | operands_begin() + getNumExplicitDefs()); | ||||||||
652 | } | ||||||||
653 | /// Returns a range that includes all operands that are register uses. | ||||||||
654 | /// This may include unrelated operands which are not register uses. | ||||||||
655 | iterator_range<mop_iterator> uses() { | ||||||||
656 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); | ||||||||
657 | } | ||||||||
658 | /// \copydoc uses() | ||||||||
659 | iterator_range<const_mop_iterator> uses() const { | ||||||||
660 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); | ||||||||
661 | } | ||||||||
662 | iterator_range<mop_iterator> explicit_uses() { | ||||||||
663 | return make_range(operands_begin() + getNumExplicitDefs(), | ||||||||
664 | operands_begin() + getNumExplicitOperands()); | ||||||||
665 | } | ||||||||
666 | iterator_range<const_mop_iterator> explicit_uses() const { | ||||||||
667 | return make_range(operands_begin() + getNumExplicitDefs(), | ||||||||
668 | operands_begin() + getNumExplicitOperands()); | ||||||||
669 | } | ||||||||
670 | |||||||||
671 | /// Returns the number of the operand iterator \p I points to. | ||||||||
672 | unsigned getOperandNo(const_mop_iterator I) const { | ||||||||
673 | return I - operands_begin(); | ||||||||
674 | } | ||||||||
675 | |||||||||
676 | /// Access to memory operands of the instruction. If there are none, that does | ||||||||
677 | /// not imply anything about whether the function accesses memory. Instead, | ||||||||
678 | /// the caller must behave conservatively. | ||||||||
679 | ArrayRef<MachineMemOperand *> memoperands() const { | ||||||||
680 | if (!Info) | ||||||||
681 | return {}; | ||||||||
682 | |||||||||
683 | if (Info.is<EIIK_MMO>()) | ||||||||
684 | return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); | ||||||||
685 | |||||||||
686 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) | ||||||||
687 | return EI->getMMOs(); | ||||||||
688 | |||||||||
689 | return {}; | ||||||||
690 | } | ||||||||
691 | |||||||||
692 | /// Access to memory operands of the instruction. | ||||||||
693 | /// | ||||||||
694 | /// If `memoperands_begin() == memoperands_end()`, that does not imply | ||||||||
695 | /// anything about whether the function accesses memory. Instead, the caller | ||||||||
696 | /// must behave conservatively. | ||||||||
697 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } | ||||||||
698 | |||||||||
699 | /// Access to memory operands of the instruction. | ||||||||
700 | /// | ||||||||
701 | /// If `memoperands_begin() == memoperands_end()`, that does not imply | ||||||||
702 | /// anything about whether the function accesses memory. Instead, the caller | ||||||||
703 | /// must behave conservatively. | ||||||||
704 | mmo_iterator memoperands_end() const { return memoperands().end(); } | ||||||||
705 | |||||||||
706 | /// Return true if we don't have any memory operands which described the | ||||||||
707 | /// memory access done by this instruction. If this is true, calling code | ||||||||
708 | /// must be conservative. | ||||||||
709 | bool memoperands_empty() const { return memoperands().empty(); } | ||||||||
710 | |||||||||
711 | /// Return true if this instruction has exactly one MachineMemOperand. | ||||||||
712 | bool hasOneMemOperand() const { return memoperands().size() == 1; } | ||||||||
713 | |||||||||
714 | /// Return the number of memory operands. | ||||||||
715 | unsigned getNumMemOperands() const { return memoperands().size(); } | ||||||||
716 | |||||||||
717 | /// Helper to extract a pre-instruction symbol if one has been added. | ||||||||
718 | MCSymbol *getPreInstrSymbol() const { | ||||||||
719 | if (!Info) | ||||||||
720 | return nullptr; | ||||||||
721 | if (MCSymbol *S = Info.get<EIIK_PreInstrSymbol>()) | ||||||||
722 | return S; | ||||||||
723 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) | ||||||||
724 | return EI->getPreInstrSymbol(); | ||||||||
725 | |||||||||
726 | return nullptr; | ||||||||
727 | } | ||||||||
728 | |||||||||
729 | /// Helper to extract a post-instruction symbol if one has been added. | ||||||||
730 | MCSymbol *getPostInstrSymbol() const { | ||||||||
731 | if (!Info) | ||||||||
732 | return nullptr; | ||||||||
733 | if (MCSymbol *S = Info.get<EIIK_PostInstrSymbol>()) | ||||||||
734 | return S; | ||||||||
735 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) | ||||||||
736 | return EI->getPostInstrSymbol(); | ||||||||
737 | |||||||||
738 | return nullptr; | ||||||||
739 | } | ||||||||
740 | |||||||||
741 | /// Helper to extract a heap alloc marker if one has been added. | ||||||||
742 | MDNode *getHeapAllocMarker() const { | ||||||||
743 | if (!Info) | ||||||||
744 | return nullptr; | ||||||||
745 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) | ||||||||
746 | return EI->getHeapAllocMarker(); | ||||||||
747 | |||||||||
748 | return nullptr; | ||||||||
749 | } | ||||||||
750 | |||||||||
751 | /// API for querying MachineInstr properties. They are the same as MCInstrDesc | ||||||||
752 | /// queries but they are bundle aware. | ||||||||
753 | |||||||||
754 | enum QueryType { | ||||||||
755 | IgnoreBundle, // Ignore bundles | ||||||||
756 | AnyInBundle, // Return true if any instruction in bundle has property | ||||||||
757 | AllInBundle // Return true if all instructions in bundle have property | ||||||||
758 | }; | ||||||||
759 | |||||||||
760 | /// Return true if the instruction (or in the case of a bundle, | ||||||||
761 | /// the instructions inside the bundle) has the specified property. | ||||||||
762 | /// The first argument is the property being queried. | ||||||||
763 | /// The second argument indicates whether the query should look inside | ||||||||
764 | /// instruction bundles. | ||||||||
765 | bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { | ||||||||
766 | assert(MCFlag < 64 &&(static_cast <bool> (MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? void (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 767, __extension__ __PRETTY_FUNCTION__)) | ||||||||
767 | "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.")(static_cast <bool> (MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? void (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 767, __extension__ __PRETTY_FUNCTION__)); | ||||||||
768 | // Inline the fast path for unbundled or bundle-internal instructions. | ||||||||
769 | if (Type
| ||||||||
770 | return getDesc().getFlags() & (1ULL << MCFlag); | ||||||||
771 | |||||||||
772 | // If this is the first instruction in a bundle, take the slow path. | ||||||||
773 | return hasPropertyInBundle(1ULL << MCFlag, Type); | ||||||||
774 | } | ||||||||
775 | |||||||||
776 | /// Return true if this is an instruction that should go through the usual | ||||||||
777 | /// legalization steps. | ||||||||
778 | bool isPreISelOpcode(QueryType Type = IgnoreBundle) const { | ||||||||
779 | return hasProperty(MCID::PreISelOpcode, Type); | ||||||||
780 | } | ||||||||
781 | |||||||||
782 | /// Return true if this instruction can have a variable number of operands. | ||||||||
783 | /// In this case, the variable operands will be after the normal | ||||||||
784 | /// operands but before the implicit definitions and uses (if any are | ||||||||
785 | /// present). | ||||||||
786 | bool isVariadic(QueryType Type = IgnoreBundle) const { | ||||||||
787 | return hasProperty(MCID::Variadic, Type); | ||||||||
788 | } | ||||||||
789 | |||||||||
790 | /// Set if this instruction has an optional definition, e.g. | ||||||||
791 | /// ARM instructions which can set condition code if 's' bit is set. | ||||||||
792 | bool hasOptionalDef(QueryType Type = IgnoreBundle) const { | ||||||||
793 | return hasProperty(MCID::HasOptionalDef, Type); | ||||||||
794 | } | ||||||||
795 | |||||||||
796 | /// Return true if this is a pseudo instruction that doesn't | ||||||||
797 | /// correspond to a real machine instruction. | ||||||||
798 | bool isPseudo(QueryType Type = IgnoreBundle) const { | ||||||||
799 | return hasProperty(MCID::Pseudo, Type); | ||||||||
800 | } | ||||||||
801 | |||||||||
802 | bool isReturn(QueryType Type = AnyInBundle) const { | ||||||||
803 | return hasProperty(MCID::Return, Type); | ||||||||
804 | } | ||||||||
805 | |||||||||
806 | /// Return true if this is an instruction that marks the end of an EH scope, | ||||||||
807 | /// i.e., a catchpad or a cleanuppad instruction. | ||||||||
808 | bool isEHScopeReturn(QueryType Type = AnyInBundle) const { | ||||||||
809 | return hasProperty(MCID::EHScopeReturn, Type); | ||||||||
810 | } | ||||||||
811 | |||||||||
812 | bool isCall(QueryType Type = AnyInBundle) const { | ||||||||
813 | return hasProperty(MCID::Call, Type); | ||||||||
814 | } | ||||||||
815 | |||||||||
816 | /// Return true if this is a call instruction that may have an associated | ||||||||
817 | /// call site entry in the debug info. | ||||||||
818 | bool isCandidateForCallSiteEntry(QueryType Type = IgnoreBundle) const; | ||||||||
819 | /// Return true if copying, moving, or erasing this instruction requires | ||||||||
820 | /// updating Call Site Info (see \ref copyCallSiteInfo, \ref moveCallSiteInfo, | ||||||||
821 | /// \ref eraseCallSiteInfo). | ||||||||
822 | bool shouldUpdateCallSiteInfo() const; | ||||||||
823 | |||||||||
824 | /// Returns true if the specified instruction stops control flow | ||||||||
825 | /// from executing the instruction immediately following it. Examples include | ||||||||
826 | /// unconditional branches and return instructions. | ||||||||
827 | bool isBarrier(QueryType Type = AnyInBundle) const { | ||||||||
828 | return hasProperty(MCID::Barrier, Type); | ||||||||
829 | } | ||||||||
830 | |||||||||
831 | /// Returns true if this instruction part of the terminator for a basic block. | ||||||||
832 | /// Typically this is things like return and branch instructions. | ||||||||
833 | /// | ||||||||
834 | /// Various passes use this to insert code into the bottom of a basic block, | ||||||||
835 | /// but before control flow occurs. | ||||||||
836 | bool isTerminator(QueryType Type = AnyInBundle) const { | ||||||||
837 | return hasProperty(MCID::Terminator, Type); | ||||||||
838 | } | ||||||||
839 | |||||||||
840 | /// Returns true if this is a conditional, unconditional, or indirect branch. | ||||||||
841 | /// Predicates below can be used to discriminate between | ||||||||
842 | /// these cases, and the TargetInstrInfo::analyzeBranch method can be used to | ||||||||
843 | /// get more information. | ||||||||
844 | bool isBranch(QueryType Type = AnyInBundle) const { | ||||||||
845 | return hasProperty(MCID::Branch, Type); | ||||||||
846 | } | ||||||||
847 | |||||||||
848 | /// Return true if this is an indirect branch, such as a | ||||||||
849 | /// branch through a register. | ||||||||
850 | bool isIndirectBranch(QueryType Type = AnyInBundle) const { | ||||||||
851 | return hasProperty(MCID::IndirectBranch, Type); | ||||||||
852 | } | ||||||||
853 | |||||||||
854 | /// Return true if this is a branch which may fall | ||||||||
855 | /// through to the next instruction or may transfer control flow to some other | ||||||||
856 | /// block. The TargetInstrInfo::analyzeBranch method can be used to get more | ||||||||
857 | /// information about this branch. | ||||||||
858 | bool isConditionalBranch(QueryType Type = AnyInBundle) const { | ||||||||
859 | return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type); | ||||||||
860 | } | ||||||||
861 | |||||||||
862 | /// Return true if this is a branch which always | ||||||||
863 | /// transfers control flow to some other block. The | ||||||||
864 | /// TargetInstrInfo::analyzeBranch method can be used to get more information | ||||||||
865 | /// about this branch. | ||||||||
866 | bool isUnconditionalBranch(QueryType Type = AnyInBundle) const { | ||||||||
867 | return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type); | ||||||||
868 | } | ||||||||
869 | |||||||||
870 | /// Return true if this instruction has a predicate operand that | ||||||||
871 | /// controls execution. It may be set to 'always', or may be set to other | ||||||||
872 | /// values. There are various methods in TargetInstrInfo that can be used to | ||||||||
873 | /// control and modify the predicate in this instruction. | ||||||||
874 | bool isPredicable(QueryType Type = AllInBundle) const { | ||||||||
875 | // If it's a bundle than all bundled instructions must be predicable for this | ||||||||
876 | // to return true. | ||||||||
877 | return hasProperty(MCID::Predicable, Type); | ||||||||
878 | } | ||||||||
879 | |||||||||
880 | /// Return true if this instruction is a comparison. | ||||||||
881 | bool isCompare(QueryType Type = IgnoreBundle) const { | ||||||||
882 | return hasProperty(MCID::Compare, Type); | ||||||||
883 | } | ||||||||
884 | |||||||||
885 | /// Return true if this instruction is a move immediate | ||||||||
886 | /// (including conditional moves) instruction. | ||||||||
887 | bool isMoveImmediate(QueryType Type = IgnoreBundle) const { | ||||||||
888 | return hasProperty(MCID::MoveImm, Type); | ||||||||
889 | } | ||||||||
890 | |||||||||
891 | /// Return true if this instruction is a register move. | ||||||||
892 | /// (including moving values from subreg to reg) | ||||||||
893 | bool isMoveReg(QueryType Type = IgnoreBundle) const { | ||||||||
894 | return hasProperty(MCID::MoveReg, Type); | ||||||||
895 | } | ||||||||
896 | |||||||||
897 | /// Return true if this instruction is a bitcast instruction. | ||||||||
898 | bool isBitcast(QueryType Type = IgnoreBundle) const { | ||||||||
899 | return hasProperty(MCID::Bitcast, Type); | ||||||||
900 | } | ||||||||
901 | |||||||||
902 | /// Return true if this instruction is a select instruction. | ||||||||
903 | bool isSelect(QueryType Type = IgnoreBundle) const { | ||||||||
904 | return hasProperty(MCID::Select, Type); | ||||||||
905 | } | ||||||||
906 | |||||||||
907 | /// Return true if this instruction cannot be safely duplicated. | ||||||||
908 | /// For example, if the instruction has a unique labels attached | ||||||||
909 | /// to it, duplicating it would cause multiple definition errors. | ||||||||
910 | bool isNotDuplicable(QueryType Type = AnyInBundle) const { | ||||||||
911 | return hasProperty(MCID::NotDuplicable, Type); | ||||||||
912 | } | ||||||||
913 | |||||||||
914 | /// Return true if this instruction is convergent. | ||||||||
915 | /// Convergent instructions can not be made control-dependent on any | ||||||||
916 | /// additional values. | ||||||||
917 | bool isConvergent(QueryType Type = AnyInBundle) const { | ||||||||
918 | if (isInlineAsm()) { | ||||||||
919 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); | ||||||||
920 | if (ExtraInfo & InlineAsm::Extra_IsConvergent) | ||||||||
921 | return true; | ||||||||
922 | } | ||||||||
923 | return hasProperty(MCID::Convergent, Type); | ||||||||
924 | } | ||||||||
925 | |||||||||
926 | /// Returns true if the specified instruction has a delay slot | ||||||||
927 | /// which must be filled by the code generator. | ||||||||
928 | bool hasDelaySlot(QueryType Type = AnyInBundle) const { | ||||||||
929 | return hasProperty(MCID::DelaySlot, Type); | ||||||||
930 | } | ||||||||
931 | |||||||||
932 | /// Return true for instructions that can be folded as | ||||||||
933 | /// memory operands in other instructions. The most common use for this | ||||||||
934 | /// is instructions that are simple loads from memory that don't modify | ||||||||
935 | /// the loaded value in any way, but it can also be used for instructions | ||||||||
936 | /// that can be expressed as constant-pool loads, such as V_SETALLONES | ||||||||
937 | /// on x86, to allow them to be folded when it is beneficial. | ||||||||
938 | /// This should only be set on instructions that return a value in their | ||||||||
939 | /// only virtual register definition. | ||||||||
940 | bool canFoldAsLoad(QueryType Type = IgnoreBundle) const { | ||||||||
941 | return hasProperty(MCID::FoldableAsLoad, Type); | ||||||||
942 | } | ||||||||
943 | |||||||||
944 | /// Return true if this instruction behaves | ||||||||
945 | /// the same way as the generic REG_SEQUENCE instructions. | ||||||||
946 | /// E.g., on ARM, | ||||||||
947 | /// dX VMOVDRR rY, rZ | ||||||||
948 | /// is equivalent to | ||||||||
949 | /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. | ||||||||
950 | /// | ||||||||
951 | /// Note that for the optimizers to be able to take advantage of | ||||||||
952 | /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be | ||||||||
953 | /// override accordingly. | ||||||||
954 | bool isRegSequenceLike(QueryType Type = IgnoreBundle) const { | ||||||||
955 | return hasProperty(MCID::RegSequence, Type); | ||||||||
956 | } | ||||||||
957 | |||||||||
958 | /// Return true if this instruction behaves | ||||||||
959 | /// the same way as the generic EXTRACT_SUBREG instructions. | ||||||||
960 | /// E.g., on ARM, | ||||||||
961 | /// rX, rY VMOVRRD dZ | ||||||||
962 | /// is equivalent to two EXTRACT_SUBREG: | ||||||||
963 | /// rX = EXTRACT_SUBREG dZ, ssub_0 | ||||||||
964 | /// rY = EXTRACT_SUBREG dZ, ssub_1 | ||||||||
965 | /// | ||||||||
966 | /// Note that for the optimizers to be able to take advantage of | ||||||||
967 | /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be | ||||||||
968 | /// override accordingly. | ||||||||
969 | bool isExtractSubregLike(QueryType Type = IgnoreBundle) const { | ||||||||
970 | return hasProperty(MCID::ExtractSubreg, Type); | ||||||||
971 | } | ||||||||
972 | |||||||||
973 | /// Return true if this instruction behaves | ||||||||
974 | /// the same way as the generic INSERT_SUBREG instructions. | ||||||||
975 | /// E.g., on ARM, | ||||||||
976 | /// dX = VSETLNi32 dY, rZ, Imm | ||||||||
977 | /// is equivalent to a INSERT_SUBREG: | ||||||||
978 | /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) | ||||||||
979 | /// | ||||||||
980 | /// Note that for the optimizers to be able to take advantage of | ||||||||
981 | /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be | ||||||||
982 | /// override accordingly. | ||||||||
983 | bool isInsertSubregLike(QueryType Type = IgnoreBundle) const { | ||||||||
984 | return hasProperty(MCID::InsertSubreg, Type); | ||||||||
985 | } | ||||||||
986 | |||||||||
987 | //===--------------------------------------------------------------------===// | ||||||||
988 | // Side Effect Analysis | ||||||||
989 | //===--------------------------------------------------------------------===// | ||||||||
990 | |||||||||
991 | /// Return true if this instruction could possibly read memory. | ||||||||
992 | /// Instructions with this flag set are not necessarily simple load | ||||||||
993 | /// instructions, they may load a value and modify it, for example. | ||||||||
994 | bool mayLoad(QueryType Type = AnyInBundle) const { | ||||||||
995 | if (isInlineAsm()) { | ||||||||
996 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); | ||||||||
997 | if (ExtraInfo & InlineAsm::Extra_MayLoad) | ||||||||
998 | return true; | ||||||||
999 | } | ||||||||
1000 | return hasProperty(MCID::MayLoad, Type); | ||||||||
1001 | } | ||||||||
1002 | |||||||||
1003 | /// Return true if this instruction could possibly modify memory. | ||||||||
1004 | /// Instructions with this flag set are not necessarily simple store | ||||||||
1005 | /// instructions, they may store a modified value based on their operands, or | ||||||||
1006 | /// may not actually modify anything, for example. | ||||||||
1007 | bool mayStore(QueryType Type = AnyInBundle) const { | ||||||||
1008 | if (isInlineAsm()) { | ||||||||
1009 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); | ||||||||
1010 | if (ExtraInfo & InlineAsm::Extra_MayStore) | ||||||||
1011 | return true; | ||||||||
1012 | } | ||||||||
1013 | return hasProperty(MCID::MayStore, Type); | ||||||||
1014 | } | ||||||||
1015 | |||||||||
1016 | /// Return true if this instruction could possibly read or modify memory. | ||||||||
1017 | bool mayLoadOrStore(QueryType Type = AnyInBundle) const { | ||||||||
1018 | return mayLoad(Type) || mayStore(Type); | ||||||||
1019 | } | ||||||||
1020 | |||||||||
1021 | /// Return true if this instruction could possibly raise a floating-point | ||||||||
1022 | /// exception. This is the case if the instruction is a floating-point | ||||||||
1023 | /// instruction that can in principle raise an exception, as indicated | ||||||||
1024 | /// by the MCID::MayRaiseFPException property, *and* at the same time, | ||||||||
1025 | /// the instruction is used in a context where we expect floating-point | ||||||||
1026 | /// exceptions are not disabled, as indicated by the NoFPExcept MI flag. | ||||||||
1027 | bool mayRaiseFPException() const { | ||||||||
1028 | return hasProperty(MCID::MayRaiseFPException) && | ||||||||
1029 | !getFlag(MachineInstr::MIFlag::NoFPExcept); | ||||||||
1030 | } | ||||||||
1031 | |||||||||
1032 | //===--------------------------------------------------------------------===// | ||||||||
1033 | // Flags that indicate whether an instruction can be modified by a method. | ||||||||
1034 | //===--------------------------------------------------------------------===// | ||||||||
1035 | |||||||||
1036 | /// Return true if this may be a 2- or 3-address | ||||||||
1037 | /// instruction (of the form "X = op Y, Z, ..."), which produces the same | ||||||||
1038 | /// result if Y and Z are exchanged. If this flag is set, then the | ||||||||
1039 | /// TargetInstrInfo::commuteInstruction method may be used to hack on the | ||||||||
1040 | /// instruction. | ||||||||
1041 | /// | ||||||||
1042 | /// Note that this flag may be set on instructions that are only commutable | ||||||||
1043 | /// sometimes. In these cases, the call to commuteInstruction will fail. | ||||||||
1044 | /// Also note that some instructions require non-trivial modification to | ||||||||
1045 | /// commute them. | ||||||||
1046 | bool isCommutable(QueryType Type = IgnoreBundle) const { | ||||||||
1047 | return hasProperty(MCID::Commutable, Type); | ||||||||
1048 | } | ||||||||
1049 | |||||||||
1050 | /// Return true if this is a 2-address instruction | ||||||||
1051 | /// which can be changed into a 3-address instruction if needed. Doing this | ||||||||
1052 | /// transformation can be profitable in the register allocator, because it | ||||||||
1053 | /// means that the instruction can use a 2-address form if possible, but | ||||||||
1054 | /// degrade into a less efficient form if the source and dest register cannot | ||||||||
1055 | /// be assigned to the same register. For example, this allows the x86 | ||||||||
1056 | /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which | ||||||||
1057 | /// is the same speed as the shift but has bigger code size. | ||||||||
1058 | /// | ||||||||
1059 | /// If this returns true, then the target must implement the | ||||||||
1060 | /// TargetInstrInfo::convertToThreeAddress method for this instruction, which | ||||||||
1061 | /// is allowed to fail if the transformation isn't valid for this specific | ||||||||
1062 | /// instruction (e.g. shl reg, 4 on x86). | ||||||||
1063 | /// | ||||||||
1064 | bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const { | ||||||||
1065 | return hasProperty(MCID::ConvertibleTo3Addr, Type); | ||||||||
1066 | } | ||||||||
1067 | |||||||||
1068 | /// Return true if this instruction requires | ||||||||
1069 | /// custom insertion support when the DAG scheduler is inserting it into a | ||||||||
1070 | /// machine basic block. If this is true for the instruction, it basically | ||||||||
1071 | /// means that it is a pseudo instruction used at SelectionDAG time that is | ||||||||
1072 | /// expanded out into magic code by the target when MachineInstrs are formed. | ||||||||
1073 | /// | ||||||||
1074 | /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method | ||||||||
1075 | /// is used to insert this into the MachineBasicBlock. | ||||||||
1076 | bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const { | ||||||||
1077 | return hasProperty(MCID::UsesCustomInserter, Type); | ||||||||
1078 | } | ||||||||
1079 | |||||||||
1080 | /// Return true if this instruction requires *adjustment* | ||||||||
1081 | /// after instruction selection by calling a target hook. For example, this | ||||||||
1082 | /// can be used to fill in ARM 's' optional operand depending on whether | ||||||||
1083 | /// the conditional flag register is used. | ||||||||
1084 | bool hasPostISelHook(QueryType Type = IgnoreBundle) const { | ||||||||
1085 | return hasProperty(MCID::HasPostISelHook, Type); | ||||||||
1086 | } | ||||||||
1087 | |||||||||
1088 | /// Returns true if this instruction is a candidate for remat. | ||||||||
1089 | /// This flag is deprecated, please don't use it anymore. If this | ||||||||
1090 | /// flag is set, the isReallyTriviallyReMaterializable() method is called to | ||||||||
1091 | /// verify the instruction is really rematable. | ||||||||
1092 | bool isRematerializable(QueryType Type = AllInBundle) const { | ||||||||
1093 | // It's only possible to re-mat a bundle if all bundled instructions are | ||||||||
1094 | // re-materializable. | ||||||||
1095 | return hasProperty(MCID::Rematerializable, Type); | ||||||||
1096 | } | ||||||||
1097 | |||||||||
1098 | /// Returns true if this instruction has the same cost (or less) than a move | ||||||||
1099 | /// instruction. This is useful during certain types of optimizations | ||||||||
1100 | /// (e.g., remat during two-address conversion or machine licm) | ||||||||
1101 | /// where we would like to remat or hoist the instruction, but not if it costs | ||||||||
1102 | /// more than moving the instruction into the appropriate register. Note, we | ||||||||
1103 | /// are not marking copies from and to the same register class with this flag. | ||||||||
1104 | bool isAsCheapAsAMove(QueryType Type = AllInBundle) const { | ||||||||
1105 | // Only returns true for a bundle if all bundled instructions are cheap. | ||||||||
1106 | return hasProperty(MCID::CheapAsAMove, Type); | ||||||||
1107 | } | ||||||||
1108 | |||||||||
1109 | /// Returns true if this instruction source operands | ||||||||
1110 | /// have special register allocation requirements that are not captured by the | ||||||||
1111 | /// operand register classes. e.g. ARM::STRD's two source registers must be an | ||||||||
1112 | /// even / odd pair, ARM::STM registers have to be in ascending order. | ||||||||
1113 | /// Post-register allocation passes should not attempt to change allocations | ||||||||
1114 | /// for sources of instructions with this flag. | ||||||||
1115 | bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const { | ||||||||
1116 | return hasProperty(MCID::ExtraSrcRegAllocReq, Type); | ||||||||
1117 | } | ||||||||
1118 | |||||||||
1119 | /// Returns true if this instruction def operands | ||||||||
1120 | /// have special register allocation requirements that are not captured by the | ||||||||
1121 | /// operand register classes. e.g. ARM::LDRD's two def registers must be an | ||||||||
1122 | /// even / odd pair, ARM::LDM registers have to be in ascending order. | ||||||||
1123 | /// Post-register allocation passes should not attempt to change allocations | ||||||||
1124 | /// for definitions of instructions with this flag. | ||||||||
1125 | bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const { | ||||||||
1126 | return hasProperty(MCID::ExtraDefRegAllocReq, Type); | ||||||||
1127 | } | ||||||||
1128 | |||||||||
1129 | enum MICheckType { | ||||||||
1130 | CheckDefs, // Check all operands for equality | ||||||||
1131 | CheckKillDead, // Check all operands including kill / dead markers | ||||||||
1132 | IgnoreDefs, // Ignore all definitions | ||||||||
1133 | IgnoreVRegDefs // Ignore virtual register definitions | ||||||||
1134 | }; | ||||||||
1135 | |||||||||
1136 | /// Return true if this instruction is identical to \p Other. | ||||||||
1137 | /// Two instructions are identical if they have the same opcode and all their | ||||||||
1138 | /// operands are identical (with respect to MachineOperand::isIdenticalTo()). | ||||||||
1139 | /// Note that this means liveness related flags (dead, undef, kill) do not | ||||||||
1140 | /// affect the notion of identical. | ||||||||
1141 | bool isIdenticalTo(const MachineInstr &Other, | ||||||||
1142 | MICheckType Check = CheckDefs) const; | ||||||||
1143 | |||||||||
1144 | /// Unlink 'this' from the containing basic block, and return it without | ||||||||
1145 | /// deleting it. | ||||||||
1146 | /// | ||||||||
1147 | /// This function can not be used on bundled instructions, use | ||||||||
1148 | /// removeFromBundle() to remove individual instructions from a bundle. | ||||||||
1149 | MachineInstr *removeFromParent(); | ||||||||
1150 | |||||||||
1151 | /// Unlink this instruction from its basic block and return it without | ||||||||
1152 | /// deleting it. | ||||||||
1153 | /// | ||||||||
1154 | /// If the instruction is part of a bundle, the other instructions in the | ||||||||
1155 | /// bundle remain bundled. | ||||||||
1156 | MachineInstr *removeFromBundle(); | ||||||||
1157 | |||||||||
1158 | /// Unlink 'this' from the containing basic block and delete it. | ||||||||
1159 | /// | ||||||||
1160 | /// If this instruction is the header of a bundle, the whole bundle is erased. | ||||||||
1161 | /// This function can not be used for instructions inside a bundle, use | ||||||||
1162 | /// eraseFromBundle() to erase individual bundled instructions. | ||||||||
1163 | void eraseFromParent(); | ||||||||
1164 | |||||||||
1165 | /// Unlink 'this' from the containing basic block and delete it. | ||||||||
1166 | /// | ||||||||
1167 | /// For all definitions mark their uses in DBG_VALUE nodes | ||||||||
1168 | /// as undefined. Otherwise like eraseFromParent(). | ||||||||
1169 | void eraseFromParentAndMarkDBGValuesForRemoval(); | ||||||||
1170 | |||||||||
1171 | /// Unlink 'this' form its basic block and delete it. | ||||||||
1172 | /// | ||||||||
1173 | /// If the instruction is part of a bundle, the other instructions in the | ||||||||
1174 | /// bundle remain bundled. | ||||||||
1175 | void eraseFromBundle(); | ||||||||
1176 | |||||||||
1177 | bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } | ||||||||
1178 | bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } | ||||||||
1179 | bool isAnnotationLabel() const { | ||||||||
1180 | return getOpcode() == TargetOpcode::ANNOTATION_LABEL; | ||||||||
1181 | } | ||||||||
1182 | |||||||||
1183 | /// Returns true if the MachineInstr represents a label. | ||||||||
1184 | bool isLabel() const { | ||||||||
1185 | return isEHLabel() || isGCLabel() || isAnnotationLabel(); | ||||||||
1186 | } | ||||||||
1187 | |||||||||
1188 | bool isCFIInstruction() const { | ||||||||
1189 | return getOpcode() == TargetOpcode::CFI_INSTRUCTION; | ||||||||
1190 | } | ||||||||
1191 | |||||||||
1192 | bool isPseudoProbe() const { | ||||||||
1193 | return getOpcode() == TargetOpcode::PSEUDO_PROBE; | ||||||||
1194 | } | ||||||||
1195 | |||||||||
1196 | // True if the instruction represents a position in the function. | ||||||||
1197 | bool isPosition() const { return isLabel() || isCFIInstruction(); } | ||||||||
1198 | |||||||||
1199 | bool isNonListDebugValue() const { | ||||||||
1200 | return getOpcode() == TargetOpcode::DBG_VALUE; | ||||||||
1201 | } | ||||||||
1202 | bool isDebugValueList() const { | ||||||||
1203 | return getOpcode() == TargetOpcode::DBG_VALUE_LIST; | ||||||||
1204 | } | ||||||||
1205 | bool isDebugValue() const { | ||||||||
1206 | return isNonListDebugValue() || isDebugValueList(); | ||||||||
1207 | } | ||||||||
1208 | bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; } | ||||||||
1209 | bool isDebugRef() const { return getOpcode() == TargetOpcode::DBG_INSTR_REF; } | ||||||||
1210 | bool isDebugInstr() const { | ||||||||
1211 | return isDebugValue() || isDebugLabel() || isDebugRef(); | ||||||||
1212 | } | ||||||||
1213 | bool isDebugOrPseudoInstr() const { | ||||||||
1214 | return isDebugInstr() || isPseudoProbe(); | ||||||||
1215 | } | ||||||||
1216 | |||||||||
1217 | bool isDebugOffsetImm() const { | ||||||||
1218 | return isNonListDebugValue() && getDebugOffset().isImm(); | ||||||||
1219 | } | ||||||||
1220 | |||||||||
1221 | /// A DBG_VALUE is indirect iff the location operand is a register and | ||||||||
1222 | /// the offset operand is an immediate. | ||||||||
1223 | bool isIndirectDebugValue() const { | ||||||||
1224 | return isDebugOffsetImm() && getDebugOperand(0).isReg(); | ||||||||
1225 | } | ||||||||
1226 | |||||||||
1227 | /// A DBG_VALUE is an entry value iff its debug expression contains the | ||||||||
1228 | /// DW_OP_LLVM_entry_value operation. | ||||||||
1229 | bool isDebugEntryValue() const; | ||||||||
1230 | |||||||||
1231 | /// Return true if the instruction is a debug value which describes a part of | ||||||||
1232 | /// a variable as unavailable. | ||||||||
1233 | bool isUndefDebugValue() const { | ||||||||
1234 | if (!isDebugValue()) | ||||||||
1235 | return false; | ||||||||
1236 | // If any $noreg locations are given, this DV is undef. | ||||||||
1237 | for (const MachineOperand &Op : debug_operands()) | ||||||||
1238 | if (Op.isReg() && !Op.getReg().isValid()) | ||||||||
1239 | return true; | ||||||||
1240 | return false; | ||||||||
1241 | } | ||||||||
1242 | |||||||||
1243 | bool isPHI() const { | ||||||||
1244 | return getOpcode() == TargetOpcode::PHI || | ||||||||
1245 | getOpcode() == TargetOpcode::G_PHI; | ||||||||
1246 | } | ||||||||
1247 | bool isKill() const { return getOpcode() == TargetOpcode::KILL; } | ||||||||
1248 | bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } | ||||||||
1249 | bool isInlineAsm() const { | ||||||||
1250 | return getOpcode() == TargetOpcode::INLINEASM || | ||||||||
1251 | getOpcode() == TargetOpcode::INLINEASM_BR; | ||||||||
1252 | } | ||||||||
1253 | |||||||||
1254 | /// FIXME: Seems like a layering violation that the AsmDialect, which is X86 | ||||||||
1255 | /// specific, be attached to a generic MachineInstr. | ||||||||
1256 | bool isMSInlineAsm() const { | ||||||||
1257 | return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; | ||||||||
1258 | } | ||||||||
1259 | |||||||||
1260 | bool isStackAligningInlineAsm() const; | ||||||||
1261 | InlineAsm::AsmDialect getInlineAsmDialect() const; | ||||||||
1262 | |||||||||
1263 | bool isInsertSubreg() const { | ||||||||
1264 | return getOpcode() == TargetOpcode::INSERT_SUBREG; | ||||||||
1265 | } | ||||||||
1266 | |||||||||
1267 | bool isSubregToReg() const { | ||||||||
1268 | return getOpcode() == TargetOpcode::SUBREG_TO_REG; | ||||||||
1269 | } | ||||||||
1270 | |||||||||
1271 | bool isRegSequence() const { | ||||||||
1272 | return getOpcode() == TargetOpcode::REG_SEQUENCE; | ||||||||
1273 | } | ||||||||
1274 | |||||||||
1275 | bool isBundle() const { | ||||||||
1276 | return getOpcode() == TargetOpcode::BUNDLE; | ||||||||
1277 | } | ||||||||
1278 | |||||||||
1279 | bool isCopy() const { | ||||||||
1280 | return getOpcode() == TargetOpcode::COPY; | ||||||||
1281 | } | ||||||||
1282 | |||||||||
1283 | bool isFullCopy() const { | ||||||||
1284 | return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg(); | ||||||||
1285 | } | ||||||||
1286 | |||||||||
1287 | bool isExtractSubreg() const { | ||||||||
1288 | return getOpcode() == TargetOpcode::EXTRACT_SUBREG; | ||||||||
1289 | } | ||||||||
1290 | |||||||||
1291 | /// Return true if the instruction behaves like a copy. | ||||||||
1292 | /// This does not include native copy instructions. | ||||||||
1293 | bool isCopyLike() const { | ||||||||
1294 | return isCopy() || isSubregToReg(); | ||||||||
1295 | } | ||||||||
1296 | |||||||||
1297 | /// Return true is the instruction is an identity copy. | ||||||||
1298 | bool isIdentityCopy() const { | ||||||||
1299 | return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() && | ||||||||
1300 | getOperand(0).getSubReg() == getOperand(1).getSubReg(); | ||||||||
1301 | } | ||||||||
1302 | |||||||||
1303 | /// Return true if this instruction doesn't produce any output in the form of | ||||||||
1304 | /// executable instructions. | ||||||||
1305 | bool isMetaInstruction() const { | ||||||||
1306 | switch (getOpcode()) { | ||||||||
1307 | default: | ||||||||
1308 | return false; | ||||||||
1309 | case TargetOpcode::IMPLICIT_DEF: | ||||||||
1310 | case TargetOpcode::KILL: | ||||||||
1311 | case TargetOpcode::CFI_INSTRUCTION: | ||||||||
1312 | case TargetOpcode::EH_LABEL: | ||||||||
1313 | case TargetOpcode::GC_LABEL: | ||||||||
1314 | case TargetOpcode::DBG_VALUE: | ||||||||
1315 | case TargetOpcode::DBG_VALUE_LIST: | ||||||||
1316 | case TargetOpcode::DBG_INSTR_REF: | ||||||||
1317 | case TargetOpcode::DBG_LABEL: | ||||||||
1318 | case TargetOpcode::LIFETIME_START: | ||||||||
1319 | case TargetOpcode::LIFETIME_END: | ||||||||
1320 | case TargetOpcode::PSEUDO_PROBE: | ||||||||
1321 | return true; | ||||||||
1322 | } | ||||||||
1323 | } | ||||||||
1324 | |||||||||
1325 | /// Return true if this is a transient instruction that is either very likely | ||||||||
1326 | /// to be eliminated during register allocation (such as copy-like | ||||||||
1327 | /// instructions), or if this instruction doesn't have an execution-time cost. | ||||||||
1328 | bool isTransient() const { | ||||||||
1329 | switch (getOpcode()) { | ||||||||
1330 | default: | ||||||||
1331 | return isMetaInstruction(); | ||||||||
1332 | // Copy-like instructions are usually eliminated during register allocation. | ||||||||
1333 | case TargetOpcode::PHI: | ||||||||
1334 | case TargetOpcode::G_PHI: | ||||||||
1335 | case TargetOpcode::COPY: | ||||||||
1336 | case TargetOpcode::INSERT_SUBREG: | ||||||||
1337 | case TargetOpcode::SUBREG_TO_REG: | ||||||||
1338 | case TargetOpcode::REG_SEQUENCE: | ||||||||
1339 | return true; | ||||||||
1340 | } | ||||||||
1341 | } | ||||||||
1342 | |||||||||
1343 | /// Return the number of instructions inside the MI bundle, excluding the | ||||||||
1344 | /// bundle header. | ||||||||
1345 | /// | ||||||||
1346 | /// This is the number of instructions that MachineBasicBlock::iterator | ||||||||
1347 | /// skips, 0 for unbundled instructions. | ||||||||
1348 | unsigned getBundleSize() const; | ||||||||
1349 | |||||||||
1350 | /// Return true if the MachineInstr reads the specified register. | ||||||||
1351 | /// If TargetRegisterInfo is passed, then it also checks if there | ||||||||
1352 | /// is a read of a super-register. | ||||||||
1353 | /// This does not count partial redefines of virtual registers as reads: | ||||||||
1354 | /// %reg1024:6 = OP. | ||||||||
1355 | bool readsRegister(Register Reg, | ||||||||
1356 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1357 | return findRegisterUseOperandIdx(Reg, false, TRI) != -1; | ||||||||
1358 | } | ||||||||
1359 | |||||||||
1360 | /// Return true if the MachineInstr reads the specified virtual register. | ||||||||
1361 | /// Take into account that a partial define is a | ||||||||
1362 | /// read-modify-write operation. | ||||||||
1363 | bool readsVirtualRegister(Register Reg) const { | ||||||||
1364 | return readsWritesVirtualRegister(Reg).first; | ||||||||
1365 | } | ||||||||
1366 | |||||||||
1367 | /// Return a pair of bools (reads, writes) indicating if this instruction | ||||||||
1368 | /// reads or writes Reg. This also considers partial defines. | ||||||||
1369 | /// If Ops is not null, all operand indices for Reg are added. | ||||||||
1370 | std::pair<bool,bool> readsWritesVirtualRegister(Register Reg, | ||||||||
1371 | SmallVectorImpl<unsigned> *Ops = nullptr) const; | ||||||||
1372 | |||||||||
1373 | /// Return true if the MachineInstr kills the specified register. | ||||||||
1374 | /// If TargetRegisterInfo is passed, then it also checks if there is | ||||||||
1375 | /// a kill of a super-register. | ||||||||
1376 | bool killsRegister(Register Reg, | ||||||||
1377 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1378 | return findRegisterUseOperandIdx(Reg, true, TRI) != -1; | ||||||||
1379 | } | ||||||||
1380 | |||||||||
1381 | /// Return true if the MachineInstr fully defines the specified register. | ||||||||
1382 | /// If TargetRegisterInfo is passed, then it also checks | ||||||||
1383 | /// if there is a def of a super-register. | ||||||||
1384 | /// NOTE: It's ignoring subreg indices on virtual registers. | ||||||||
1385 | bool definesRegister(Register Reg, | ||||||||
1386 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1387 | return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; | ||||||||
1388 | } | ||||||||
1389 | |||||||||
1390 | /// Return true if the MachineInstr modifies (fully define or partially | ||||||||
1391 | /// define) the specified register. | ||||||||
1392 | /// NOTE: It's ignoring subreg indices on virtual registers. | ||||||||
1393 | bool modifiesRegister(Register Reg, | ||||||||
1394 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1395 | return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; | ||||||||
1396 | } | ||||||||
1397 | |||||||||
1398 | /// Returns true if the register is dead in this machine instruction. | ||||||||
1399 | /// If TargetRegisterInfo is passed, then it also checks | ||||||||
1400 | /// if there is a dead def of a super-register. | ||||||||
1401 | bool registerDefIsDead(Register Reg, | ||||||||
1402 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1403 | return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; | ||||||||
1404 | } | ||||||||
1405 | |||||||||
1406 | /// Returns true if the MachineInstr has an implicit-use operand of exactly | ||||||||
1407 | /// the given register (not considering sub/super-registers). | ||||||||
1408 | bool hasRegisterImplicitUseOperand(Register Reg) const; | ||||||||
1409 | |||||||||
1410 | /// Returns the operand index that is a use of the specific register or -1 | ||||||||
1411 | /// if it is not found. It further tightens the search criteria to a use | ||||||||
1412 | /// that kills the register if isKill is true. | ||||||||
1413 | int findRegisterUseOperandIdx(Register Reg, bool isKill = false, | ||||||||
1414 | const TargetRegisterInfo *TRI = nullptr) const; | ||||||||
1415 | |||||||||
1416 | /// Wrapper for findRegisterUseOperandIdx, it returns | ||||||||
1417 | /// a pointer to the MachineOperand rather than an index. | ||||||||
1418 | MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false, | ||||||||
1419 | const TargetRegisterInfo *TRI = nullptr) { | ||||||||
1420 | int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI); | ||||||||
1421 | return (Idx == -1) ? nullptr : &getOperand(Idx); | ||||||||
1422 | } | ||||||||
1423 | |||||||||
1424 | const MachineOperand *findRegisterUseOperand( | ||||||||
1425 | Register Reg, bool isKill = false, | ||||||||
1426 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1427 | return const_cast<MachineInstr *>(this)-> | ||||||||
1428 | findRegisterUseOperand(Reg, isKill, TRI); | ||||||||
1429 | } | ||||||||
1430 | |||||||||
1431 | /// Returns the operand index that is a def of the specified register or | ||||||||
1432 | /// -1 if it is not found. If isDead is true, defs that are not dead are | ||||||||
1433 | /// skipped. If Overlap is true, then it also looks for defs that merely | ||||||||
1434 | /// overlap the specified register. If TargetRegisterInfo is non-null, | ||||||||
1435 | /// then it also checks if there is a def of a super-register. | ||||||||
1436 | /// This may also return a register mask operand when Overlap is true. | ||||||||
1437 | int findRegisterDefOperandIdx(Register Reg, | ||||||||
1438 | bool isDead = false, bool Overlap = false, | ||||||||
1439 | const TargetRegisterInfo *TRI = nullptr) const; | ||||||||
1440 | |||||||||
1441 | /// Wrapper for findRegisterDefOperandIdx, it returns | ||||||||
1442 | /// a pointer to the MachineOperand rather than an index. | ||||||||
1443 | MachineOperand * | ||||||||
1444 | findRegisterDefOperand(Register Reg, bool isDead = false, | ||||||||
1445 | bool Overlap = false, | ||||||||
1446 | const TargetRegisterInfo *TRI = nullptr) { | ||||||||
1447 | int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); | ||||||||
1448 | return (Idx == -1) ? nullptr : &getOperand(Idx); | ||||||||
1449 | } | ||||||||
1450 | |||||||||
1451 | const MachineOperand * | ||||||||
1452 | findRegisterDefOperand(Register Reg, bool isDead = false, | ||||||||
1453 | bool Overlap = false, | ||||||||
1454 | const TargetRegisterInfo *TRI = nullptr) const { | ||||||||
1455 | return const_cast<MachineInstr *>(this)->findRegisterDefOperand( | ||||||||
1456 | Reg, isDead, Overlap, TRI); | ||||||||
1457 | } | ||||||||
1458 | |||||||||
1459 | /// Find the index of the first operand in the | ||||||||
1460 | /// operand list that is used to represent the predicate. It returns -1 if | ||||||||
1461 | /// none is found. | ||||||||
1462 | int findFirstPredOperandIdx() const; | ||||||||
1463 | |||||||||
1464 | /// Find the index of the flag word operand that | ||||||||
1465 | /// corresponds to operand OpIdx on an inline asm instruction. Returns -1 if | ||||||||
1466 | /// getOperand(OpIdx) does not belong to an inline asm operand group. | ||||||||
1467 | /// | ||||||||
1468 | /// If GroupNo is not NULL, it will receive the number of the operand group | ||||||||
1469 | /// containing OpIdx. | ||||||||
1470 | /// | ||||||||
1471 | /// The flag operand is an immediate that can be decoded with methods like | ||||||||
1472 | /// InlineAsm::hasRegClassConstraint(). | ||||||||
1473 | int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const; | ||||||||
1474 | |||||||||
1475 | /// Compute the static register class constraint for operand OpIdx. | ||||||||
1476 | /// For normal instructions, this is derived from the MCInstrDesc. | ||||||||
1477 | /// For inline assembly it is derived from the flag words. | ||||||||
1478 | /// | ||||||||
1479 | /// Returns NULL if the static register class constraint cannot be | ||||||||
1480 | /// determined. | ||||||||
1481 | const TargetRegisterClass* | ||||||||
1482 | getRegClassConstraint(unsigned OpIdx, | ||||||||
1483 | const TargetInstrInfo *TII, | ||||||||
1484 | const TargetRegisterInfo *TRI) const; | ||||||||
1485 | |||||||||
1486 | /// Applies the constraints (def/use) implied by this MI on \p Reg to | ||||||||
1487 | /// the given \p CurRC. | ||||||||
1488 | /// If \p ExploreBundle is set and MI is part of a bundle, all the | ||||||||
1489 | /// instructions inside the bundle will be taken into account. In other words, | ||||||||
1490 | /// this method accumulates all the constraints of the operand of this MI and | ||||||||
1491 | /// the related bundle if MI is a bundle or inside a bundle. | ||||||||
1492 | /// | ||||||||
1493 | /// Returns the register class that satisfies both \p CurRC and the | ||||||||
1494 | /// constraints set by MI. Returns NULL if such a register class does not | ||||||||
1495 | /// exist. | ||||||||
1496 | /// | ||||||||
1497 | /// \pre CurRC must not be NULL. | ||||||||
1498 | const TargetRegisterClass *getRegClassConstraintEffectForVReg( | ||||||||
1499 | Register Reg, const TargetRegisterClass *CurRC, | ||||||||
1500 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, | ||||||||
1501 | bool ExploreBundle = false) const; | ||||||||
1502 | |||||||||
1503 | /// Applies the constraints (def/use) implied by the \p OpIdx operand | ||||||||
1504 | /// to the given \p CurRC. | ||||||||
1505 | /// | ||||||||
1506 | /// Returns the register class that satisfies both \p CurRC and the | ||||||||
1507 | /// constraints set by \p OpIdx MI. Returns NULL if such a register class | ||||||||
1508 | /// does not exist. | ||||||||
1509 | /// | ||||||||
1510 | /// \pre CurRC must not be NULL. | ||||||||
1511 | /// \pre The operand at \p OpIdx must be a register. | ||||||||
1512 | const TargetRegisterClass * | ||||||||
1513 | getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC, | ||||||||
1514 | const TargetInstrInfo *TII, | ||||||||
1515 | const TargetRegisterInfo *TRI) const; | ||||||||
1516 | |||||||||
1517 | /// Add a tie between the register operands at DefIdx and UseIdx. | ||||||||
1518 | /// The tie will cause the register allocator to ensure that the two | ||||||||
1519 | /// operands are assigned the same physical register. | ||||||||
1520 | /// | ||||||||
1521 | /// Tied operands are managed automatically for explicit operands in the | ||||||||
1522 | /// MCInstrDesc. This method is for exceptional cases like inline asm. | ||||||||
1523 | void tieOperands(unsigned DefIdx, unsigned UseIdx); | ||||||||
1524 | |||||||||
1525 | /// Given the index of a tied register operand, find the | ||||||||
1526 | /// operand it is tied to. Defs are tied to uses and vice versa. Returns the | ||||||||
1527 | /// index of the tied operand which must exist. | ||||||||
1528 | unsigned findTiedOperandIdx(unsigned OpIdx) const; | ||||||||
1529 | |||||||||
1530 | /// Given the index of a register def operand, | ||||||||
1531 | /// check if the register def is tied to a source operand, due to either | ||||||||
1532 | /// two-address elimination or inline assembly constraints. Returns the | ||||||||
1533 | /// first tied use operand index by reference if UseOpIdx is not null. | ||||||||
1534 | bool isRegTiedToUseOperand(unsigned DefOpIdx, | ||||||||
1535 | unsigned *UseOpIdx = nullptr) const { | ||||||||
1536 | const MachineOperand &MO = getOperand(DefOpIdx); | ||||||||
1537 | if (!MO.isReg() || !MO.isDef() || !MO.isTied()) | ||||||||
1538 | return false; | ||||||||
1539 | if (UseOpIdx) | ||||||||
1540 | *UseOpIdx = findTiedOperandIdx(DefOpIdx); | ||||||||
1541 | return true; | ||||||||
1542 | } | ||||||||
1543 | |||||||||
1544 | /// Return true if the use operand of the specified index is tied to a def | ||||||||
1545 | /// operand. It also returns the def operand index by reference if DefOpIdx | ||||||||
1546 | /// is not null. | ||||||||
1547 | bool isRegTiedToDefOperand(unsigned UseOpIdx, | ||||||||
1548 | unsigned *DefOpIdx = nullptr) const { | ||||||||
1549 | const MachineOperand &MO = getOperand(UseOpIdx); | ||||||||
1550 | if (!MO.isReg() || !MO.isUse() || !MO.isTied()) | ||||||||
1551 | return false; | ||||||||
1552 | if (DefOpIdx) | ||||||||
1553 | *DefOpIdx = findTiedOperandIdx(UseOpIdx); | ||||||||
1554 | return true; | ||||||||
1555 | } | ||||||||
1556 | |||||||||
1557 | /// Clears kill flags on all operands. | ||||||||
1558 | void clearKillInfo(); | ||||||||
1559 | |||||||||
1560 | /// Replace all occurrences of FromReg with ToReg:SubIdx, | ||||||||
1561 | /// properly composing subreg indices where necessary. | ||||||||
1562 | void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, | ||||||||
1563 | const TargetRegisterInfo &RegInfo); | ||||||||
1564 | |||||||||
1565 | /// We have determined MI kills a register. Look for the | ||||||||
1566 | /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, | ||||||||
1567 | /// add a implicit operand if it's not found. Returns true if the operand | ||||||||
1568 | /// exists / is added. | ||||||||
1569 | bool addRegisterKilled(Register IncomingReg, | ||||||||
1570 | const TargetRegisterInfo *RegInfo, | ||||||||
1571 | bool AddIfNotFound = false); | ||||||||
1572 | |||||||||
1573 | /// Clear all kill flags affecting Reg. If RegInfo is provided, this includes | ||||||||
1574 | /// all aliasing registers. | ||||||||
1575 | void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo); | ||||||||
1576 | |||||||||
1577 | /// We have determined MI defined a register without a use. | ||||||||
1578 | /// Look for the operand that defines it and mark it as IsDead. If | ||||||||
1579 | /// AddIfNotFound is true, add a implicit operand if it's not found. Returns | ||||||||
1580 | /// true if the operand exists / is added. | ||||||||
1581 | bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, | ||||||||
1582 | bool AddIfNotFound = false); | ||||||||
1583 | |||||||||
1584 | /// Clear all dead flags on operands defining register @p Reg. | ||||||||
1585 | void clearRegisterDeads(Register Reg); | ||||||||
1586 | |||||||||
1587 | /// Mark all subregister defs of register @p Reg with the undef flag. | ||||||||
1588 | /// This function is used when we determined to have a subregister def in an | ||||||||
1589 | /// otherwise undefined super register. | ||||||||
1590 | void setRegisterDefReadUndef(Register Reg, bool IsUndef = true); | ||||||||
1591 | |||||||||
1592 | /// We have determined MI defines a register. Make sure there is an operand | ||||||||
1593 | /// defining Reg. | ||||||||
1594 | void addRegisterDefined(Register Reg, | ||||||||
1595 | const TargetRegisterInfo *RegInfo = nullptr); | ||||||||
1596 | |||||||||
1597 | /// Mark every physreg used by this instruction as | ||||||||
1598 | /// dead except those in the UsedRegs list. | ||||||||
1599 | /// | ||||||||
1600 | /// On instructions with register mask operands, also add implicit-def | ||||||||
1601 | /// operands for all registers in UsedRegs. | ||||||||
1602 | void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs, | ||||||||
1603 | const TargetRegisterInfo &TRI); | ||||||||
1604 | |||||||||
1605 | /// Return true if it is safe to move this instruction. If | ||||||||
1606 | /// SawStore is set to true, it means that there is a store (or call) between | ||||||||
1607 | /// the instruction's location and its intended destination. | ||||||||
1608 | bool isSafeToMove(AAResults *AA, bool &SawStore) const; | ||||||||
1609 | |||||||||
1610 | /// Returns true if this instruction's memory access aliases the memory | ||||||||
1611 | /// access of Other. | ||||||||
1612 | // | ||||||||
1613 | /// Assumes any physical registers used to compute addresses | ||||||||
1614 | /// have the same value for both instructions. Returns false if neither | ||||||||
1615 | /// instruction writes to memory. | ||||||||
1616 | /// | ||||||||
1617 | /// @param AA Optional alias analysis, used to compare memory operands. | ||||||||
1618 | /// @param Other MachineInstr to check aliasing against. | ||||||||
1619 | /// @param UseTBAA Whether to pass TBAA information to alias analysis. | ||||||||
1620 | bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; | ||||||||
1621 | |||||||||
1622 | /// Return true if this instruction may have an ordered | ||||||||
1623 | /// or volatile memory reference, or if the information describing the memory | ||||||||
1624 | /// reference is not available. Return false if it is known to have no | ||||||||
1625 | /// ordered or volatile memory references. | ||||||||
1626 | bool hasOrderedMemoryRef() const; | ||||||||
1627 | |||||||||
1628 | /// Return true if this load instruction never traps and points to a memory | ||||||||
1629 | /// location whose value doesn't change during the execution of this function. | ||||||||
1630 | /// | ||||||||
1631 | /// Examples include loading a value from the constant pool or from the | ||||||||
1632 | /// argument area of a function (if it does not change). If the instruction | ||||||||
1633 | /// does multiple loads, this returns true only if all of the loads are | ||||||||
1634 | /// dereferenceable and invariant. | ||||||||
1635 | bool isDereferenceableInvariantLoad(AAResults *AA) const; | ||||||||
1636 | |||||||||
1637 | /// If the specified instruction is a PHI that always merges together the | ||||||||
1638 | /// same virtual register, return the register, otherwise return 0. | ||||||||
1639 | unsigned isConstantValuePHI() const; | ||||||||
1640 | |||||||||
1641 | /// Return true if this instruction has side effects that are not modeled | ||||||||
1642 | /// by mayLoad / mayStore, etc. | ||||||||
1643 | /// For all instructions, the property is encoded in MCInstrDesc::Flags | ||||||||
1644 | /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is | ||||||||
1645 | /// INLINEASM instruction, in which case the side effect property is encoded | ||||||||
1646 | /// in one of its operands (see InlineAsm::Extra_HasSideEffect). | ||||||||
1647 | /// | ||||||||
1648 | bool hasUnmodeledSideEffects() const; | ||||||||
1649 | |||||||||
1650 | /// Returns true if it is illegal to fold a load across this instruction. | ||||||||
1651 | bool isLoadFoldBarrier() const; | ||||||||
1652 | |||||||||
1653 | /// Return true if all the defs of this instruction are dead. | ||||||||
1654 | bool allDefsAreDead() const; | ||||||||
1655 | |||||||||
1656 | /// Return a valid size if the instruction is a spill instruction. | ||||||||
1657 | Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const; | ||||||||
1658 | |||||||||
1659 | /// Return a valid size if the instruction is a folded spill instruction. | ||||||||
1660 | Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const; | ||||||||
1661 | |||||||||
1662 | /// Return a valid size if the instruction is a restore instruction. | ||||||||
1663 | Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const; | ||||||||
1664 | |||||||||
1665 | /// Return a valid size if the instruction is a folded restore instruction. | ||||||||
1666 | Optional<unsigned> | ||||||||
1667 | getFoldedRestoreSize(const TargetInstrInfo *TII) const; | ||||||||
1668 | |||||||||
1669 | /// Copy implicit register operands from specified | ||||||||
1670 | /// instruction to this instruction. | ||||||||
1671 | void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI); | ||||||||
1672 | |||||||||
1673 | /// Debugging support | ||||||||
1674 | /// @{ | ||||||||
1675 | /// Determine the generic type to be printed (if needed) on uses and defs. | ||||||||
1676 | LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes, | ||||||||
1677 | const MachineRegisterInfo &MRI) const; | ||||||||
1678 | |||||||||
1679 | /// Return true when an instruction has tied register that can't be determined | ||||||||
1680 | /// by the instruction's descriptor. This is useful for MIR printing, to | ||||||||
1681 | /// determine whether we need to print the ties or not. | ||||||||
1682 | bool hasComplexRegisterTies() const; | ||||||||
1683 | |||||||||
1684 | /// Print this MI to \p OS. | ||||||||
1685 | /// Don't print information that can be inferred from other instructions if | ||||||||
1686 | /// \p IsStandalone is false. It is usually true when only a fragment of the | ||||||||
1687 | /// function is printed. | ||||||||
1688 | /// Only print the defs and the opcode if \p SkipOpers is true. | ||||||||
1689 | /// Otherwise, also print operands if \p SkipDebugLoc is true. | ||||||||
1690 | /// Otherwise, also print the debug loc, with a terminating newline. | ||||||||
1691 | /// \p TII is used to print the opcode name. If it's not present, but the | ||||||||
1692 | /// MI is in a function, the opcode will be printed using the function's TII. | ||||||||
1693 | void print(raw_ostream &OS, bool IsStandalone = true, bool SkipOpers = false, | ||||||||
1694 | bool SkipDebugLoc = false, bool AddNewLine = true, | ||||||||
1695 | const TargetInstrInfo *TII = nullptr) const; | ||||||||
1696 | void print(raw_ostream &OS, ModuleSlotTracker &MST, bool IsStandalone = true, | ||||||||
1697 | bool SkipOpers = false, bool SkipDebugLoc = false, | ||||||||
1698 | bool AddNewLine = true, | ||||||||
1699 | const TargetInstrInfo *TII = nullptr) const; | ||||||||
1700 | void dump() const; | ||||||||
1701 | /// Print on dbgs() the current instruction and the instructions defining its | ||||||||
1702 | /// operands and so on until we reach \p MaxDepth. | ||||||||
1703 | void dumpr(const MachineRegisterInfo &MRI, | ||||||||
1704 | unsigned MaxDepth = UINT_MAX(2147483647 *2U +1U)) const; | ||||||||
1705 | /// @} | ||||||||
1706 | |||||||||
1707 | //===--------------------------------------------------------------------===// | ||||||||
1708 | // Accessors used to build up machine instructions. | ||||||||
1709 | |||||||||
1710 | /// Add the specified operand to the instruction. If it is an implicit | ||||||||
1711 | /// operand, it is added to the end of the operand list. If it is an | ||||||||
1712 | /// explicit operand it is added at the end of the explicit operand list | ||||||||
1713 | /// (before the first implicit operand). | ||||||||
1714 | /// | ||||||||
1715 | /// MF must be the machine function that was used to allocate this | ||||||||
1716 | /// instruction. | ||||||||
1717 | /// | ||||||||
1718 | /// MachineInstrBuilder provides a more convenient interface for creating | ||||||||
1719 | /// instructions and adding operands. | ||||||||
1720 | void addOperand(MachineFunction &MF, const MachineOperand &Op); | ||||||||
1721 | |||||||||
1722 | /// Add an operand without providing an MF reference. This only works for | ||||||||
1723 | /// instructions that are inserted in a basic block. | ||||||||
1724 | /// | ||||||||
1725 | /// MachineInstrBuilder and the two-argument addOperand(MF, MO) should be | ||||||||
1726 | /// preferred. | ||||||||
1727 | void addOperand(const MachineOperand &Op); | ||||||||
1728 | |||||||||
1729 | /// Replace the instruction descriptor (thus opcode) of | ||||||||
1730 | /// the current instruction with a new one. | ||||||||
1731 | void setDesc(const MCInstrDesc &tid) { MCID = &tid; } | ||||||||
1732 | |||||||||
1733 | /// Replace current source information with new such. | ||||||||
1734 | /// Avoid using this, the constructor argument is preferable. | ||||||||
1735 | void setDebugLoc(DebugLoc dl) { | ||||||||
1736 | debugLoc = std::move(dl); | ||||||||
1737 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 1737, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1738 | } | ||||||||
1739 | |||||||||
1740 | /// Erase an operand from an instruction, leaving it with one | ||||||||
1741 | /// fewer operand than it started with. | ||||||||
1742 | void RemoveOperand(unsigned OpNo); | ||||||||
1743 | |||||||||
1744 | /// Clear this MachineInstr's memory reference descriptor list. This resets | ||||||||
1745 | /// the memrefs to their most conservative state. This should be used only | ||||||||
1746 | /// as a last resort since it greatly pessimizes our knowledge of the memory | ||||||||
1747 | /// access performed by the instruction. | ||||||||
1748 | void dropMemRefs(MachineFunction &MF); | ||||||||
1749 | |||||||||
1750 | /// Assign this MachineInstr's memory reference descriptor list. | ||||||||
1751 | /// | ||||||||
1752 | /// Unlike other methods, this *will* allocate them into a new array | ||||||||
1753 | /// associated with the provided `MachineFunction`. | ||||||||
1754 | void setMemRefs(MachineFunction &MF, ArrayRef<MachineMemOperand *> MemRefs); | ||||||||
1755 | |||||||||
1756 | /// Add a MachineMemOperand to the machine instruction. | ||||||||
1757 | /// This function should be used only occasionally. The setMemRefs function | ||||||||
1758 | /// is the primary method for setting up a MachineInstr's MemRefs list. | ||||||||
1759 | void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); | ||||||||
1760 | |||||||||
1761 | /// Clone another MachineInstr's memory reference descriptor list and replace | ||||||||
1762 | /// ours with it. | ||||||||
1763 | /// | ||||||||
1764 | /// Note that `*this` may be the incoming MI! | ||||||||
1765 | /// | ||||||||
1766 | /// Prefer this API whenever possible as it can avoid allocations in common | ||||||||
1767 | /// cases. | ||||||||
1768 | void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); | ||||||||
1769 | |||||||||
1770 | /// Clone the merge of multiple MachineInstrs' memory reference descriptors | ||||||||
1771 | /// list and replace ours with it. | ||||||||
1772 | /// | ||||||||
1773 | /// Note that `*this` may be one of the incoming MIs! | ||||||||
1774 | /// | ||||||||
1775 | /// Prefer this API whenever possible as it can avoid allocations in common | ||||||||
1776 | /// cases. | ||||||||
1777 | void cloneMergedMemRefs(MachineFunction &MF, | ||||||||
1778 | ArrayRef<const MachineInstr *> MIs); | ||||||||
1779 | |||||||||
1780 | /// Set a symbol that will be emitted just prior to the instruction itself. | ||||||||
1781 | /// | ||||||||
1782 | /// Setting this to a null pointer will remove any such symbol. | ||||||||
1783 | /// | ||||||||
1784 | /// FIXME: This is not fully implemented yet. | ||||||||
1785 | void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); | ||||||||
1786 | |||||||||
1787 | /// Set a symbol that will be emitted just after the instruction itself. | ||||||||
1788 | /// | ||||||||
1789 | /// Setting this to a null pointer will remove any such symbol. | ||||||||
1790 | /// | ||||||||
1791 | /// FIXME: This is not fully implemented yet. | ||||||||
1792 | void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); | ||||||||
1793 | |||||||||
1794 | /// Clone another MachineInstr's pre- and post- instruction symbols and | ||||||||
1795 | /// replace ours with it. | ||||||||
1796 | void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); | ||||||||
1797 | |||||||||
1798 | /// Set a marker on instructions that denotes where we should create and emit | ||||||||
1799 | /// heap alloc site labels. This waits until after instruction selection and | ||||||||
1800 | /// optimizations to create the label, so it should still work if the | ||||||||
1801 | /// instruction is removed or duplicated. | ||||||||
1802 | void setHeapAllocMarker(MachineFunction &MF, MDNode *MD); | ||||||||
1803 | |||||||||
1804 | /// Return the MIFlags which represent both MachineInstrs. This | ||||||||
1805 | /// should be used when merging two MachineInstrs into one. This routine does | ||||||||
1806 | /// not modify the MIFlags of this MachineInstr. | ||||||||
1807 | uint16_t mergeFlagsWith(const MachineInstr& Other) const; | ||||||||
1808 | |||||||||
1809 | static uint16_t copyFlagsFromInstruction(const Instruction &I); | ||||||||
1810 | |||||||||
1811 | /// Copy all flags to MachineInst MIFlags | ||||||||
1812 | void copyIRFlags(const Instruction &I); | ||||||||
1813 | |||||||||
1814 | /// Break any tie involving OpIdx. | ||||||||
1815 | void untieRegOperand(unsigned OpIdx) { | ||||||||
1816 | MachineOperand &MO = getOperand(OpIdx); | ||||||||
1817 | if (MO.isReg() && MO.isTied()) { | ||||||||
1818 | getOperand(findTiedOperandIdx(OpIdx)).TiedTo = 0; | ||||||||
1819 | MO.TiedTo = 0; | ||||||||
1820 | } | ||||||||
1821 | } | ||||||||
1822 | |||||||||
1823 | /// Add all implicit def and use operands to this instruction. | ||||||||
1824 | void addImplicitDefUseOperands(MachineFunction &MF); | ||||||||
1825 | |||||||||
1826 | /// Scan instructions immediately following MI and collect any matching | ||||||||
1827 | /// DBG_VALUEs. | ||||||||
1828 | void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); | ||||||||
1829 | |||||||||
1830 | /// Find all DBG_VALUEs that point to the register def in this instruction | ||||||||
1831 | /// and point them to \p Reg instead. | ||||||||
1832 | void changeDebugValuesDefReg(Register Reg); | ||||||||
1833 | |||||||||
1834 | /// Returns the Intrinsic::ID for this instruction. | ||||||||
1835 | /// \pre Must have an intrinsic ID operand. | ||||||||
1836 | unsigned getIntrinsicID() const { | ||||||||
1837 | return getOperand(getNumExplicitDefs()).getIntrinsicID(); | ||||||||
1838 | } | ||||||||
1839 | |||||||||
1840 | /// Sets all register debug operands in this debug value instruction to be | ||||||||
1841 | /// undef. | ||||||||
1842 | void setDebugValueUndef() { | ||||||||
1843 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 1843, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1844 | for (MachineOperand &MO : debug_operands()) { | ||||||||
1845 | if (MO.isReg()) { | ||||||||
1846 | MO.setReg(0); | ||||||||
1847 | MO.setSubReg(0); | ||||||||
1848 | } | ||||||||
1849 | } | ||||||||
1850 | } | ||||||||
1851 | |||||||||
1852 | PseudoProbeAttributes getPseudoProbeAttribute() const { | ||||||||
1853 | assert(isPseudoProbe() && "Must be a pseudo probe instruction")(static_cast <bool> (isPseudoProbe() && "Must be a pseudo probe instruction" ) ? void (0) : __assert_fail ("isPseudoProbe() && \"Must be a pseudo probe instruction\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 1853, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1854 | return (PseudoProbeAttributes)getOperand(3).getImm(); | ||||||||
1855 | } | ||||||||
1856 | |||||||||
1857 | void addPseudoProbeAttribute(PseudoProbeAttributes Attr) { | ||||||||
1858 | assert(isPseudoProbe() && "Must be a pseudo probe instruction")(static_cast <bool> (isPseudoProbe() && "Must be a pseudo probe instruction" ) ? void (0) : __assert_fail ("isPseudoProbe() && \"Must be a pseudo probe instruction\"" , "/build/llvm-toolchain-snapshot-13~++20210506100649+6304c0836a4d/llvm/include/llvm/CodeGen/MachineInstr.h" , 1858, __extension__ __PRETTY_FUNCTION__)); | ||||||||
1859 | MachineOperand &AttrOperand = getOperand(3); | ||||||||
1860 | AttrOperand.setImm(AttrOperand.getImm() | (uint32_t)Attr); | ||||||||
1861 | } | ||||||||
1862 | |||||||||
1863 | private: | ||||||||
1864 | /// If this instruction is embedded into a MachineFunction, return the | ||||||||
1865 | /// MachineRegisterInfo object for the current function, otherwise | ||||||||
1866 | /// return null. | ||||||||
1867 | MachineRegisterInfo *getRegInfo(); | ||||||||
1868 | |||||||||
1869 | /// Unlink all of the register operands in this instruction from their | ||||||||
1870 | /// respective use lists. This requires that the operands already be on their | ||||||||
1871 | /// use lists. | ||||||||
1872 | void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); | ||||||||
1873 | |||||||||
1874 | /// Add all of the register operands in this instruction from their | ||||||||
1875 | /// respective use lists. This requires that the operands not be on their | ||||||||
1876 | /// use lists yet. | ||||||||
1877 | void AddRegOperandsToUseLists(MachineRegisterInfo&); | ||||||||
1878 | |||||||||
1879 | /// Slow path for hasProperty when we're dealing with a bundle. | ||||||||
1880 | bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; | ||||||||
1881 | |||||||||
1882 | /// Implements the logic of getRegClassConstraintEffectForVReg for the | ||||||||
1883 | /// this MI and the given operand index \p OpIdx. | ||||||||
1884 | /// If the related operand does not constrained Reg, this returns CurRC. | ||||||||
1885 | const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( | ||||||||
1886 | unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, | ||||||||
1887 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; | ||||||||
1888 | |||||||||
1889 | /// Stores extra instruction information inline or allocates as ExtraInfo | ||||||||
1890 | /// based on the number of pointers. | ||||||||
1891 | void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs, | ||||||||
1892 | MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, | ||||||||
1893 | MDNode *HeapAllocMarker); | ||||||||
1894 | }; | ||||||||
1895 | |||||||||
1896 | /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the | ||||||||
1897 | /// instruction rather than by pointer value. | ||||||||
1898 | /// The hashing and equality testing functions ignore definitions so this is | ||||||||
1899 | /// useful for CSE, etc. | ||||||||
1900 | struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> { | ||||||||
1901 | static inline MachineInstr *getEmptyKey() { | ||||||||
1902 | return nullptr; | ||||||||
1903 | } | ||||||||
1904 | |||||||||
1905 | static inline MachineInstr *getTombstoneKey() { | ||||||||
1906 | return reinterpret_cast<MachineInstr*>(-1); | ||||||||
1907 | } | ||||||||
1908 | |||||||||
1909 | static unsigned getHashValue(const MachineInstr* const &MI); | ||||||||
1910 | |||||||||
1911 | static bool isEqual(const MachineInstr* const &LHS, | ||||||||
1912 | const MachineInstr* const &RHS) { | ||||||||
1913 | if (RHS == getEmptyKey() || RHS == getTombstoneKey() || | ||||||||
1914 | LHS == getEmptyKey() || LHS == getTombstoneKey()) | ||||||||
1915 | return LHS == RHS; | ||||||||
1916 | return LHS->isIdenticalTo(*RHS, MachineInstr::IgnoreVRegDefs); | ||||||||
1917 | } | ||||||||
1918 | }; | ||||||||
1919 | |||||||||
1920 | //===----------------------------------------------------------------------===// | ||||||||
1921 | // Debugging Support | ||||||||
1922 | |||||||||
1923 | inline raw_ostream& operator<<(raw_ostream &OS, const MachineInstr &MI) { | ||||||||
1924 | MI.print(OS); | ||||||||
1925 | return OS; | ||||||||
1926 | } | ||||||||
1927 | |||||||||
1928 | } // end namespace llvm | ||||||||
1929 | |||||||||
1930 | #endif // LLVM_CODEGEN_MACHINEINSTR_H |