File: | llvm/lib/CodeGen/InlineSpiller.cpp |
Warning: | line 491, column 55 The left operand of '==' is a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- InlineSpiller.cpp - Insert spills and restores inline --------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // The inline spiller modifies the machine function directly instead of | |||
10 | // inserting spills and restores in VirtRegMap. | |||
11 | // | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "SplitKit.h" | |||
15 | #include "llvm/ADT/ArrayRef.h" | |||
16 | #include "llvm/ADT/DenseMap.h" | |||
17 | #include "llvm/ADT/MapVector.h" | |||
18 | #include "llvm/ADT/None.h" | |||
19 | #include "llvm/ADT/STLExtras.h" | |||
20 | #include "llvm/ADT/SetVector.h" | |||
21 | #include "llvm/ADT/SmallPtrSet.h" | |||
22 | #include "llvm/ADT/SmallVector.h" | |||
23 | #include "llvm/ADT/Statistic.h" | |||
24 | #include "llvm/Analysis/AliasAnalysis.h" | |||
25 | #include "llvm/CodeGen/LiveInterval.h" | |||
26 | #include "llvm/CodeGen/LiveIntervalCalc.h" | |||
27 | #include "llvm/CodeGen/LiveIntervals.h" | |||
28 | #include "llvm/CodeGen/LiveRangeEdit.h" | |||
29 | #include "llvm/CodeGen/LiveStacks.h" | |||
30 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
31 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" | |||
32 | #include "llvm/CodeGen/MachineDominators.h" | |||
33 | #include "llvm/CodeGen/MachineFunction.h" | |||
34 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
35 | #include "llvm/CodeGen/MachineInstr.h" | |||
36 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
37 | #include "llvm/CodeGen/MachineInstrBundle.h" | |||
38 | #include "llvm/CodeGen/MachineLoopInfo.h" | |||
39 | #include "llvm/CodeGen/MachineOperand.h" | |||
40 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
41 | #include "llvm/CodeGen/SlotIndexes.h" | |||
42 | #include "llvm/CodeGen/Spiller.h" | |||
43 | #include "llvm/CodeGen/StackMaps.h" | |||
44 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
45 | #include "llvm/CodeGen/TargetOpcodes.h" | |||
46 | #include "llvm/CodeGen/TargetRegisterInfo.h" | |||
47 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
48 | #include "llvm/CodeGen/VirtRegMap.h" | |||
49 | #include "llvm/Config/llvm-config.h" | |||
50 | #include "llvm/Support/BlockFrequency.h" | |||
51 | #include "llvm/Support/BranchProbability.h" | |||
52 | #include "llvm/Support/CommandLine.h" | |||
53 | #include "llvm/Support/Compiler.h" | |||
54 | #include "llvm/Support/Debug.h" | |||
55 | #include "llvm/Support/ErrorHandling.h" | |||
56 | #include "llvm/Support/raw_ostream.h" | |||
57 | #include <cassert> | |||
58 | #include <iterator> | |||
59 | #include <tuple> | |||
60 | #include <utility> | |||
61 | #include <vector> | |||
62 | ||||
63 | using namespace llvm; | |||
64 | ||||
65 | #define DEBUG_TYPE"regalloc" "regalloc" | |||
66 | ||||
67 | STATISTIC(NumSpilledRanges, "Number of spilled live ranges")static llvm::Statistic NumSpilledRanges = {"regalloc", "NumSpilledRanges" , "Number of spilled live ranges"}; | |||
68 | STATISTIC(NumSnippets, "Number of spilled snippets")static llvm::Statistic NumSnippets = {"regalloc", "NumSnippets" , "Number of spilled snippets"}; | |||
69 | STATISTIC(NumSpills, "Number of spills inserted")static llvm::Statistic NumSpills = {"regalloc", "NumSpills", "Number of spills inserted" }; | |||
70 | STATISTIC(NumSpillsRemoved, "Number of spills removed")static llvm::Statistic NumSpillsRemoved = {"regalloc", "NumSpillsRemoved" , "Number of spills removed"}; | |||
71 | STATISTIC(NumReloads, "Number of reloads inserted")static llvm::Statistic NumReloads = {"regalloc", "NumReloads" , "Number of reloads inserted"}; | |||
72 | STATISTIC(NumReloadsRemoved, "Number of reloads removed")static llvm::Statistic NumReloadsRemoved = {"regalloc", "NumReloadsRemoved" , "Number of reloads removed"}; | |||
73 | STATISTIC(NumFolded, "Number of folded stack accesses")static llvm::Statistic NumFolded = {"regalloc", "NumFolded", "Number of folded stack accesses" }; | |||
74 | STATISTIC(NumFoldedLoads, "Number of folded loads")static llvm::Statistic NumFoldedLoads = {"regalloc", "NumFoldedLoads" , "Number of folded loads"}; | |||
75 | STATISTIC(NumRemats, "Number of rematerialized defs for spilling")static llvm::Statistic NumRemats = {"regalloc", "NumRemats", "Number of rematerialized defs for spilling" }; | |||
76 | ||||
77 | static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden, | |||
78 | cl::desc("Disable inline spill hoisting")); | |||
79 | static cl::opt<bool> | |||
80 | RestrictStatepointRemat("restrict-statepoint-remat", | |||
81 | cl::init(false), cl::Hidden, | |||
82 | cl::desc("Restrict remat for statepoint operands")); | |||
83 | ||||
84 | namespace { | |||
85 | ||||
86 | class HoistSpillHelper : private LiveRangeEdit::Delegate { | |||
87 | MachineFunction &MF; | |||
88 | LiveIntervals &LIS; | |||
89 | LiveStacks &LSS; | |||
90 | AliasAnalysis *AA; | |||
91 | MachineDominatorTree &MDT; | |||
92 | MachineLoopInfo &Loops; | |||
93 | VirtRegMap &VRM; | |||
94 | MachineRegisterInfo &MRI; | |||
95 | const TargetInstrInfo &TII; | |||
96 | const TargetRegisterInfo &TRI; | |||
97 | const MachineBlockFrequencyInfo &MBFI; | |||
98 | ||||
99 | InsertPointAnalysis IPA; | |||
100 | ||||
101 | // Map from StackSlot to the LiveInterval of the original register. | |||
102 | // Note the LiveInterval of the original register may have been deleted | |||
103 | // after it is spilled. We keep a copy here to track the range where | |||
104 | // spills can be moved. | |||
105 | DenseMap<int, std::unique_ptr<LiveInterval>> StackSlotToOrigLI; | |||
106 | ||||
107 | // Map from pair of (StackSlot and Original VNI) to a set of spills which | |||
108 | // have the same stackslot and have equal values defined by Original VNI. | |||
109 | // These spills are mergeable and are hoist candiates. | |||
110 | using MergeableSpillsMap = | |||
111 | MapVector<std::pair<int, VNInfo *>, SmallPtrSet<MachineInstr *, 16>>; | |||
112 | MergeableSpillsMap MergeableSpills; | |||
113 | ||||
114 | /// This is the map from original register to a set containing all its | |||
115 | /// siblings. To hoist a spill to another BB, we need to find out a live | |||
116 | /// sibling there and use it as the source of the new spill. | |||
117 | DenseMap<Register, SmallSetVector<Register, 16>> Virt2SiblingsMap; | |||
118 | ||||
119 | bool isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
120 | MachineBasicBlock &BB, Register &LiveReg); | |||
121 | ||||
122 | void rmRedundantSpills( | |||
123 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
124 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
125 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill); | |||
126 | ||||
127 | void getVisitOrders( | |||
128 | MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills, | |||
129 | SmallVectorImpl<MachineDomTreeNode *> &Orders, | |||
130 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
131 | DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep, | |||
132 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill); | |||
133 | ||||
134 | void runHoistSpills(LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
135 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
136 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
137 | DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns); | |||
138 | ||||
139 | public: | |||
140 | HoistSpillHelper(MachineFunctionPass &pass, MachineFunction &mf, | |||
141 | VirtRegMap &vrm) | |||
142 | : MF(mf), LIS(pass.getAnalysis<LiveIntervals>()), | |||
143 | LSS(pass.getAnalysis<LiveStacks>()), | |||
144 | AA(&pass.getAnalysis<AAResultsWrapperPass>().getAAResults()), | |||
145 | MDT(pass.getAnalysis<MachineDominatorTree>()), | |||
146 | Loops(pass.getAnalysis<MachineLoopInfo>()), VRM(vrm), | |||
147 | MRI(mf.getRegInfo()), TII(*mf.getSubtarget().getInstrInfo()), | |||
148 | TRI(*mf.getSubtarget().getRegisterInfo()), | |||
149 | MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()), | |||
150 | IPA(LIS, mf.getNumBlockIDs()) {} | |||
151 | ||||
152 | void addToMergeableSpills(MachineInstr &Spill, int StackSlot, | |||
153 | unsigned Original); | |||
154 | bool rmFromMergeableSpills(MachineInstr &Spill, int StackSlot); | |||
155 | void hoistAllSpills(); | |||
156 | void LRE_DidCloneVirtReg(Register, Register) override; | |||
157 | }; | |||
158 | ||||
159 | class InlineSpiller : public Spiller { | |||
160 | MachineFunction &MF; | |||
161 | LiveIntervals &LIS; | |||
162 | LiveStacks &LSS; | |||
163 | AliasAnalysis *AA; | |||
164 | MachineDominatorTree &MDT; | |||
165 | MachineLoopInfo &Loops; | |||
166 | VirtRegMap &VRM; | |||
167 | MachineRegisterInfo &MRI; | |||
168 | const TargetInstrInfo &TII; | |||
169 | const TargetRegisterInfo &TRI; | |||
170 | const MachineBlockFrequencyInfo &MBFI; | |||
171 | ||||
172 | // Variables that are valid during spill(), but used by multiple methods. | |||
173 | LiveRangeEdit *Edit; | |||
174 | LiveInterval *StackInt; | |||
175 | int StackSlot; | |||
176 | Register Original; | |||
177 | ||||
178 | // All registers to spill to StackSlot, including the main register. | |||
179 | SmallVector<Register, 8> RegsToSpill; | |||
180 | ||||
181 | // All COPY instructions to/from snippets. | |||
182 | // They are ignored since both operands refer to the same stack slot. | |||
183 | SmallPtrSet<MachineInstr*, 8> SnippetCopies; | |||
184 | ||||
185 | // Values that failed to remat at some point. | |||
186 | SmallPtrSet<VNInfo*, 8> UsedValues; | |||
187 | ||||
188 | // Dead defs generated during spilling. | |||
189 | SmallVector<MachineInstr*, 8> DeadDefs; | |||
190 | ||||
191 | // Object records spills information and does the hoisting. | |||
192 | HoistSpillHelper HSpiller; | |||
193 | ||||
194 | // Live range weight calculator. | |||
195 | VirtRegAuxInfo &VRAI; | |||
196 | ||||
197 | ~InlineSpiller() override = default; | |||
198 | ||||
199 | public: | |||
200 | InlineSpiller(MachineFunctionPass &Pass, MachineFunction &MF, VirtRegMap &VRM, | |||
201 | VirtRegAuxInfo &VRAI) | |||
202 | : MF(MF), LIS(Pass.getAnalysis<LiveIntervals>()), | |||
203 | LSS(Pass.getAnalysis<LiveStacks>()), | |||
204 | AA(&Pass.getAnalysis<AAResultsWrapperPass>().getAAResults()), | |||
205 | MDT(Pass.getAnalysis<MachineDominatorTree>()), | |||
206 | Loops(Pass.getAnalysis<MachineLoopInfo>()), VRM(VRM), | |||
207 | MRI(MF.getRegInfo()), TII(*MF.getSubtarget().getInstrInfo()), | |||
208 | TRI(*MF.getSubtarget().getRegisterInfo()), | |||
209 | MBFI(Pass.getAnalysis<MachineBlockFrequencyInfo>()), | |||
210 | HSpiller(Pass, MF, VRM), VRAI(VRAI) {} | |||
211 | ||||
212 | void spill(LiveRangeEdit &) override; | |||
213 | void postOptimization() override; | |||
214 | ||||
215 | private: | |||
216 | bool isSnippet(const LiveInterval &SnipLI); | |||
217 | void collectRegsToSpill(); | |||
218 | ||||
219 | bool isRegToSpill(Register Reg) { return is_contained(RegsToSpill, Reg); } | |||
220 | ||||
221 | bool isSibling(Register Reg); | |||
222 | bool hoistSpillInsideBB(LiveInterval &SpillLI, MachineInstr &CopyMI); | |||
223 | void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI); | |||
224 | ||||
225 | void markValueUsed(LiveInterval*, VNInfo*); | |||
226 | bool canGuaranteeAssignmentAfterRemat(Register VReg, MachineInstr &MI); | |||
227 | bool reMaterializeFor(LiveInterval &, MachineInstr &MI); | |||
228 | void reMaterializeAll(); | |||
229 | ||||
230 | bool coalesceStackAccess(MachineInstr *MI, Register Reg); | |||
231 | bool foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>>, | |||
232 | MachineInstr *LoadMI = nullptr); | |||
233 | void insertReload(Register VReg, SlotIndex, MachineBasicBlock::iterator MI); | |||
234 | void insertSpill(Register VReg, bool isKill, MachineBasicBlock::iterator MI); | |||
235 | ||||
236 | void spillAroundUses(Register Reg); | |||
237 | void spillAll(); | |||
238 | }; | |||
239 | ||||
240 | } // end anonymous namespace | |||
241 | ||||
242 | Spiller::~Spiller() = default; | |||
243 | ||||
244 | void Spiller::anchor() {} | |||
245 | ||||
246 | Spiller *llvm::createInlineSpiller(MachineFunctionPass &Pass, | |||
247 | MachineFunction &MF, VirtRegMap &VRM, | |||
248 | VirtRegAuxInfo &VRAI) { | |||
249 | return new InlineSpiller(Pass, MF, VRM, VRAI); | |||
250 | } | |||
251 | ||||
252 | //===----------------------------------------------------------------------===// | |||
253 | // Snippets | |||
254 | //===----------------------------------------------------------------------===// | |||
255 | ||||
256 | // When spilling a virtual register, we also spill any snippets it is connected | |||
257 | // to. The snippets are small live ranges that only have a single real use, | |||
258 | // leftovers from live range splitting. Spilling them enables memory operand | |||
259 | // folding or tightens the live range around the single use. | |||
260 | // | |||
261 | // This minimizes register pressure and maximizes the store-to-load distance for | |||
262 | // spill slots which can be important in tight loops. | |||
263 | ||||
264 | /// isFullCopyOf - If MI is a COPY to or from Reg, return the other register, | |||
265 | /// otherwise return 0. | |||
266 | static Register isFullCopyOf(const MachineInstr &MI, Register Reg) { | |||
267 | if (!MI.isFullCopy()) | |||
268 | return Register(); | |||
269 | if (MI.getOperand(0).getReg() == Reg) | |||
270 | return MI.getOperand(1).getReg(); | |||
271 | if (MI.getOperand(1).getReg() == Reg) | |||
272 | return MI.getOperand(0).getReg(); | |||
273 | return Register(); | |||
274 | } | |||
275 | ||||
276 | static void getVDefInterval(const MachineInstr &MI, LiveIntervals &LIS) { | |||
277 | for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { | |||
278 | const MachineOperand &MO = MI.getOperand(I); | |||
279 | if (MO.isReg() && MO.isDef() && Register::isVirtualRegister(MO.getReg())) | |||
280 | LIS.getInterval(MO.getReg()); | |||
281 | } | |||
282 | } | |||
283 | ||||
284 | /// isSnippet - Identify if a live interval is a snippet that should be spilled. | |||
285 | /// It is assumed that SnipLI is a virtual register with the same original as | |||
286 | /// Edit->getReg(). | |||
287 | bool InlineSpiller::isSnippet(const LiveInterval &SnipLI) { | |||
288 | Register Reg = Edit->getReg(); | |||
289 | ||||
290 | // A snippet is a tiny live range with only a single instruction using it | |||
291 | // besides copies to/from Reg or spills/fills. We accept: | |||
292 | // | |||
293 | // %snip = COPY %Reg / FILL fi# | |||
294 | // %snip = USE %snip | |||
295 | // %Reg = COPY %snip / SPILL %snip, fi# | |||
296 | // | |||
297 | if (SnipLI.getNumValNums() > 2 || !LIS.intervalIsInOneMBB(SnipLI)) | |||
298 | return false; | |||
299 | ||||
300 | MachineInstr *UseMI = nullptr; | |||
301 | ||||
302 | // Check that all uses satisfy our criteria. | |||
303 | for (MachineRegisterInfo::reg_instr_nodbg_iterator | |||
304 | RI = MRI.reg_instr_nodbg_begin(SnipLI.reg()), | |||
305 | E = MRI.reg_instr_nodbg_end(); | |||
306 | RI != E;) { | |||
307 | MachineInstr &MI = *RI++; | |||
308 | ||||
309 | // Allow copies to/from Reg. | |||
310 | if (isFullCopyOf(MI, Reg)) | |||
311 | continue; | |||
312 | ||||
313 | // Allow stack slot loads. | |||
314 | int FI; | |||
315 | if (SnipLI.reg() == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) | |||
316 | continue; | |||
317 | ||||
318 | // Allow stack slot stores. | |||
319 | if (SnipLI.reg() == TII.isStoreToStackSlot(MI, FI) && FI == StackSlot) | |||
320 | continue; | |||
321 | ||||
322 | // Allow a single additional instruction. | |||
323 | if (UseMI && &MI != UseMI) | |||
324 | return false; | |||
325 | UseMI = &MI; | |||
326 | } | |||
327 | return true; | |||
328 | } | |||
329 | ||||
330 | /// collectRegsToSpill - Collect live range snippets that only have a single | |||
331 | /// real use. | |||
332 | void InlineSpiller::collectRegsToSpill() { | |||
333 | Register Reg = Edit->getReg(); | |||
334 | ||||
335 | // Main register always spills. | |||
336 | RegsToSpill.assign(1, Reg); | |||
337 | SnippetCopies.clear(); | |||
338 | ||||
339 | // Snippets all have the same original, so there can't be any for an original | |||
340 | // register. | |||
341 | if (Original == Reg) | |||
342 | return; | |||
343 | ||||
344 | for (MachineInstr &MI : | |||
345 | llvm::make_early_inc_range(MRI.reg_instructions(Reg))) { | |||
346 | Register SnipReg = isFullCopyOf(MI, Reg); | |||
347 | if (!isSibling(SnipReg)) | |||
348 | continue; | |||
349 | LiveInterval &SnipLI = LIS.getInterval(SnipReg); | |||
350 | if (!isSnippet(SnipLI)) | |||
351 | continue; | |||
352 | SnippetCopies.insert(&MI); | |||
353 | if (isRegToSpill(SnipReg)) | |||
354 | continue; | |||
355 | RegsToSpill.push_back(SnipReg); | |||
356 | LLVM_DEBUG(dbgs() << "\talso spill snippet " << SnipLI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\talso spill snippet " << SnipLI << '\n'; } } while (false); | |||
357 | ++NumSnippets; | |||
358 | } | |||
359 | } | |||
360 | ||||
361 | bool InlineSpiller::isSibling(Register Reg) { | |||
362 | return Reg.isVirtual() && VRM.getOriginal(Reg) == Original; | |||
363 | } | |||
364 | ||||
365 | /// It is beneficial to spill to earlier place in the same BB in case | |||
366 | /// as follows: | |||
367 | /// There is an alternative def earlier in the same MBB. | |||
368 | /// Hoist the spill as far as possible in SpillMBB. This can ease | |||
369 | /// register pressure: | |||
370 | /// | |||
371 | /// x = def | |||
372 | /// y = use x | |||
373 | /// s = copy x | |||
374 | /// | |||
375 | /// Hoisting the spill of s to immediately after the def removes the | |||
376 | /// interference between x and y: | |||
377 | /// | |||
378 | /// x = def | |||
379 | /// spill x | |||
380 | /// y = use killed x | |||
381 | /// | |||
382 | /// This hoist only helps when the copy kills its source. | |||
383 | /// | |||
384 | bool InlineSpiller::hoistSpillInsideBB(LiveInterval &SpillLI, | |||
385 | MachineInstr &CopyMI) { | |||
386 | SlotIndex Idx = LIS.getInstructionIndex(CopyMI); | |||
387 | #ifndef NDEBUG | |||
388 | VNInfo *VNI = SpillLI.getVNInfoAt(Idx.getRegSlot()); | |||
389 | assert(VNI && VNI->def == Idx.getRegSlot() && "Not defined by copy")(static_cast <bool> (VNI && VNI->def == Idx. getRegSlot() && "Not defined by copy") ? void (0) : __assert_fail ("VNI && VNI->def == Idx.getRegSlot() && \"Not defined by copy\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 389, __extension__ __PRETTY_FUNCTION__)); | |||
390 | #endif | |||
391 | ||||
392 | Register SrcReg = CopyMI.getOperand(1).getReg(); | |||
393 | LiveInterval &SrcLI = LIS.getInterval(SrcReg); | |||
394 | VNInfo *SrcVNI = SrcLI.getVNInfoAt(Idx); | |||
395 | LiveQueryResult SrcQ = SrcLI.Query(Idx); | |||
396 | MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(SrcVNI->def); | |||
397 | if (DefMBB != CopyMI.getParent() || !SrcQ.isKill()) | |||
398 | return false; | |||
399 | ||||
400 | // Conservatively extend the stack slot range to the range of the original | |||
401 | // value. We may be able to do better with stack slot coloring by being more | |||
402 | // careful here. | |||
403 | assert(StackInt && "No stack slot assigned yet.")(static_cast <bool> (StackInt && "No stack slot assigned yet." ) ? void (0) : __assert_fail ("StackInt && \"No stack slot assigned yet.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 403, __extension__ __PRETTY_FUNCTION__)); | |||
404 | LiveInterval &OrigLI = LIS.getInterval(Original); | |||
405 | VNInfo *OrigVNI = OrigLI.getVNInfoAt(Idx); | |||
406 | StackInt->MergeValueInAsValue(OrigLI, OrigVNI, StackInt->getValNumInfo(0)); | |||
407 | LLVM_DEBUG(dbgs() << "\tmerged orig valno " << OrigVNI->id << ": "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tmerged orig valno " << OrigVNI->id << ": " << *StackInt << '\n' ; } } while (false) | |||
408 | << *StackInt << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tmerged orig valno " << OrigVNI->id << ": " << *StackInt << '\n' ; } } while (false); | |||
409 | ||||
410 | // We are going to spill SrcVNI immediately after its def, so clear out | |||
411 | // any later spills of the same value. | |||
412 | eliminateRedundantSpills(SrcLI, SrcVNI); | |||
413 | ||||
414 | MachineBasicBlock *MBB = LIS.getMBBFromIndex(SrcVNI->def); | |||
415 | MachineBasicBlock::iterator MII; | |||
416 | if (SrcVNI->isPHIDef()) | |||
417 | MII = MBB->SkipPHIsLabelsAndDebug(MBB->begin()); | |||
418 | else { | |||
419 | MachineInstr *DefMI = LIS.getInstructionFromIndex(SrcVNI->def); | |||
420 | assert(DefMI && "Defining instruction disappeared")(static_cast <bool> (DefMI && "Defining instruction disappeared" ) ? void (0) : __assert_fail ("DefMI && \"Defining instruction disappeared\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 420, __extension__ __PRETTY_FUNCTION__)); | |||
421 | MII = DefMI; | |||
422 | ++MII; | |||
423 | } | |||
424 | MachineInstrSpan MIS(MII, MBB); | |||
425 | // Insert spill without kill flag immediately after def. | |||
426 | TII.storeRegToStackSlot(*MBB, MII, SrcReg, false, StackSlot, | |||
427 | MRI.getRegClass(SrcReg), &TRI); | |||
428 | LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MII); | |||
429 | for (const MachineInstr &MI : make_range(MIS.begin(), MII)) | |||
430 | getVDefInterval(MI, LIS); | |||
431 | --MII; // Point to store instruction. | |||
432 | LLVM_DEBUG(dbgs() << "\thoisted: " << SrcVNI->def << '\t' << *MII)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\thoisted: " << SrcVNI ->def << '\t' << *MII; } } while (false); | |||
433 | ||||
434 | // If there is only 1 store instruction is required for spill, add it | |||
435 | // to mergeable list. In X86 AMX, 2 intructions are required to store. | |||
436 | // We disable the merge for this case. | |||
437 | if (MIS.begin() == MII) | |||
438 | HSpiller.addToMergeableSpills(*MII, StackSlot, Original); | |||
439 | ++NumSpills; | |||
440 | return true; | |||
441 | } | |||
442 | ||||
443 | /// eliminateRedundantSpills - SLI:VNI is known to be on the stack. Remove any | |||
444 | /// redundant spills of this value in SLI.reg and sibling copies. | |||
445 | void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { | |||
446 | assert(VNI && "Missing value")(static_cast <bool> (VNI && "Missing value") ? void (0) : __assert_fail ("VNI && \"Missing value\"", "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 446, __extension__ __PRETTY_FUNCTION__)); | |||
447 | SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList; | |||
448 | WorkList.push_back(std::make_pair(&SLI, VNI)); | |||
449 | assert(StackInt && "No stack slot assigned yet.")(static_cast <bool> (StackInt && "No stack slot assigned yet." ) ? void (0) : __assert_fail ("StackInt && \"No stack slot assigned yet.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 449, __extension__ __PRETTY_FUNCTION__)); | |||
450 | ||||
451 | do { | |||
452 | LiveInterval *LI; | |||
453 | std::tie(LI, VNI) = WorkList.pop_back_val(); | |||
454 | Register Reg = LI->reg(); | |||
455 | LLVM_DEBUG(dbgs() << "Checking redundant spills for " << VNI->id << '@'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Checking redundant spills for " << VNI->id << '@' << VNI->def << " in " << *LI << '\n'; } } while (false) | |||
456 | << VNI->def << " in " << *LI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Checking redundant spills for " << VNI->id << '@' << VNI->def << " in " << *LI << '\n'; } } while (false); | |||
457 | ||||
458 | // Regs to spill are taken care of. | |||
459 | if (isRegToSpill(Reg)) | |||
460 | continue; | |||
461 | ||||
462 | // Add all of VNI's live range to StackInt. | |||
463 | StackInt->MergeValueInAsValue(*LI, VNI, StackInt->getValNumInfo(0)); | |||
464 | LLVM_DEBUG(dbgs() << "Merged to stack int: " << *StackInt << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Merged to stack int: " << *StackInt << '\n'; } } while (false); | |||
465 | ||||
466 | // Find all spills and copies of VNI. | |||
467 | for (MachineRegisterInfo::use_instr_nodbg_iterator | |||
468 | UI = MRI.use_instr_nodbg_begin(Reg), E = MRI.use_instr_nodbg_end(); | |||
469 | UI != E; ) { | |||
470 | MachineInstr &MI = *UI++; | |||
471 | if (!MI.isCopy() && !MI.mayStore()) | |||
472 | continue; | |||
473 | SlotIndex Idx = LIS.getInstructionIndex(MI); | |||
474 | if (LI->getVNInfoAt(Idx) != VNI) | |||
475 | continue; | |||
476 | ||||
477 | // Follow sibling copies down the dominator tree. | |||
478 | if (Register DstReg = isFullCopyOf(MI, Reg)) { | |||
479 | if (isSibling(DstReg)) { | |||
480 | LiveInterval &DstLI = LIS.getInterval(DstReg); | |||
481 | VNInfo *DstVNI = DstLI.getVNInfoAt(Idx.getRegSlot()); | |||
482 | assert(DstVNI && "Missing defined value")(static_cast <bool> (DstVNI && "Missing defined value" ) ? void (0) : __assert_fail ("DstVNI && \"Missing defined value\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 482, __extension__ __PRETTY_FUNCTION__)); | |||
483 | assert(DstVNI->def == Idx.getRegSlot() && "Wrong copy def slot")(static_cast <bool> (DstVNI->def == Idx.getRegSlot() && "Wrong copy def slot") ? void (0) : __assert_fail ("DstVNI->def == Idx.getRegSlot() && \"Wrong copy def slot\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 483, __extension__ __PRETTY_FUNCTION__)); | |||
484 | WorkList.push_back(std::make_pair(&DstLI, DstVNI)); | |||
485 | } | |||
486 | continue; | |||
487 | } | |||
488 | ||||
489 | // Erase spills. | |||
490 | int FI; | |||
491 | if (Reg == TII.isStoreToStackSlot(MI, FI) && FI == StackSlot) { | |||
| ||||
492 | LLVM_DEBUG(dbgs() << "Redundant spill " << Idx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Redundant spill " << Idx << '\t' << MI; } } while (false); | |||
493 | // eliminateDeadDefs won't normally remove stores, so switch opcode. | |||
494 | MI.setDesc(TII.get(TargetOpcode::KILL)); | |||
495 | DeadDefs.push_back(&MI); | |||
496 | ++NumSpillsRemoved; | |||
497 | if (HSpiller.rmFromMergeableSpills(MI, StackSlot)) | |||
498 | --NumSpills; | |||
499 | } | |||
500 | } | |||
501 | } while (!WorkList.empty()); | |||
502 | } | |||
503 | ||||
504 | //===----------------------------------------------------------------------===// | |||
505 | // Rematerialization | |||
506 | //===----------------------------------------------------------------------===// | |||
507 | ||||
508 | /// markValueUsed - Remember that VNI failed to rematerialize, so its defining | |||
509 | /// instruction cannot be eliminated. See through snippet copies | |||
510 | void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) { | |||
511 | SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList; | |||
512 | WorkList.push_back(std::make_pair(LI, VNI)); | |||
513 | do { | |||
514 | std::tie(LI, VNI) = WorkList.pop_back_val(); | |||
515 | if (!UsedValues.insert(VNI).second) | |||
516 | continue; | |||
517 | ||||
518 | if (VNI->isPHIDef()) { | |||
519 | MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); | |||
520 | for (MachineBasicBlock *P : MBB->predecessors()) { | |||
521 | VNInfo *PVNI = LI->getVNInfoBefore(LIS.getMBBEndIdx(P)); | |||
522 | if (PVNI) | |||
523 | WorkList.push_back(std::make_pair(LI, PVNI)); | |||
524 | } | |||
525 | continue; | |||
526 | } | |||
527 | ||||
528 | // Follow snippet copies. | |||
529 | MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def); | |||
530 | if (!SnippetCopies.count(MI)) | |||
531 | continue; | |||
532 | LiveInterval &SnipLI = LIS.getInterval(MI->getOperand(1).getReg()); | |||
533 | assert(isRegToSpill(SnipLI.reg()) && "Unexpected register in copy")(static_cast <bool> (isRegToSpill(SnipLI.reg()) && "Unexpected register in copy") ? void (0) : __assert_fail ("isRegToSpill(SnipLI.reg()) && \"Unexpected register in copy\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 533, __extension__ __PRETTY_FUNCTION__)); | |||
534 | VNInfo *SnipVNI = SnipLI.getVNInfoAt(VNI->def.getRegSlot(true)); | |||
535 | assert(SnipVNI && "Snippet undefined before copy")(static_cast <bool> (SnipVNI && "Snippet undefined before copy" ) ? void (0) : __assert_fail ("SnipVNI && \"Snippet undefined before copy\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 535, __extension__ __PRETTY_FUNCTION__)); | |||
536 | WorkList.push_back(std::make_pair(&SnipLI, SnipVNI)); | |||
537 | } while (!WorkList.empty()); | |||
538 | } | |||
539 | ||||
540 | bool InlineSpiller::canGuaranteeAssignmentAfterRemat(Register VReg, | |||
541 | MachineInstr &MI) { | |||
542 | if (!RestrictStatepointRemat) | |||
543 | return true; | |||
544 | // Here's a quick explanation of the problem we're trying to handle here: | |||
545 | // * There are some pseudo instructions with more vreg uses than there are | |||
546 | // physical registers on the machine. | |||
547 | // * This is normally handled by spilling the vreg, and folding the reload | |||
548 | // into the user instruction. (Thus decreasing the number of used vregs | |||
549 | // until the remainder can be assigned to physregs.) | |||
550 | // * However, since we may try to spill vregs in any order, we can end up | |||
551 | // trying to spill each operand to the instruction, and then rematting it | |||
552 | // instead. When that happens, the new live intervals (for the remats) are | |||
553 | // expected to be trivially assignable (i.e. RS_Done). However, since we | |||
554 | // may have more remats than physregs, we're guaranteed to fail to assign | |||
555 | // one. | |||
556 | // At the moment, we only handle this for STATEPOINTs since they're the only | |||
557 | // pseudo op where we've seen this. If we start seeing other instructions | |||
558 | // with the same problem, we need to revisit this. | |||
559 | if (MI.getOpcode() != TargetOpcode::STATEPOINT) | |||
560 | return true; | |||
561 | // For STATEPOINTs we allow re-materialization for fixed arguments only hoping | |||
562 | // that number of physical registers is enough to cover all fixed arguments. | |||
563 | // If it is not true we need to revisit it. | |||
564 | for (unsigned Idx = StatepointOpers(&MI).getVarIdx(), | |||
565 | EndIdx = MI.getNumOperands(); | |||
566 | Idx < EndIdx; ++Idx) { | |||
567 | MachineOperand &MO = MI.getOperand(Idx); | |||
568 | if (MO.isReg() && MO.getReg() == VReg) | |||
569 | return false; | |||
570 | } | |||
571 | return true; | |||
572 | } | |||
573 | ||||
574 | /// reMaterializeFor - Attempt to rematerialize before MI instead of reloading. | |||
575 | bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) { | |||
576 | // Analyze instruction | |||
577 | SmallVector<std::pair<MachineInstr *, unsigned>, 8> Ops; | |||
578 | VirtRegInfo RI = AnalyzeVirtRegInBundle(MI, VirtReg.reg(), &Ops); | |||
579 | ||||
580 | if (!RI.Reads) | |||
581 | return false; | |||
582 | ||||
583 | SlotIndex UseIdx = LIS.getInstructionIndex(MI).getRegSlot(true); | |||
584 | VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex()); | |||
585 | ||||
586 | if (!ParentVNI) { | |||
587 | LLVM_DEBUG(dbgs() << "\tadding <undef> flags: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tadding <undef> flags: " ; } } while (false); | |||
588 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
589 | MachineOperand &MO = MI.getOperand(i); | |||
590 | if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg()) | |||
591 | MO.setIsUndef(); | |||
592 | } | |||
593 | LLVM_DEBUG(dbgs() << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << UseIdx << '\t' << MI; } } while (false); | |||
594 | return true; | |||
595 | } | |||
596 | ||||
597 | if (SnippetCopies.count(&MI)) | |||
598 | return false; | |||
599 | ||||
600 | LiveInterval &OrigLI = LIS.getInterval(Original); | |||
601 | VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx); | |||
602 | LiveRangeEdit::Remat RM(ParentVNI); | |||
603 | RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def); | |||
604 | ||||
605 | if (!Edit->canRematerializeAt(RM, OrigVNI, UseIdx, false)) { | |||
606 | markValueUsed(&VirtReg, ParentVNI); | |||
607 | LLVM_DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI; } } while (false); | |||
608 | return false; | |||
609 | } | |||
610 | ||||
611 | // If the instruction also writes VirtReg.reg, it had better not require the | |||
612 | // same register for uses and defs. | |||
613 | if (RI.Tied) { | |||
614 | markValueUsed(&VirtReg, ParentVNI); | |||
615 | LLVM_DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << MI; } } while (false); | |||
616 | return false; | |||
617 | } | |||
618 | ||||
619 | // Before rematerializing into a register for a single instruction, try to | |||
620 | // fold a load into the instruction. That avoids allocating a new register. | |||
621 | if (RM.OrigMI->canFoldAsLoad() && | |||
622 | foldMemoryOperand(Ops, RM.OrigMI)) { | |||
623 | Edit->markRematerialized(RM.ParentVNI); | |||
624 | ++NumFoldedLoads; | |||
625 | return true; | |||
626 | } | |||
627 | ||||
628 | // If we can't guarantee that we'll be able to actually assign the new vreg, | |||
629 | // we can't remat. | |||
630 | if (!canGuaranteeAssignmentAfterRemat(VirtReg.reg(), MI)) { | |||
631 | markValueUsed(&VirtReg, ParentVNI); | |||
632 | LLVM_DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI; } } while (false); | |||
633 | return false; | |||
634 | } | |||
635 | ||||
636 | // Allocate a new register for the remat. | |||
637 | Register NewVReg = Edit->createFrom(Original); | |||
638 | ||||
639 | // Finally we can rematerialize OrigMI before MI. | |||
640 | SlotIndex DefIdx = | |||
641 | Edit->rematerializeAt(*MI.getParent(), MI, NewVReg, RM, TRI); | |||
642 | ||||
643 | // We take the DebugLoc from MI, since OrigMI may be attributed to a | |||
644 | // different source location. | |||
645 | auto *NewMI = LIS.getInstructionFromIndex(DefIdx); | |||
646 | NewMI->setDebugLoc(MI.getDebugLoc()); | |||
647 | ||||
648 | (void)DefIdx; | |||
649 | LLVM_DEBUG(dbgs() << "\tremat: " << DefIdx << '\t'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremat: " << DefIdx << '\t' << *LIS.getInstructionFromIndex(DefIdx); } } while (false) | |||
650 | << *LIS.getInstructionFromIndex(DefIdx))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremat: " << DefIdx << '\t' << *LIS.getInstructionFromIndex(DefIdx); } } while (false); | |||
651 | ||||
652 | // Replace operands | |||
653 | for (const auto &OpPair : Ops) { | |||
654 | MachineOperand &MO = OpPair.first->getOperand(OpPair.second); | |||
655 | if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg()) { | |||
656 | MO.setReg(NewVReg); | |||
657 | MO.setIsKill(); | |||
658 | } | |||
659 | } | |||
660 | LLVM_DEBUG(dbgs() << "\t " << UseIdx << '\t' << MI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t " << UseIdx << '\t' << MI << '\n'; } } while (false); | |||
661 | ||||
662 | ++NumRemats; | |||
663 | return true; | |||
664 | } | |||
665 | ||||
666 | /// reMaterializeAll - Try to rematerialize as many uses as possible, | |||
667 | /// and trim the live ranges after. | |||
668 | void InlineSpiller::reMaterializeAll() { | |||
669 | if (!Edit->anyRematerializable(AA)) | |||
670 | return; | |||
671 | ||||
672 | UsedValues.clear(); | |||
673 | ||||
674 | // Try to remat before all uses of snippets. | |||
675 | bool anyRemat = false; | |||
676 | for (Register Reg : RegsToSpill) { | |||
677 | LiveInterval &LI = LIS.getInterval(Reg); | |||
678 | for (MachineRegisterInfo::reg_bundle_iterator | |||
679 | RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end(); | |||
680 | RegI != E; ) { | |||
681 | MachineInstr &MI = *RegI++; | |||
682 | ||||
683 | // Debug values are not allowed to affect codegen. | |||
684 | if (MI.isDebugValue()) | |||
685 | continue; | |||
686 | ||||
687 | assert(!MI.isDebugInstr() && "Did not expect to find a use in debug "(static_cast <bool> (!MI.isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? void (0) : __assert_fail ("!MI.isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 688, __extension__ __PRETTY_FUNCTION__)) | |||
688 | "instruction that isn't a DBG_VALUE")(static_cast <bool> (!MI.isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? void (0) : __assert_fail ("!MI.isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 688, __extension__ __PRETTY_FUNCTION__)); | |||
689 | ||||
690 | anyRemat |= reMaterializeFor(LI, MI); | |||
691 | } | |||
692 | } | |||
693 | if (!anyRemat) | |||
694 | return; | |||
695 | ||||
696 | // Remove any values that were completely rematted. | |||
697 | for (Register Reg : RegsToSpill) { | |||
698 | LiveInterval &LI = LIS.getInterval(Reg); | |||
699 | for (LiveInterval::vni_iterator I = LI.vni_begin(), E = LI.vni_end(); | |||
700 | I != E; ++I) { | |||
701 | VNInfo *VNI = *I; | |||
702 | if (VNI->isUnused() || VNI->isPHIDef() || UsedValues.count(VNI)) | |||
703 | continue; | |||
704 | MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def); | |||
705 | MI->addRegisterDead(Reg, &TRI); | |||
706 | if (!MI->allDefsAreDead()) | |||
707 | continue; | |||
708 | LLVM_DEBUG(dbgs() << "All defs dead: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "All defs dead: " << *MI ; } } while (false); | |||
709 | DeadDefs.push_back(MI); | |||
710 | } | |||
711 | } | |||
712 | ||||
713 | // Eliminate dead code after remat. Note that some snippet copies may be | |||
714 | // deleted here. | |||
715 | if (DeadDefs.empty()) | |||
716 | return; | |||
717 | LLVM_DEBUG(dbgs() << "Remat created " << DeadDefs.size() << " dead defs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Remat created " << DeadDefs .size() << " dead defs.\n"; } } while (false); | |||
718 | Edit->eliminateDeadDefs(DeadDefs, RegsToSpill, AA); | |||
719 | ||||
720 | // LiveRangeEdit::eliminateDeadDef is used to remove dead define instructions | |||
721 | // after rematerialization. To remove a VNI for a vreg from its LiveInterval, | |||
722 | // LiveIntervals::removeVRegDefAt is used. However, after non-PHI VNIs are all | |||
723 | // removed, PHI VNI are still left in the LiveInterval. | |||
724 | // So to get rid of unused reg, we need to check whether it has non-dbg | |||
725 | // reference instead of whether it has non-empty interval. | |||
726 | unsigned ResultPos = 0; | |||
727 | for (Register Reg : RegsToSpill) { | |||
728 | if (MRI.reg_nodbg_empty(Reg)) { | |||
729 | Edit->eraseVirtReg(Reg); | |||
730 | continue; | |||
731 | } | |||
732 | ||||
733 | assert(LIS.hasInterval(Reg) &&(static_cast <bool> (LIS.hasInterval(Reg) && (! LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && "Empty and not used live-range?!") ? void (0) : __assert_fail ("LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && \"Empty and not used live-range?!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 735, __extension__ __PRETTY_FUNCTION__)) | |||
734 | (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) &&(static_cast <bool> (LIS.hasInterval(Reg) && (! LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && "Empty and not used live-range?!") ? void (0) : __assert_fail ("LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && \"Empty and not used live-range?!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 735, __extension__ __PRETTY_FUNCTION__)) | |||
735 | "Empty and not used live-range?!")(static_cast <bool> (LIS.hasInterval(Reg) && (! LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && "Empty and not used live-range?!") ? void (0) : __assert_fail ("LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && \"Empty and not used live-range?!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 735, __extension__ __PRETTY_FUNCTION__)); | |||
736 | ||||
737 | RegsToSpill[ResultPos++] = Reg; | |||
738 | } | |||
739 | RegsToSpill.erase(RegsToSpill.begin() + ResultPos, RegsToSpill.end()); | |||
740 | LLVM_DEBUG(dbgs() << RegsToSpill.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << RegsToSpill.size() << " registers to spill after remat.\n" ; } } while (false) | |||
741 | << " registers to spill after remat.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << RegsToSpill.size() << " registers to spill after remat.\n" ; } } while (false); | |||
742 | } | |||
743 | ||||
744 | //===----------------------------------------------------------------------===// | |||
745 | // Spilling | |||
746 | //===----------------------------------------------------------------------===// | |||
747 | ||||
748 | /// If MI is a load or store of StackSlot, it can be removed. | |||
749 | bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, Register Reg) { | |||
750 | int FI = 0; | |||
751 | Register InstrReg = TII.isLoadFromStackSlot(*MI, FI); | |||
752 | bool IsLoad = InstrReg; | |||
753 | if (!IsLoad) | |||
754 | InstrReg = TII.isStoreToStackSlot(*MI, FI); | |||
755 | ||||
756 | // We have a stack access. Is it the right register and slot? | |||
757 | if (InstrReg != Reg || FI != StackSlot) | |||
758 | return false; | |||
759 | ||||
760 | if (!IsLoad) | |||
761 | HSpiller.rmFromMergeableSpills(*MI, StackSlot); | |||
762 | ||||
763 | LLVM_DEBUG(dbgs() << "Coalescing stack access: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Coalescing stack access: " << *MI; } } while (false); | |||
764 | LIS.RemoveMachineInstrFromMaps(*MI); | |||
765 | MI->eraseFromParent(); | |||
766 | ||||
767 | if (IsLoad) { | |||
768 | ++NumReloadsRemoved; | |||
769 | --NumReloads; | |||
770 | } else { | |||
771 | ++NumSpillsRemoved; | |||
772 | --NumSpills; | |||
773 | } | |||
774 | ||||
775 | return true; | |||
776 | } | |||
777 | ||||
778 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
779 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | |||
780 | // Dump the range of instructions from B to E with their slot indexes. | |||
781 | static void dumpMachineInstrRangeWithSlotIndex(MachineBasicBlock::iterator B, | |||
782 | MachineBasicBlock::iterator E, | |||
783 | LiveIntervals const &LIS, | |||
784 | const char *const header, | |||
785 | Register VReg = Register()) { | |||
786 | char NextLine = '\n'; | |||
787 | char SlotIndent = '\t'; | |||
788 | ||||
789 | if (std::next(B) == E) { | |||
790 | NextLine = ' '; | |||
791 | SlotIndent = ' '; | |||
792 | } | |||
793 | ||||
794 | dbgs() << '\t' << header << ": " << NextLine; | |||
795 | ||||
796 | for (MachineBasicBlock::iterator I = B; I != E; ++I) { | |||
797 | SlotIndex Idx = LIS.getInstructionIndex(*I).getRegSlot(); | |||
798 | ||||
799 | // If a register was passed in and this instruction has it as a | |||
800 | // destination that is marked as an early clobber, print the | |||
801 | // early-clobber slot index. | |||
802 | if (VReg) { | |||
803 | MachineOperand *MO = I->findRegisterDefOperand(VReg); | |||
804 | if (MO && MO->isEarlyClobber()) | |||
805 | Idx = Idx.getRegSlot(true); | |||
806 | } | |||
807 | ||||
808 | dbgs() << SlotIndent << Idx << '\t' << *I; | |||
809 | } | |||
810 | } | |||
811 | #endif | |||
812 | ||||
813 | /// foldMemoryOperand - Try folding stack slot references in Ops into their | |||
814 | /// instructions. | |||
815 | /// | |||
816 | /// @param Ops Operand indices from AnalyzeVirtRegInBundle(). | |||
817 | /// @param LoadMI Load instruction to use instead of stack slot when non-null. | |||
818 | /// @return True on success. | |||
819 | bool InlineSpiller:: | |||
820 | foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops, | |||
821 | MachineInstr *LoadMI) { | |||
822 | if (Ops.empty()) | |||
823 | return false; | |||
824 | // Don't attempt folding in bundles. | |||
825 | MachineInstr *MI = Ops.front().first; | |||
826 | if (Ops.back().first != MI || MI->isBundled()) | |||
827 | return false; | |||
828 | ||||
829 | bool WasCopy = MI->isCopy(); | |||
830 | Register ImpReg; | |||
831 | ||||
832 | // TII::foldMemoryOperand will do what we need here for statepoint | |||
833 | // (fold load into use and remove corresponding def). We will replace | |||
834 | // uses of removed def with loads (spillAroundUses). | |||
835 | // For that to work we need to untie def and use to pass it through | |||
836 | // foldMemoryOperand and signal foldPatchpoint that it is allowed to | |||
837 | // fold them. | |||
838 | bool UntieRegs = MI->getOpcode() == TargetOpcode::STATEPOINT; | |||
839 | ||||
840 | // Spill subregs if the target allows it. | |||
841 | // We always want to spill subregs for stackmap/patchpoint pseudos. | |||
842 | bool SpillSubRegs = TII.isSubregFoldable() || | |||
843 | MI->getOpcode() == TargetOpcode::STATEPOINT || | |||
844 | MI->getOpcode() == TargetOpcode::PATCHPOINT || | |||
845 | MI->getOpcode() == TargetOpcode::STACKMAP; | |||
846 | ||||
847 | // TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied | |||
848 | // operands. | |||
849 | SmallVector<unsigned, 8> FoldOps; | |||
850 | for (const auto &OpPair : Ops) { | |||
851 | unsigned Idx = OpPair.second; | |||
852 | assert(MI == OpPair.first && "Instruction conflict during operand folding")(static_cast <bool> (MI == OpPair.first && "Instruction conflict during operand folding" ) ? void (0) : __assert_fail ("MI == OpPair.first && \"Instruction conflict during operand folding\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 852, __extension__ __PRETTY_FUNCTION__)); | |||
853 | MachineOperand &MO = MI->getOperand(Idx); | |||
854 | if (MO.isImplicit()) { | |||
855 | ImpReg = MO.getReg(); | |||
856 | continue; | |||
857 | } | |||
858 | ||||
859 | if (!SpillSubRegs && MO.getSubReg()) | |||
860 | return false; | |||
861 | // We cannot fold a load instruction into a def. | |||
862 | if (LoadMI && MO.isDef()) | |||
863 | return false; | |||
864 | // Tied use operands should not be passed to foldMemoryOperand. | |||
865 | if (UntieRegs || !MI->isRegTiedToDefOperand(Idx)) | |||
866 | FoldOps.push_back(Idx); | |||
867 | } | |||
868 | ||||
869 | // If we only have implicit uses, we won't be able to fold that. | |||
870 | // Moreover, TargetInstrInfo::foldMemoryOperand will assert if we try! | |||
871 | if (FoldOps.empty()) | |||
872 | return false; | |||
873 | ||||
874 | MachineInstrSpan MIS(MI, MI->getParent()); | |||
875 | ||||
876 | SmallVector<std::pair<unsigned, unsigned> > TiedOps; | |||
877 | if (UntieRegs) | |||
878 | for (unsigned Idx : FoldOps) { | |||
879 | MachineOperand &MO = MI->getOperand(Idx); | |||
880 | if (!MO.isTied()) | |||
881 | continue; | |||
882 | unsigned Tied = MI->findTiedOperandIdx(Idx); | |||
883 | if (MO.isUse()) | |||
884 | TiedOps.emplace_back(Tied, Idx); | |||
885 | else { | |||
886 | assert(MO.isDef() && "Tied to not use and def?")(static_cast <bool> (MO.isDef() && "Tied to not use and def?" ) ? void (0) : __assert_fail ("MO.isDef() && \"Tied to not use and def?\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 886, __extension__ __PRETTY_FUNCTION__)); | |||
887 | TiedOps.emplace_back(Idx, Tied); | |||
888 | } | |||
889 | MI->untieRegOperand(Idx); | |||
890 | } | |||
891 | ||||
892 | MachineInstr *FoldMI = | |||
893 | LoadMI ? TII.foldMemoryOperand(*MI, FoldOps, *LoadMI, &LIS) | |||
894 | : TII.foldMemoryOperand(*MI, FoldOps, StackSlot, &LIS, &VRM); | |||
895 | if (!FoldMI) { | |||
896 | // Re-tie operands. | |||
897 | for (auto Tied : TiedOps) | |||
898 | MI->tieOperands(Tied.first, Tied.second); | |||
899 | return false; | |||
900 | } | |||
901 | ||||
902 | // Remove LIS for any dead defs in the original MI not in FoldMI. | |||
903 | for (MIBundleOperands MO(*MI); MO.isValid(); ++MO) { | |||
904 | if (!MO->isReg()) | |||
905 | continue; | |||
906 | Register Reg = MO->getReg(); | |||
907 | if (!Reg || Register::isVirtualRegister(Reg) || MRI.isReserved(Reg)) { | |||
908 | continue; | |||
909 | } | |||
910 | // Skip non-Defs, including undef uses and internal reads. | |||
911 | if (MO->isUse()) | |||
912 | continue; | |||
913 | PhysRegInfo RI = AnalyzePhysRegInBundle(*FoldMI, Reg, &TRI); | |||
914 | if (RI.FullyDefined) | |||
915 | continue; | |||
916 | // FoldMI does not define this physreg. Remove the LI segment. | |||
917 | assert(MO->isDead() && "Cannot fold physreg def")(static_cast <bool> (MO->isDead() && "Cannot fold physreg def" ) ? void (0) : __assert_fail ("MO->isDead() && \"Cannot fold physreg def\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 917, __extension__ __PRETTY_FUNCTION__)); | |||
918 | SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot(); | |||
919 | LIS.removePhysRegDefAt(Reg.asMCReg(), Idx); | |||
920 | } | |||
921 | ||||
922 | int FI; | |||
923 | if (TII.isStoreToStackSlot(*MI, FI) && | |||
924 | HSpiller.rmFromMergeableSpills(*MI, FI)) | |||
925 | --NumSpills; | |||
926 | LIS.ReplaceMachineInstrInMaps(*MI, *FoldMI); | |||
927 | // Update the call site info. | |||
928 | if (MI->isCandidateForCallSiteEntry()) | |||
929 | MI->getMF()->moveCallSiteInfo(MI, FoldMI); | |||
930 | ||||
931 | // If we've folded a store into an instruction labelled with debug-info, | |||
932 | // record a substitution from the old operand to the memory operand. Handle | |||
933 | // the simple common case where operand 0 is the one being folded, plus when | |||
934 | // the destination operand is also a tied def. More values could be | |||
935 | // substituted / preserved with more analysis. | |||
936 | if (MI->peekDebugInstrNum() && Ops[0].second == 0) { | |||
937 | // Helper lambda. | |||
938 | auto MakeSubstitution = [this,FoldMI,MI,&Ops]() { | |||
939 | // Substitute old operand zero to the new instructions memory operand. | |||
940 | unsigned OldOperandNum = Ops[0].second; | |||
941 | unsigned NewNum = FoldMI->getDebugInstrNum(); | |||
942 | unsigned OldNum = MI->getDebugInstrNum(); | |||
943 | MF.makeDebugValueSubstitution({OldNum, OldOperandNum}, | |||
944 | {NewNum, MachineFunction::DebugOperandMemNumber}); | |||
945 | }; | |||
946 | ||||
947 | const MachineOperand &Op0 = MI->getOperand(Ops[0].second); | |||
948 | if (Ops.size() == 1 && Op0.isDef()) { | |||
949 | MakeSubstitution(); | |||
950 | } else if (Ops.size() == 2 && Op0.isDef() && MI->getOperand(1).isTied() && | |||
951 | Op0.getReg() == MI->getOperand(1).getReg()) { | |||
952 | MakeSubstitution(); | |||
953 | } | |||
954 | } else if (MI->peekDebugInstrNum()) { | |||
955 | // This is a debug-labelled instruction, but the operand being folded isn't | |||
956 | // at operand zero. Most likely this means it's a load being folded in. | |||
957 | // Substitute any register defs from operand zero up to the one being | |||
958 | // folded -- past that point, we don't know what the new operand indexes | |||
959 | // will be. | |||
960 | MF.substituteDebugValuesForInst(*MI, *FoldMI, Ops[0].second); | |||
961 | } | |||
962 | ||||
963 | MI->eraseFromParent(); | |||
964 | ||||
965 | // Insert any new instructions other than FoldMI into the LIS maps. | |||
966 | assert(!MIS.empty() && "Unexpected empty span of instructions!")(static_cast <bool> (!MIS.empty() && "Unexpected empty span of instructions!" ) ? void (0) : __assert_fail ("!MIS.empty() && \"Unexpected empty span of instructions!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 966, __extension__ __PRETTY_FUNCTION__)); | |||
967 | for (MachineInstr &MI : MIS) | |||
968 | if (&MI != FoldMI) | |||
969 | LIS.InsertMachineInstrInMaps(MI); | |||
970 | ||||
971 | // TII.foldMemoryOperand may have left some implicit operands on the | |||
972 | // instruction. Strip them. | |||
973 | if (ImpReg) | |||
974 | for (unsigned i = FoldMI->getNumOperands(); i; --i) { | |||
975 | MachineOperand &MO = FoldMI->getOperand(i - 1); | |||
976 | if (!MO.isReg() || !MO.isImplicit()) | |||
977 | break; | |||
978 | if (MO.getReg() == ImpReg) | |||
979 | FoldMI->RemoveOperand(i - 1); | |||
980 | } | |||
981 | ||||
982 | LLVM_DEBUG(dumpMachineInstrRangeWithSlotIndex(MIS.begin(), MIS.end(), LIS,do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MIS.end(), LIS, "folded"); } } while (false) | |||
983 | "folded"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MIS.end(), LIS, "folded"); } } while (false); | |||
984 | ||||
985 | if (!WasCopy) | |||
986 | ++NumFolded; | |||
987 | else if (Ops.front().second == 0) { | |||
988 | ++NumSpills; | |||
989 | // If there is only 1 store instruction is required for spill, add it | |||
990 | // to mergeable list. In X86 AMX, 2 intructions are required to store. | |||
991 | // We disable the merge for this case. | |||
992 | if (std::distance(MIS.begin(), MIS.end()) <= 1) | |||
993 | HSpiller.addToMergeableSpills(*FoldMI, StackSlot, Original); | |||
994 | } else | |||
995 | ++NumReloads; | |||
996 | return true; | |||
997 | } | |||
998 | ||||
999 | void InlineSpiller::insertReload(Register NewVReg, | |||
1000 | SlotIndex Idx, | |||
1001 | MachineBasicBlock::iterator MI) { | |||
1002 | MachineBasicBlock &MBB = *MI->getParent(); | |||
1003 | ||||
1004 | MachineInstrSpan MIS(MI, &MBB); | |||
1005 | TII.loadRegFromStackSlot(MBB, MI, NewVReg, StackSlot, | |||
1006 | MRI.getRegClass(NewVReg), &TRI); | |||
1007 | ||||
1008 | LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MI); | |||
1009 | ||||
1010 | LLVM_DEBUG(dumpMachineInstrRangeWithSlotIndex(MIS.begin(), MI, LIS, "reload",do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MI, LIS, "reload", NewVReg); } } while (false) | |||
1011 | NewVReg))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MI, LIS, "reload", NewVReg); } } while (false); | |||
1012 | ++NumReloads; | |||
1013 | } | |||
1014 | ||||
1015 | /// Check if \p Def fully defines a VReg with an undefined value. | |||
1016 | /// If that's the case, that means the value of VReg is actually | |||
1017 | /// not relevant. | |||
1018 | static bool isRealSpill(const MachineInstr &Def) { | |||
1019 | if (!Def.isImplicitDef()) | |||
1020 | return true; | |||
1021 | assert(Def.getNumOperands() == 1 &&(static_cast <bool> (Def.getNumOperands() == 1 && "Implicit def with more than one definition") ? void (0) : __assert_fail ("Def.getNumOperands() == 1 && \"Implicit def with more than one definition\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1022, __extension__ __PRETTY_FUNCTION__)) | |||
1022 | "Implicit def with more than one definition")(static_cast <bool> (Def.getNumOperands() == 1 && "Implicit def with more than one definition") ? void (0) : __assert_fail ("Def.getNumOperands() == 1 && \"Implicit def with more than one definition\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1022, __extension__ __PRETTY_FUNCTION__)); | |||
1023 | // We can say that the VReg defined by Def is undef, only if it is | |||
1024 | // fully defined by Def. Otherwise, some of the lanes may not be | |||
1025 | // undef and the value of the VReg matters. | |||
1026 | return Def.getOperand(0).getSubReg(); | |||
1027 | } | |||
1028 | ||||
1029 | /// insertSpill - Insert a spill of NewVReg after MI. | |||
1030 | void InlineSpiller::insertSpill(Register NewVReg, bool isKill, | |||
1031 | MachineBasicBlock::iterator MI) { | |||
1032 | // Spill are not terminators, so inserting spills after terminators will | |||
1033 | // violate invariants in MachineVerifier. | |||
1034 | assert(!MI->isTerminator() && "Inserting a spill after a terminator")(static_cast <bool> (!MI->isTerminator() && "Inserting a spill after a terminator" ) ? void (0) : __assert_fail ("!MI->isTerminator() && \"Inserting a spill after a terminator\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1034, __extension__ __PRETTY_FUNCTION__)); | |||
1035 | MachineBasicBlock &MBB = *MI->getParent(); | |||
1036 | ||||
1037 | MachineInstrSpan MIS(MI, &MBB); | |||
1038 | MachineBasicBlock::iterator SpillBefore = std::next(MI); | |||
1039 | bool IsRealSpill = isRealSpill(*MI); | |||
1040 | ||||
1041 | if (IsRealSpill) | |||
1042 | TII.storeRegToStackSlot(MBB, SpillBefore, NewVReg, isKill, StackSlot, | |||
1043 | MRI.getRegClass(NewVReg), &TRI); | |||
1044 | else | |||
1045 | // Don't spill undef value. | |||
1046 | // Anything works for undef, in particular keeping the memory | |||
1047 | // uninitialized is a viable option and it saves code size and | |||
1048 | // run time. | |||
1049 | BuildMI(MBB, SpillBefore, MI->getDebugLoc(), TII.get(TargetOpcode::KILL)) | |||
1050 | .addReg(NewVReg, getKillRegState(isKill)); | |||
1051 | ||||
1052 | MachineBasicBlock::iterator Spill = std::next(MI); | |||
1053 | LIS.InsertMachineInstrRangeInMaps(Spill, MIS.end()); | |||
1054 | for (const MachineInstr &MI : make_range(Spill, MIS.end())) | |||
1055 | getVDefInterval(MI, LIS); | |||
1056 | ||||
1057 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(Spill, MIS .end(), LIS, "spill"); } } while (false) | |||
1058 | dumpMachineInstrRangeWithSlotIndex(Spill, MIS.end(), LIS, "spill"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(Spill, MIS .end(), LIS, "spill"); } } while (false); | |||
1059 | ++NumSpills; | |||
1060 | // If there is only 1 store instruction is required for spill, add it | |||
1061 | // to mergeable list. In X86 AMX, 2 intructions are required to store. | |||
1062 | // We disable the merge for this case. | |||
1063 | if (IsRealSpill && std::distance(Spill, MIS.end()) <= 1) | |||
1064 | HSpiller.addToMergeableSpills(*Spill, StackSlot, Original); | |||
1065 | } | |||
1066 | ||||
1067 | /// spillAroundUses - insert spill code around each use of Reg. | |||
1068 | void InlineSpiller::spillAroundUses(Register Reg) { | |||
1069 | LLVM_DEBUG(dbgs() << "spillAroundUses " << printReg(Reg) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "spillAroundUses " << printReg (Reg) << '\n'; } } while (false); | |||
1070 | LiveInterval &OldLI = LIS.getInterval(Reg); | |||
1071 | ||||
1072 | // Iterate over instructions using Reg. | |||
1073 | for (MachineRegisterInfo::reg_bundle_iterator | |||
1074 | RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end(); | |||
1075 | RegI != E; ) { | |||
1076 | MachineInstr *MI = &*(RegI++); | |||
1077 | ||||
1078 | // Debug values are not allowed to affect codegen. | |||
1079 | if (MI->isDebugValue()) { | |||
1080 | // Modify DBG_VALUE now that the value is in a spill slot. | |||
1081 | MachineBasicBlock *MBB = MI->getParent(); | |||
1082 | LLVM_DEBUG(dbgs() << "Modifying debug info due to spill:\t" << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Modifying debug info due to spill:\t" << *MI; } } while (false); | |||
1083 | buildDbgValueForSpill(*MBB, MI, *MI, StackSlot, Reg); | |||
1084 | MBB->erase(MI); | |||
1085 | continue; | |||
1086 | } | |||
1087 | ||||
1088 | assert(!MI->isDebugInstr() && "Did not expect to find a use in debug "(static_cast <bool> (!MI->isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? void (0) : __assert_fail ("!MI->isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1089, __extension__ __PRETTY_FUNCTION__)) | |||
1089 | "instruction that isn't a DBG_VALUE")(static_cast <bool> (!MI->isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? void (0) : __assert_fail ("!MI->isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1089, __extension__ __PRETTY_FUNCTION__)); | |||
1090 | ||||
1091 | // Ignore copies to/from snippets. We'll delete them. | |||
1092 | if (SnippetCopies.count(MI)) | |||
1093 | continue; | |||
1094 | ||||
1095 | // Stack slot accesses may coalesce away. | |||
1096 | if (coalesceStackAccess(MI, Reg)) | |||
1097 | continue; | |||
1098 | ||||
1099 | // Analyze instruction. | |||
1100 | SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops; | |||
1101 | VirtRegInfo RI = AnalyzeVirtRegInBundle(*MI, Reg, &Ops); | |||
1102 | ||||
1103 | // Find the slot index where this instruction reads and writes OldLI. | |||
1104 | // This is usually the def slot, except for tied early clobbers. | |||
1105 | SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot(); | |||
1106 | if (VNInfo *VNI = OldLI.getVNInfoAt(Idx.getRegSlot(true))) | |||
1107 | if (SlotIndex::isSameInstr(Idx, VNI->def)) | |||
1108 | Idx = VNI->def; | |||
1109 | ||||
1110 | // Check for a sibling copy. | |||
1111 | Register SibReg = isFullCopyOf(*MI, Reg); | |||
1112 | if (SibReg && isSibling(SibReg)) { | |||
1113 | // This may actually be a copy between snippets. | |||
1114 | if (isRegToSpill(SibReg)) { | |||
1115 | LLVM_DEBUG(dbgs() << "Found new snippet copy: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Found new snippet copy: " << *MI; } } while (false); | |||
1116 | SnippetCopies.insert(MI); | |||
1117 | continue; | |||
1118 | } | |||
1119 | if (RI.Writes) { | |||
1120 | if (hoistSpillInsideBB(OldLI, *MI)) { | |||
1121 | // This COPY is now dead, the value is already in the stack slot. | |||
1122 | MI->getOperand(0).setIsDead(); | |||
1123 | DeadDefs.push_back(MI); | |||
1124 | continue; | |||
1125 | } | |||
1126 | } else { | |||
1127 | // This is a reload for a sib-reg copy. Drop spills downstream. | |||
1128 | LiveInterval &SibLI = LIS.getInterval(SibReg); | |||
1129 | eliminateRedundantSpills(SibLI, SibLI.getVNInfoAt(Idx)); | |||
1130 | // The COPY will fold to a reload below. | |||
1131 | } | |||
1132 | } | |||
1133 | ||||
1134 | // Attempt to fold memory ops. | |||
1135 | if (foldMemoryOperand(Ops)) | |||
1136 | continue; | |||
1137 | ||||
1138 | // Create a new virtual register for spill/fill. | |||
1139 | // FIXME: Infer regclass from instruction alone. | |||
1140 | Register NewVReg = Edit->createFrom(Reg); | |||
1141 | ||||
1142 | if (RI.Reads) | |||
1143 | insertReload(NewVReg, Idx, MI); | |||
1144 | ||||
1145 | // Rewrite instruction operands. | |||
1146 | bool hasLiveDef = false; | |||
1147 | for (const auto &OpPair : Ops) { | |||
1148 | MachineOperand &MO = OpPair.first->getOperand(OpPair.second); | |||
1149 | MO.setReg(NewVReg); | |||
1150 | if (MO.isUse()) { | |||
1151 | if (!OpPair.first->isRegTiedToDefOperand(OpPair.second)) | |||
1152 | MO.setIsKill(); | |||
1153 | } else { | |||
1154 | if (!MO.isDead()) | |||
1155 | hasLiveDef = true; | |||
1156 | } | |||
1157 | } | |||
1158 | LLVM_DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\trewrite: " << Idx << '\t' << *MI << '\n'; } } while (false); | |||
1159 | ||||
1160 | // FIXME: Use a second vreg if instruction has no tied ops. | |||
1161 | if (RI.Writes) | |||
1162 | if (hasLiveDef) | |||
1163 | insertSpill(NewVReg, true, MI); | |||
1164 | } | |||
1165 | } | |||
1166 | ||||
1167 | /// spillAll - Spill all registers remaining after rematerialization. | |||
1168 | void InlineSpiller::spillAll() { | |||
1169 | // Update LiveStacks now that we are committed to spilling. | |||
1170 | if (StackSlot == VirtRegMap::NO_STACK_SLOT) { | |||
1171 | StackSlot = VRM.assignVirt2StackSlot(Original); | |||
1172 | StackInt = &LSS.getOrCreateInterval(StackSlot, MRI.getRegClass(Original)); | |||
1173 | StackInt->getNextValue(SlotIndex(), LSS.getVNInfoAllocator()); | |||
1174 | } else | |||
1175 | StackInt = &LSS.getInterval(StackSlot); | |||
1176 | ||||
1177 | if (Original != Edit->getReg()) | |||
1178 | VRM.assignVirt2StackSlot(Edit->getReg(), StackSlot); | |||
1179 | ||||
1180 | assert(StackInt->getNumValNums() == 1 && "Bad stack interval values")(static_cast <bool> (StackInt->getNumValNums() == 1 && "Bad stack interval values") ? void (0) : __assert_fail ("StackInt->getNumValNums() == 1 && \"Bad stack interval values\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1180, __extension__ __PRETTY_FUNCTION__)); | |||
1181 | for (Register Reg : RegsToSpill) | |||
1182 | StackInt->MergeSegmentsInAsValue(LIS.getInterval(Reg), | |||
1183 | StackInt->getValNumInfo(0)); | |||
1184 | LLVM_DEBUG(dbgs() << "Merged spilled regs: " << *StackInt << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Merged spilled regs: " << *StackInt << '\n'; } } while (false); | |||
1185 | ||||
1186 | // Spill around uses of all RegsToSpill. | |||
1187 | for (Register Reg : RegsToSpill) | |||
1188 | spillAroundUses(Reg); | |||
1189 | ||||
1190 | // Hoisted spills may cause dead code. | |||
1191 | if (!DeadDefs.empty()) { | |||
1192 | LLVM_DEBUG(dbgs() << "Eliminating " << DeadDefs.size() << " dead defs\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Eliminating " << DeadDefs .size() << " dead defs\n"; } } while (false); | |||
1193 | Edit->eliminateDeadDefs(DeadDefs, RegsToSpill, AA); | |||
1194 | } | |||
1195 | ||||
1196 | // Finally delete the SnippetCopies. | |||
1197 | for (Register Reg : RegsToSpill) { | |||
1198 | for (MachineRegisterInfo::reg_instr_iterator | |||
1199 | RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end(); | |||
1200 | RI != E; ) { | |||
1201 | MachineInstr &MI = *(RI++); | |||
1202 | assert(SnippetCopies.count(&MI) && "Remaining use wasn't a snippet copy")(static_cast <bool> (SnippetCopies.count(&MI) && "Remaining use wasn't a snippet copy") ? void (0) : __assert_fail ("SnippetCopies.count(&MI) && \"Remaining use wasn't a snippet copy\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1202, __extension__ __PRETTY_FUNCTION__)); | |||
1203 | // FIXME: Do this with a LiveRangeEdit callback. | |||
1204 | LIS.RemoveMachineInstrFromMaps(MI); | |||
1205 | MI.eraseFromParent(); | |||
1206 | } | |||
1207 | } | |||
1208 | ||||
1209 | // Delete all spilled registers. | |||
1210 | for (Register Reg : RegsToSpill) | |||
1211 | Edit->eraseVirtReg(Reg); | |||
1212 | } | |||
1213 | ||||
1214 | void InlineSpiller::spill(LiveRangeEdit &edit) { | |||
1215 | ++NumSpilledRanges; | |||
1216 | Edit = &edit; | |||
1217 | assert(!Register::isStackSlot(edit.getReg()) &&(static_cast <bool> (!Register::isStackSlot(edit.getReg ()) && "Trying to spill a stack slot.") ? void (0) : __assert_fail ("!Register::isStackSlot(edit.getReg()) && \"Trying to spill a stack slot.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1218, __extension__ __PRETTY_FUNCTION__)) | |||
| ||||
1218 | "Trying to spill a stack slot.")(static_cast <bool> (!Register::isStackSlot(edit.getReg ()) && "Trying to spill a stack slot.") ? void (0) : __assert_fail ("!Register::isStackSlot(edit.getReg()) && \"Trying to spill a stack slot.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1218, __extension__ __PRETTY_FUNCTION__)); | |||
1219 | // Share a stack slot among all descendants of Original. | |||
1220 | Original = VRM.getOriginal(edit.getReg()); | |||
1221 | StackSlot = VRM.getStackSlot(Original); | |||
1222 | StackInt = nullptr; | |||
1223 | ||||
1224 | LLVM_DEBUG(dbgs() << "Inline spilling "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false) | |||
1225 | << TRI.getRegClassName(MRI.getRegClass(edit.getReg()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false) | |||
1226 | << ':' << edit.getParent() << "\nFrom original "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false) | |||
1227 | << printReg(Original) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false); | |||
1228 | assert(edit.getParent().isSpillable() &&(static_cast <bool> (edit.getParent().isSpillable() && "Attempting to spill already spilled value.") ? void (0) : __assert_fail ("edit.getParent().isSpillable() && \"Attempting to spill already spilled value.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1229, __extension__ __PRETTY_FUNCTION__)) | |||
1229 | "Attempting to spill already spilled value.")(static_cast <bool> (edit.getParent().isSpillable() && "Attempting to spill already spilled value.") ? void (0) : __assert_fail ("edit.getParent().isSpillable() && \"Attempting to spill already spilled value.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1229, __extension__ __PRETTY_FUNCTION__)); | |||
1230 | assert(DeadDefs.empty() && "Previous spill didn't remove dead defs")(static_cast <bool> (DeadDefs.empty() && "Previous spill didn't remove dead defs" ) ? void (0) : __assert_fail ("DeadDefs.empty() && \"Previous spill didn't remove dead defs\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1230, __extension__ __PRETTY_FUNCTION__)); | |||
1231 | ||||
1232 | collectRegsToSpill(); | |||
1233 | reMaterializeAll(); | |||
1234 | ||||
1235 | // Remat may handle everything. | |||
1236 | if (!RegsToSpill.empty()) | |||
1237 | spillAll(); | |||
1238 | ||||
1239 | Edit->calculateRegClassAndHint(MF, VRAI); | |||
1240 | } | |||
1241 | ||||
1242 | /// Optimizations after all the reg selections and spills are done. | |||
1243 | void InlineSpiller::postOptimization() { HSpiller.hoistAllSpills(); } | |||
1244 | ||||
1245 | /// When a spill is inserted, add the spill to MergeableSpills map. | |||
1246 | void HoistSpillHelper::addToMergeableSpills(MachineInstr &Spill, int StackSlot, | |||
1247 | unsigned Original) { | |||
1248 | BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator(); | |||
1249 | LiveInterval &OrigLI = LIS.getInterval(Original); | |||
1250 | // save a copy of LiveInterval in StackSlotToOrigLI because the original | |||
1251 | // LiveInterval may be cleared after all its references are spilled. | |||
1252 | if (StackSlotToOrigLI.find(StackSlot) == StackSlotToOrigLI.end()) { | |||
1253 | auto LI = std::make_unique<LiveInterval>(OrigLI.reg(), OrigLI.weight()); | |||
1254 | LI->assign(OrigLI, Allocator); | |||
1255 | StackSlotToOrigLI[StackSlot] = std::move(LI); | |||
1256 | } | |||
1257 | SlotIndex Idx = LIS.getInstructionIndex(Spill); | |||
1258 | VNInfo *OrigVNI = StackSlotToOrigLI[StackSlot]->getVNInfoAt(Idx.getRegSlot()); | |||
1259 | std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI); | |||
1260 | MergeableSpills[MIdx].insert(&Spill); | |||
1261 | } | |||
1262 | ||||
1263 | /// When a spill is removed, remove the spill from MergeableSpills map. | |||
1264 | /// Return true if the spill is removed successfully. | |||
1265 | bool HoistSpillHelper::rmFromMergeableSpills(MachineInstr &Spill, | |||
1266 | int StackSlot) { | |||
1267 | auto It = StackSlotToOrigLI.find(StackSlot); | |||
1268 | if (It == StackSlotToOrigLI.end()) | |||
1269 | return false; | |||
1270 | SlotIndex Idx = LIS.getInstructionIndex(Spill); | |||
1271 | VNInfo *OrigVNI = It->second->getVNInfoAt(Idx.getRegSlot()); | |||
1272 | std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI); | |||
1273 | return MergeableSpills[MIdx].erase(&Spill); | |||
1274 | } | |||
1275 | ||||
1276 | /// Check BB to see if it is a possible target BB to place a hoisted spill, | |||
1277 | /// i.e., there should be a living sibling of OrigReg at the insert point. | |||
1278 | bool HoistSpillHelper::isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
1279 | MachineBasicBlock &BB, Register &LiveReg) { | |||
1280 | SlotIndex Idx = IPA.getLastInsertPoint(OrigLI, BB); | |||
1281 | // The original def could be after the last insert point in the root block, | |||
1282 | // we can't hoist to here. | |||
1283 | if (Idx < OrigVNI.def) { | |||
1284 | // TODO: We could be better here. If LI is not alive in landing pad | |||
1285 | // we could hoist spill after LIP. | |||
1286 | LLVM_DEBUG(dbgs() << "can't spill in root block - def after LIP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "can't spill in root block - def after LIP\n" ; } } while (false); | |||
1287 | return false; | |||
1288 | } | |||
1289 | Register OrigReg = OrigLI.reg(); | |||
1290 | SmallSetVector<Register, 16> &Siblings = Virt2SiblingsMap[OrigReg]; | |||
1291 | assert(OrigLI.getVNInfoAt(Idx) == &OrigVNI && "Unexpected VNI")(static_cast <bool> (OrigLI.getVNInfoAt(Idx) == &OrigVNI && "Unexpected VNI") ? void (0) : __assert_fail ("OrigLI.getVNInfoAt(Idx) == &OrigVNI && \"Unexpected VNI\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1291, __extension__ __PRETTY_FUNCTION__)); | |||
1292 | ||||
1293 | for (const Register &SibReg : Siblings) { | |||
1294 | LiveInterval &LI = LIS.getInterval(SibReg); | |||
1295 | VNInfo *VNI = LI.getVNInfoAt(Idx); | |||
1296 | if (VNI) { | |||
1297 | LiveReg = SibReg; | |||
1298 | return true; | |||
1299 | } | |||
1300 | } | |||
1301 | return false; | |||
1302 | } | |||
1303 | ||||
1304 | /// Remove redundant spills in the same BB. Save those redundant spills in | |||
1305 | /// SpillsToRm, and save the spill to keep and its BB in SpillBBToSpill map. | |||
1306 | void HoistSpillHelper::rmRedundantSpills( | |||
1307 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
1308 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
1309 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) { | |||
1310 | // For each spill saw, check SpillBBToSpill[] and see if its BB already has | |||
1311 | // another spill inside. If a BB contains more than one spill, only keep the | |||
1312 | // earlier spill with smaller SlotIndex. | |||
1313 | for (const auto CurrentSpill : Spills) { | |||
1314 | MachineBasicBlock *Block = CurrentSpill->getParent(); | |||
1315 | MachineDomTreeNode *Node = MDT.getBase().getNode(Block); | |||
1316 | MachineInstr *PrevSpill = SpillBBToSpill[Node]; | |||
1317 | if (PrevSpill) { | |||
1318 | SlotIndex PIdx = LIS.getInstructionIndex(*PrevSpill); | |||
1319 | SlotIndex CIdx = LIS.getInstructionIndex(*CurrentSpill); | |||
1320 | MachineInstr *SpillToRm = (CIdx > PIdx) ? CurrentSpill : PrevSpill; | |||
1321 | MachineInstr *SpillToKeep = (CIdx > PIdx) ? PrevSpill : CurrentSpill; | |||
1322 | SpillsToRm.push_back(SpillToRm); | |||
1323 | SpillBBToSpill[MDT.getBase().getNode(Block)] = SpillToKeep; | |||
1324 | } else { | |||
1325 | SpillBBToSpill[MDT.getBase().getNode(Block)] = CurrentSpill; | |||
1326 | } | |||
1327 | } | |||
1328 | for (const auto SpillToRm : SpillsToRm) | |||
1329 | Spills.erase(SpillToRm); | |||
1330 | } | |||
1331 | ||||
1332 | /// Starting from \p Root find a top-down traversal order of the dominator | |||
1333 | /// tree to visit all basic blocks containing the elements of \p Spills. | |||
1334 | /// Redundant spills will be found and put into \p SpillsToRm at the same | |||
1335 | /// time. \p SpillBBToSpill will be populated as part of the process and | |||
1336 | /// maps a basic block to the first store occurring in the basic block. | |||
1337 | /// \post SpillsToRm.union(Spills\@post) == Spills\@pre | |||
1338 | void HoistSpillHelper::getVisitOrders( | |||
1339 | MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills, | |||
1340 | SmallVectorImpl<MachineDomTreeNode *> &Orders, | |||
1341 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
1342 | DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep, | |||
1343 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) { | |||
1344 | // The set contains all the possible BB nodes to which we may hoist | |||
1345 | // original spills. | |||
1346 | SmallPtrSet<MachineDomTreeNode *, 8> WorkSet; | |||
1347 | // Save the BB nodes on the path from the first BB node containing | |||
1348 | // non-redundant spill to the Root node. | |||
1349 | SmallPtrSet<MachineDomTreeNode *, 8> NodesOnPath; | |||
1350 | // All the spills to be hoisted must originate from a single def instruction | |||
1351 | // to the OrigReg. It means the def instruction should dominate all the spills | |||
1352 | // to be hoisted. We choose the BB where the def instruction is located as | |||
1353 | // the Root. | |||
1354 | MachineDomTreeNode *RootIDomNode = MDT[Root]->getIDom(); | |||
1355 | // For every node on the dominator tree with spill, walk up on the dominator | |||
1356 | // tree towards the Root node until it is reached. If there is other node | |||
1357 | // containing spill in the middle of the path, the previous spill saw will | |||
1358 | // be redundant and the node containing it will be removed. All the nodes on | |||
1359 | // the path starting from the first node with non-redundant spill to the Root | |||
1360 | // node will be added to the WorkSet, which will contain all the possible | |||
1361 | // locations where spills may be hoisted to after the loop below is done. | |||
1362 | for (const auto Spill : Spills) { | |||
1363 | MachineBasicBlock *Block = Spill->getParent(); | |||
1364 | MachineDomTreeNode *Node = MDT[Block]; | |||
1365 | MachineInstr *SpillToRm = nullptr; | |||
1366 | while (Node != RootIDomNode) { | |||
1367 | // If Node dominates Block, and it already contains a spill, the spill in | |||
1368 | // Block will be redundant. | |||
1369 | if (Node != MDT[Block] && SpillBBToSpill[Node]) { | |||
1370 | SpillToRm = SpillBBToSpill[MDT[Block]]; | |||
1371 | break; | |||
1372 | /// If we see the Node already in WorkSet, the path from the Node to | |||
1373 | /// the Root node must already be traversed by another spill. | |||
1374 | /// Then no need to repeat. | |||
1375 | } else if (WorkSet.count(Node)) { | |||
1376 | break; | |||
1377 | } else { | |||
1378 | NodesOnPath.insert(Node); | |||
1379 | } | |||
1380 | Node = Node->getIDom(); | |||
1381 | } | |||
1382 | if (SpillToRm) { | |||
1383 | SpillsToRm.push_back(SpillToRm); | |||
1384 | } else { | |||
1385 | // Add a BB containing the original spills to SpillsToKeep -- i.e., | |||
1386 | // set the initial status before hoisting start. The value of BBs | |||
1387 | // containing original spills is set to 0, in order to descriminate | |||
1388 | // with BBs containing hoisted spills which will be inserted to | |||
1389 | // SpillsToKeep later during hoisting. | |||
1390 | SpillsToKeep[MDT[Block]] = 0; | |||
1391 | WorkSet.insert(NodesOnPath.begin(), NodesOnPath.end()); | |||
1392 | } | |||
1393 | NodesOnPath.clear(); | |||
1394 | } | |||
1395 | ||||
1396 | // Sort the nodes in WorkSet in top-down order and save the nodes | |||
1397 | // in Orders. Orders will be used for hoisting in runHoistSpills. | |||
1398 | unsigned idx = 0; | |||
1399 | Orders.push_back(MDT.getBase().getNode(Root)); | |||
1400 | do { | |||
1401 | MachineDomTreeNode *Node = Orders[idx++]; | |||
1402 | for (MachineDomTreeNode *Child : Node->children()) { | |||
1403 | if (WorkSet.count(Child)) | |||
1404 | Orders.push_back(Child); | |||
1405 | } | |||
1406 | } while (idx != Orders.size()); | |||
1407 | assert(Orders.size() == WorkSet.size() &&(static_cast <bool> (Orders.size() == WorkSet.size() && "Orders have different size with WorkSet") ? void (0) : __assert_fail ("Orders.size() == WorkSet.size() && \"Orders have different size with WorkSet\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1408, __extension__ __PRETTY_FUNCTION__)) | |||
1408 | "Orders have different size with WorkSet")(static_cast <bool> (Orders.size() == WorkSet.size() && "Orders have different size with WorkSet") ? void (0) : __assert_fail ("Orders.size() == WorkSet.size() && \"Orders have different size with WorkSet\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1408, __extension__ __PRETTY_FUNCTION__)); | |||
1409 | ||||
1410 | #ifndef NDEBUG | |||
1411 | LLVM_DEBUG(dbgs() << "Orders size is " << Orders.size() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Orders size is " << Orders .size() << "\n"; } } while (false); | |||
1412 | SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin(); | |||
1413 | for (; RIt != Orders.rend(); RIt++) | |||
1414 | LLVM_DEBUG(dbgs() << "BB" << (*RIt)->getBlock()->getNumber() << ",")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "BB" << (*RIt)->getBlock ()->getNumber() << ","; } } while (false); | |||
1415 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\n"; } } while (false); | |||
1416 | #endif | |||
1417 | } | |||
1418 | ||||
1419 | /// Try to hoist spills according to BB hotness. The spills to removed will | |||
1420 | /// be saved in \p SpillsToRm. The spills to be inserted will be saved in | |||
1421 | /// \p SpillsToIns. | |||
1422 | void HoistSpillHelper::runHoistSpills( | |||
1423 | LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
1424 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
1425 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
1426 | DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns) { | |||
1427 | // Visit order of dominator tree nodes. | |||
1428 | SmallVector<MachineDomTreeNode *, 32> Orders; | |||
1429 | // SpillsToKeep contains all the nodes where spills are to be inserted | |||
1430 | // during hoisting. If the spill to be inserted is an original spill | |||
1431 | // (not a hoisted one), the value of the map entry is 0. If the spill | |||
1432 | // is a hoisted spill, the value of the map entry is the VReg to be used | |||
1433 | // as the source of the spill. | |||
1434 | DenseMap<MachineDomTreeNode *, unsigned> SpillsToKeep; | |||
1435 | // Map from BB to the first spill inside of it. | |||
1436 | DenseMap<MachineDomTreeNode *, MachineInstr *> SpillBBToSpill; | |||
1437 | ||||
1438 | rmRedundantSpills(Spills, SpillsToRm, SpillBBToSpill); | |||
1439 | ||||
1440 | MachineBasicBlock *Root = LIS.getMBBFromIndex(OrigVNI.def); | |||
1441 | getVisitOrders(Root, Spills, Orders, SpillsToRm, SpillsToKeep, | |||
1442 | SpillBBToSpill); | |||
1443 | ||||
1444 | // SpillsInSubTreeMap keeps the map from a dom tree node to a pair of | |||
1445 | // nodes set and the cost of all the spills inside those nodes. | |||
1446 | // The nodes set are the locations where spills are to be inserted | |||
1447 | // in the subtree of current node. | |||
1448 | using NodesCostPair = | |||
1449 | std::pair<SmallPtrSet<MachineDomTreeNode *, 16>, BlockFrequency>; | |||
1450 | DenseMap<MachineDomTreeNode *, NodesCostPair> SpillsInSubTreeMap; | |||
1451 | ||||
1452 | // Iterate Orders set in reverse order, which will be a bottom-up order | |||
1453 | // in the dominator tree. Once we visit a dom tree node, we know its | |||
1454 | // children have already been visited and the spill locations in the | |||
1455 | // subtrees of all the children have been determined. | |||
1456 | SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin(); | |||
1457 | for (; RIt != Orders.rend(); RIt++) { | |||
1458 | MachineBasicBlock *Block = (*RIt)->getBlock(); | |||
1459 | ||||
1460 | // If Block contains an original spill, simply continue. | |||
1461 | if (SpillsToKeep.find(*RIt) != SpillsToKeep.end() && !SpillsToKeep[*RIt]) { | |||
1462 | SpillsInSubTreeMap[*RIt].first.insert(*RIt); | |||
1463 | // SpillsInSubTreeMap[*RIt].second contains the cost of spill. | |||
1464 | SpillsInSubTreeMap[*RIt].second = MBFI.getBlockFreq(Block); | |||
1465 | continue; | |||
1466 | } | |||
1467 | ||||
1468 | // Collect spills in subtree of current node (*RIt) to | |||
1469 | // SpillsInSubTreeMap[*RIt].first. | |||
1470 | for (MachineDomTreeNode *Child : (*RIt)->children()) { | |||
1471 | if (SpillsInSubTreeMap.find(Child) == SpillsInSubTreeMap.end()) | |||
1472 | continue; | |||
1473 | // The stmt "SpillsInSubTree = SpillsInSubTreeMap[*RIt].first" below | |||
1474 | // should be placed before getting the begin and end iterators of | |||
1475 | // SpillsInSubTreeMap[Child].first, or else the iterators may be | |||
1476 | // invalidated when SpillsInSubTreeMap[*RIt] is seen the first time | |||
1477 | // and the map grows and then the original buckets in the map are moved. | |||
1478 | SmallPtrSet<MachineDomTreeNode *, 16> &SpillsInSubTree = | |||
1479 | SpillsInSubTreeMap[*RIt].first; | |||
1480 | BlockFrequency &SubTreeCost = SpillsInSubTreeMap[*RIt].second; | |||
1481 | SubTreeCost += SpillsInSubTreeMap[Child].second; | |||
1482 | auto BI = SpillsInSubTreeMap[Child].first.begin(); | |||
1483 | auto EI = SpillsInSubTreeMap[Child].first.end(); | |||
1484 | SpillsInSubTree.insert(BI, EI); | |||
1485 | SpillsInSubTreeMap.erase(Child); | |||
1486 | } | |||
1487 | ||||
1488 | SmallPtrSet<MachineDomTreeNode *, 16> &SpillsInSubTree = | |||
1489 | SpillsInSubTreeMap[*RIt].first; | |||
1490 | BlockFrequency &SubTreeCost = SpillsInSubTreeMap[*RIt].second; | |||
1491 | // No spills in subtree, simply continue. | |||
1492 | if (SpillsInSubTree.empty()) | |||
1493 | continue; | |||
1494 | ||||
1495 | // Check whether Block is a possible candidate to insert spill. | |||
1496 | Register LiveReg; | |||
1497 | if (!isSpillCandBB(OrigLI, OrigVNI, *Block, LiveReg)) | |||
1498 | continue; | |||
1499 | ||||
1500 | // If there are multiple spills that could be merged, bias a little | |||
1501 | // to hoist the spill. | |||
1502 | BranchProbability MarginProb = (SpillsInSubTree.size() > 1) | |||
1503 | ? BranchProbability(9, 10) | |||
1504 | : BranchProbability(1, 1); | |||
1505 | if (SubTreeCost > MBFI.getBlockFreq(Block) * MarginProb) { | |||
1506 | // Hoist: Move spills to current Block. | |||
1507 | for (const auto SpillBB : SpillsInSubTree) { | |||
1508 | // When SpillBB is a BB contains original spill, insert the spill | |||
1509 | // to SpillsToRm. | |||
1510 | if (SpillsToKeep.find(SpillBB) != SpillsToKeep.end() && | |||
1511 | !SpillsToKeep[SpillBB]) { | |||
1512 | MachineInstr *SpillToRm = SpillBBToSpill[SpillBB]; | |||
1513 | SpillsToRm.push_back(SpillToRm); | |||
1514 | } | |||
1515 | // SpillBB will not contain spill anymore, remove it from SpillsToKeep. | |||
1516 | SpillsToKeep.erase(SpillBB); | |||
1517 | } | |||
1518 | // Current Block is the BB containing the new hoisted spill. Add it to | |||
1519 | // SpillsToKeep. LiveReg is the source of the new spill. | |||
1520 | SpillsToKeep[*RIt] = LiveReg; | |||
1521 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
1522 | dbgs() << "spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
1523 | for (const auto Rspill : SpillsInSubTree)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
1524 | dbgs() << Rspill->getBlock()->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
1525 | dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
1526 | << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
1527 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false); | |||
1528 | SpillsInSubTree.clear(); | |||
1529 | SpillsInSubTree.insert(*RIt); | |||
1530 | SubTreeCost = MBFI.getBlockFreq(Block); | |||
1531 | } | |||
1532 | } | |||
1533 | // For spills in SpillsToKeep with LiveReg set (i.e., not original spill), | |||
1534 | // save them to SpillsToIns. | |||
1535 | for (const auto &Ent : SpillsToKeep) { | |||
1536 | if (Ent.second) | |||
1537 | SpillsToIns[Ent.first->getBlock()] = Ent.second; | |||
1538 | } | |||
1539 | } | |||
1540 | ||||
1541 | /// For spills with equal values, remove redundant spills and hoist those left | |||
1542 | /// to less hot spots. | |||
1543 | /// | |||
1544 | /// Spills with equal values will be collected into the same set in | |||
1545 | /// MergeableSpills when spill is inserted. These equal spills are originated | |||
1546 | /// from the same defining instruction and are dominated by the instruction. | |||
1547 | /// Before hoisting all the equal spills, redundant spills inside in the same | |||
1548 | /// BB are first marked to be deleted. Then starting from the spills left, walk | |||
1549 | /// up on the dominator tree towards the Root node where the define instruction | |||
1550 | /// is located, mark the dominated spills to be deleted along the way and | |||
1551 | /// collect the BB nodes on the path from non-dominated spills to the define | |||
1552 | /// instruction into a WorkSet. The nodes in WorkSet are the candidate places | |||
1553 | /// where we are considering to hoist the spills. We iterate the WorkSet in | |||
1554 | /// bottom-up order, and for each node, we will decide whether to hoist spills | |||
1555 | /// inside its subtree to that node. In this way, we can get benefit locally | |||
1556 | /// even if hoisting all the equal spills to one cold place is impossible. | |||
1557 | void HoistSpillHelper::hoistAllSpills() { | |||
1558 | SmallVector<Register, 4> NewVRegs; | |||
1559 | LiveRangeEdit Edit(nullptr, NewVRegs, MF, LIS, &VRM, this); | |||
1560 | ||||
1561 | for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) { | |||
1562 | Register Reg = Register::index2VirtReg(i); | |||
1563 | Register Original = VRM.getPreSplitReg(Reg); | |||
1564 | if (!MRI.def_empty(Reg)) | |||
1565 | Virt2SiblingsMap[Original].insert(Reg); | |||
1566 | } | |||
1567 | ||||
1568 | // Each entry in MergeableSpills contains a spill set with equal values. | |||
1569 | for (auto &Ent : MergeableSpills) { | |||
1570 | int Slot = Ent.first.first; | |||
1571 | LiveInterval &OrigLI = *StackSlotToOrigLI[Slot]; | |||
1572 | VNInfo *OrigVNI = Ent.first.second; | |||
1573 | SmallPtrSet<MachineInstr *, 16> &EqValSpills = Ent.second; | |||
1574 | if (Ent.second.empty()) | |||
1575 | continue; | |||
1576 | ||||
1577 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
1578 | dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
1579 | << "Equal spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
1580 | for (const auto spill : EqValSpills)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
1581 | dbgs() << spill->getParent()->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
1582 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
1583 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false); | |||
1584 | ||||
1585 | // SpillsToRm is the spill set to be removed from EqValSpills. | |||
1586 | SmallVector<MachineInstr *, 16> SpillsToRm; | |||
1587 | // SpillsToIns is the spill set to be newly inserted after hoisting. | |||
1588 | DenseMap<MachineBasicBlock *, unsigned> SpillsToIns; | |||
1589 | ||||
1590 | runHoistSpills(OrigLI, *OrigVNI, EqValSpills, SpillsToRm, SpillsToIns); | |||
1591 | ||||
1592 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1593 | dbgs() << "Finally inserted spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1594 | for (const auto &Ispill : SpillsToIns)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1595 | dbgs() << Ispill.first->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1596 | dbgs() << "\nFinally removed spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1597 | for (const auto Rspill : SpillsToRm)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1598 | dbgs() << Rspill->getParent()->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1599 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
1600 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false); | |||
1601 | ||||
1602 | // Stack live range update. | |||
1603 | LiveInterval &StackIntvl = LSS.getInterval(Slot); | |||
1604 | if (!SpillsToIns.empty() || !SpillsToRm.empty()) | |||
1605 | StackIntvl.MergeValueInAsValue(OrigLI, OrigVNI, | |||
1606 | StackIntvl.getValNumInfo(0)); | |||
1607 | ||||
1608 | // Insert hoisted spills. | |||
1609 | for (auto const &Insert : SpillsToIns) { | |||
1610 | MachineBasicBlock *BB = Insert.first; | |||
1611 | Register LiveReg = Insert.second; | |||
1612 | MachineBasicBlock::iterator MII = IPA.getLastInsertPointIter(OrigLI, *BB); | |||
1613 | MachineInstrSpan MIS(MII, BB); | |||
1614 | TII.storeRegToStackSlot(*BB, MII, LiveReg, false, Slot, | |||
1615 | MRI.getRegClass(LiveReg), &TRI); | |||
1616 | LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MII); | |||
1617 | for (const MachineInstr &MI : make_range(MIS.begin(), MII)) | |||
1618 | getVDefInterval(MI, LIS); | |||
1619 | ++NumSpills; | |||
1620 | } | |||
1621 | ||||
1622 | // Remove redundant spills or change them to dead instructions. | |||
1623 | NumSpills -= SpillsToRm.size(); | |||
1624 | for (auto const RMEnt : SpillsToRm) { | |||
1625 | RMEnt->setDesc(TII.get(TargetOpcode::KILL)); | |||
1626 | for (unsigned i = RMEnt->getNumOperands(); i; --i) { | |||
1627 | MachineOperand &MO = RMEnt->getOperand(i - 1); | |||
1628 | if (MO.isReg() && MO.isImplicit() && MO.isDef() && !MO.isDead()) | |||
1629 | RMEnt->RemoveOperand(i - 1); | |||
1630 | } | |||
1631 | } | |||
1632 | Edit.eliminateDeadDefs(SpillsToRm, None, AA); | |||
1633 | } | |||
1634 | } | |||
1635 | ||||
1636 | /// For VirtReg clone, the \p New register should have the same physreg or | |||
1637 | /// stackslot as the \p old register. | |||
1638 | void HoistSpillHelper::LRE_DidCloneVirtReg(Register New, Register Old) { | |||
1639 | if (VRM.hasPhys(Old)) | |||
1640 | VRM.assignVirt2Phys(New, VRM.getPhys(Old)); | |||
1641 | else if (VRM.getStackSlot(Old) != VirtRegMap::NO_STACK_SLOT) | |||
1642 | VRM.assignVirt2StackSlot(New, VRM.getStackSlot(Old)); | |||
1643 | else | |||
1644 | llvm_unreachable("VReg should be assigned either physreg or stackslot")::llvm::llvm_unreachable_internal("VReg should be assigned either physreg or stackslot" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/lib/CodeGen/InlineSpiller.cpp" , 1644); | |||
1645 | if (VRM.hasShape(Old)) | |||
1646 | VRM.assignVirt2Shape(New, VRM.getShape(Old)); | |||
1647 | } |
1 | //===- llvm/ADT/STLExtras.h - Useful STL related functions ------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains some templates that are useful if you are working with the |
10 | // STL at all. |
11 | // |
12 | // No library is required when using these functions. |
13 | // |
14 | //===----------------------------------------------------------------------===// |
15 | |
16 | #ifndef LLVM_ADT_STLEXTRAS_H |
17 | #define LLVM_ADT_STLEXTRAS_H |
18 | |
19 | #include "llvm/ADT/Optional.h" |
20 | #include "llvm/ADT/STLForwardCompat.h" |
21 | #include "llvm/ADT/iterator.h" |
22 | #include "llvm/ADT/iterator_range.h" |
23 | #include "llvm/Config/abi-breaking.h" |
24 | #include "llvm/Support/ErrorHandling.h" |
25 | #include <algorithm> |
26 | #include <cassert> |
27 | #include <cstddef> |
28 | #include <cstdint> |
29 | #include <cstdlib> |
30 | #include <functional> |
31 | #include <initializer_list> |
32 | #include <iterator> |
33 | #include <limits> |
34 | #include <memory> |
35 | #include <tuple> |
36 | #include <type_traits> |
37 | #include <utility> |
38 | |
39 | #ifdef EXPENSIVE_CHECKS |
40 | #include <random> // for std::mt19937 |
41 | #endif |
42 | |
43 | namespace llvm { |
44 | |
45 | // Only used by compiler if both template types are the same. Useful when |
46 | // using SFINAE to test for the existence of member functions. |
47 | template <typename T, T> struct SameType; |
48 | |
49 | namespace detail { |
50 | |
51 | template <typename RangeT> |
52 | using IterOfRange = decltype(std::begin(std::declval<RangeT &>())); |
53 | |
54 | template <typename RangeT> |
55 | using ValueOfRange = typename std::remove_reference<decltype( |
56 | *std::begin(std::declval<RangeT &>()))>::type; |
57 | |
58 | } // end namespace detail |
59 | |
60 | //===----------------------------------------------------------------------===// |
61 | // Extra additions to <type_traits> |
62 | //===----------------------------------------------------------------------===// |
63 | |
64 | template <typename T> struct make_const_ptr { |
65 | using type = |
66 | typename std::add_pointer<typename std::add_const<T>::type>::type; |
67 | }; |
68 | |
69 | template <typename T> struct make_const_ref { |
70 | using type = typename std::add_lvalue_reference< |
71 | typename std::add_const<T>::type>::type; |
72 | }; |
73 | |
74 | namespace detail { |
75 | template <typename...> using void_t = void; |
76 | template <class, template <class...> class Op, class... Args> struct detector { |
77 | using value_t = std::false_type; |
78 | }; |
79 | template <template <class...> class Op, class... Args> |
80 | struct detector<void_t<Op<Args...>>, Op, Args...> { |
81 | using value_t = std::true_type; |
82 | }; |
83 | } // end namespace detail |
84 | |
85 | /// Detects if a given trait holds for some set of arguments 'Args'. |
86 | /// For example, the given trait could be used to detect if a given type |
87 | /// has a copy assignment operator: |
88 | /// template<class T> |
89 | /// using has_copy_assign_t = decltype(std::declval<T&>() |
90 | /// = std::declval<const T&>()); |
91 | /// bool fooHasCopyAssign = is_detected<has_copy_assign_t, FooClass>::value; |
92 | template <template <class...> class Op, class... Args> |
93 | using is_detected = typename detail::detector<void, Op, Args...>::value_t; |
94 | |
95 | namespace detail { |
96 | template <typename Callable, typename... Args> |
97 | using is_invocable = |
98 | decltype(std::declval<Callable &>()(std::declval<Args>()...)); |
99 | } // namespace detail |
100 | |
101 | /// Check if a Callable type can be invoked with the given set of arg types. |
102 | template <typename Callable, typename... Args> |
103 | using is_invocable = is_detected<detail::is_invocable, Callable, Args...>; |
104 | |
105 | /// This class provides various trait information about a callable object. |
106 | /// * To access the number of arguments: Traits::num_args |
107 | /// * To access the type of an argument: Traits::arg_t<Index> |
108 | /// * To access the type of the result: Traits::result_t |
109 | template <typename T, bool isClass = std::is_class<T>::value> |
110 | struct function_traits : public function_traits<decltype(&T::operator())> {}; |
111 | |
112 | /// Overload for class function types. |
113 | template <typename ClassType, typename ReturnType, typename... Args> |
114 | struct function_traits<ReturnType (ClassType::*)(Args...) const, false> { |
115 | /// The number of arguments to this function. |
116 | enum { num_args = sizeof...(Args) }; |
117 | |
118 | /// The result type of this function. |
119 | using result_t = ReturnType; |
120 | |
121 | /// The type of an argument to this function. |
122 | template <size_t Index> |
123 | using arg_t = typename std::tuple_element<Index, std::tuple<Args...>>::type; |
124 | }; |
125 | /// Overload for class function types. |
126 | template <typename ClassType, typename ReturnType, typename... Args> |
127 | struct function_traits<ReturnType (ClassType::*)(Args...), false> |
128 | : function_traits<ReturnType (ClassType::*)(Args...) const> {}; |
129 | /// Overload for non-class function types. |
130 | template <typename ReturnType, typename... Args> |
131 | struct function_traits<ReturnType (*)(Args...), false> { |
132 | /// The number of arguments to this function. |
133 | enum { num_args = sizeof...(Args) }; |
134 | |
135 | /// The result type of this function. |
136 | using result_t = ReturnType; |
137 | |
138 | /// The type of an argument to this function. |
139 | template <size_t i> |
140 | using arg_t = typename std::tuple_element<i, std::tuple<Args...>>::type; |
141 | }; |
142 | /// Overload for non-class function type references. |
143 | template <typename ReturnType, typename... Args> |
144 | struct function_traits<ReturnType (&)(Args...), false> |
145 | : public function_traits<ReturnType (*)(Args...)> {}; |
146 | |
147 | //===----------------------------------------------------------------------===// |
148 | // Extra additions to <functional> |
149 | //===----------------------------------------------------------------------===// |
150 | |
151 | template <class Ty> struct identity { |
152 | using argument_type = Ty; |
153 | |
154 | Ty &operator()(Ty &self) const { |
155 | return self; |
156 | } |
157 | const Ty &operator()(const Ty &self) const { |
158 | return self; |
159 | } |
160 | }; |
161 | |
162 | /// An efficient, type-erasing, non-owning reference to a callable. This is |
163 | /// intended for use as the type of a function parameter that is not used |
164 | /// after the function in question returns. |
165 | /// |
166 | /// This class does not own the callable, so it is not in general safe to store |
167 | /// a function_ref. |
168 | template<typename Fn> class function_ref; |
169 | |
170 | template<typename Ret, typename ...Params> |
171 | class function_ref<Ret(Params...)> { |
172 | Ret (*callback)(intptr_t callable, Params ...params) = nullptr; |
173 | intptr_t callable; |
174 | |
175 | template<typename Callable> |
176 | static Ret callback_fn(intptr_t callable, Params ...params) { |
177 | return (*reinterpret_cast<Callable*>(callable))( |
178 | std::forward<Params>(params)...); |
179 | } |
180 | |
181 | public: |
182 | function_ref() = default; |
183 | function_ref(std::nullptr_t) {} |
184 | |
185 | template <typename Callable> |
186 | function_ref( |
187 | Callable &&callable, |
188 | // This is not the copy-constructor. |
189 | std::enable_if_t<!std::is_same<remove_cvref_t<Callable>, |
190 | function_ref>::value> * = nullptr, |
191 | // Functor must be callable and return a suitable type. |
192 | std::enable_if_t<std::is_void<Ret>::value || |
193 | std::is_convertible<decltype(std::declval<Callable>()( |
194 | std::declval<Params>()...)), |
195 | Ret>::value> * = nullptr) |
196 | : callback(callback_fn<typename std::remove_reference<Callable>::type>), |
197 | callable(reinterpret_cast<intptr_t>(&callable)) {} |
198 | |
199 | Ret operator()(Params ...params) const { |
200 | return callback(callable, std::forward<Params>(params)...); |
201 | } |
202 | |
203 | explicit operator bool() const { return callback; } |
204 | }; |
205 | |
206 | //===----------------------------------------------------------------------===// |
207 | // Extra additions to <iterator> |
208 | //===----------------------------------------------------------------------===// |
209 | |
210 | namespace adl_detail { |
211 | |
212 | using std::begin; |
213 | |
214 | template <typename ContainerTy> |
215 | decltype(auto) adl_begin(ContainerTy &&container) { |
216 | return begin(std::forward<ContainerTy>(container)); |
217 | } |
218 | |
219 | using std::end; |
220 | |
221 | template <typename ContainerTy> |
222 | decltype(auto) adl_end(ContainerTy &&container) { |
223 | return end(std::forward<ContainerTy>(container)); |
224 | } |
225 | |
226 | using std::swap; |
227 | |
228 | template <typename T> |
229 | void adl_swap(T &&lhs, T &&rhs) noexcept(noexcept(swap(std::declval<T>(), |
230 | std::declval<T>()))) { |
231 | swap(std::forward<T>(lhs), std::forward<T>(rhs)); |
232 | } |
233 | |
234 | } // end namespace adl_detail |
235 | |
236 | template <typename ContainerTy> |
237 | decltype(auto) adl_begin(ContainerTy &&container) { |
238 | return adl_detail::adl_begin(std::forward<ContainerTy>(container)); |
239 | } |
240 | |
241 | template <typename ContainerTy> |
242 | decltype(auto) adl_end(ContainerTy &&container) { |
243 | return adl_detail::adl_end(std::forward<ContainerTy>(container)); |
244 | } |
245 | |
246 | template <typename T> |
247 | void adl_swap(T &&lhs, T &&rhs) noexcept( |
248 | noexcept(adl_detail::adl_swap(std::declval<T>(), std::declval<T>()))) { |
249 | adl_detail::adl_swap(std::forward<T>(lhs), std::forward<T>(rhs)); |
250 | } |
251 | |
252 | /// Test whether \p RangeOrContainer is empty. Similar to C++17 std::empty. |
253 | template <typename T> |
254 | constexpr bool empty(const T &RangeOrContainer) { |
255 | return adl_begin(RangeOrContainer) == adl_end(RangeOrContainer); |
256 | } |
257 | |
258 | /// Returns true if the given container only contains a single element. |
259 | template <typename ContainerTy> bool hasSingleElement(ContainerTy &&C) { |
260 | auto B = std::begin(C), E = std::end(C); |
261 | return B != E && std::next(B) == E; |
262 | } |
263 | |
264 | /// Return a range covering \p RangeOrContainer with the first N elements |
265 | /// excluded. |
266 | template <typename T> auto drop_begin(T &&RangeOrContainer, size_t N = 1) { |
267 | return make_range(std::next(adl_begin(RangeOrContainer), N), |
268 | adl_end(RangeOrContainer)); |
269 | } |
270 | |
271 | // mapped_iterator - This is a simple iterator adapter that causes a function to |
272 | // be applied whenever operator* is invoked on the iterator. |
273 | |
274 | template <typename ItTy, typename FuncTy, |
275 | typename ReferenceTy = |
276 | decltype(std::declval<FuncTy>()(*std::declval<ItTy>()))> |
277 | class mapped_iterator |
278 | : public iterator_adaptor_base< |
279 | mapped_iterator<ItTy, FuncTy>, ItTy, |
280 | typename std::iterator_traits<ItTy>::iterator_category, |
281 | std::remove_reference_t<ReferenceTy>, |
282 | typename std::iterator_traits<ItTy>::difference_type, |
283 | std::remove_reference_t<ReferenceTy> *, ReferenceTy> { |
284 | public: |
285 | mapped_iterator(ItTy U, FuncTy F) |
286 | : mapped_iterator::iterator_adaptor_base(std::move(U)), F(std::move(F)) {} |
287 | |
288 | ItTy getCurrent() { return this->I; } |
289 | |
290 | const FuncTy &getFunction() const { return F; } |
291 | |
292 | ReferenceTy operator*() const { return F(*this->I); } |
293 | |
294 | private: |
295 | FuncTy F; |
296 | }; |
297 | |
298 | // map_iterator - Provide a convenient way to create mapped_iterators, just like |
299 | // make_pair is useful for creating pairs... |
300 | template <class ItTy, class FuncTy> |
301 | inline mapped_iterator<ItTy, FuncTy> map_iterator(ItTy I, FuncTy F) { |
302 | return mapped_iterator<ItTy, FuncTy>(std::move(I), std::move(F)); |
303 | } |
304 | |
305 | template <class ContainerTy, class FuncTy> |
306 | auto map_range(ContainerTy &&C, FuncTy F) { |
307 | return make_range(map_iterator(C.begin(), F), map_iterator(C.end(), F)); |
308 | } |
309 | |
310 | /// Helper to determine if type T has a member called rbegin(). |
311 | template <typename Ty> class has_rbegin_impl { |
312 | using yes = char[1]; |
313 | using no = char[2]; |
314 | |
315 | template <typename Inner> |
316 | static yes& test(Inner *I, decltype(I->rbegin()) * = nullptr); |
317 | |
318 | template <typename> |
319 | static no& test(...); |
320 | |
321 | public: |
322 | static const bool value = sizeof(test<Ty>(nullptr)) == sizeof(yes); |
323 | }; |
324 | |
325 | /// Metafunction to determine if T& or T has a member called rbegin(). |
326 | template <typename Ty> |
327 | struct has_rbegin : has_rbegin_impl<typename std::remove_reference<Ty>::type> { |
328 | }; |
329 | |
330 | // Returns an iterator_range over the given container which iterates in reverse. |
331 | // Note that the container must have rbegin()/rend() methods for this to work. |
332 | template <typename ContainerTy> |
333 | auto reverse(ContainerTy &&C, |
334 | std::enable_if_t<has_rbegin<ContainerTy>::value> * = nullptr) { |
335 | return make_range(C.rbegin(), C.rend()); |
336 | } |
337 | |
338 | // Returns a std::reverse_iterator wrapped around the given iterator. |
339 | template <typename IteratorTy> |
340 | std::reverse_iterator<IteratorTy> make_reverse_iterator(IteratorTy It) { |
341 | return std::reverse_iterator<IteratorTy>(It); |
342 | } |
343 | |
344 | // Returns an iterator_range over the given container which iterates in reverse. |
345 | // Note that the container must have begin()/end() methods which return |
346 | // bidirectional iterators for this to work. |
347 | template <typename ContainerTy> |
348 | auto reverse(ContainerTy &&C, |
349 | std::enable_if_t<!has_rbegin<ContainerTy>::value> * = nullptr) { |
350 | return make_range(llvm::make_reverse_iterator(std::end(C)), |
351 | llvm::make_reverse_iterator(std::begin(C))); |
352 | } |
353 | |
354 | /// An iterator adaptor that filters the elements of given inner iterators. |
355 | /// |
356 | /// The predicate parameter should be a callable object that accepts the wrapped |
357 | /// iterator's reference type and returns a bool. When incrementing or |
358 | /// decrementing the iterator, it will call the predicate on each element and |
359 | /// skip any where it returns false. |
360 | /// |
361 | /// \code |
362 | /// int A[] = { 1, 2, 3, 4 }; |
363 | /// auto R = make_filter_range(A, [](int N) { return N % 2 == 1; }); |
364 | /// // R contains { 1, 3 }. |
365 | /// \endcode |
366 | /// |
367 | /// Note: filter_iterator_base implements support for forward iteration. |
368 | /// filter_iterator_impl exists to provide support for bidirectional iteration, |
369 | /// conditional on whether the wrapped iterator supports it. |
370 | template <typename WrappedIteratorT, typename PredicateT, typename IterTag> |
371 | class filter_iterator_base |
372 | : public iterator_adaptor_base< |
373 | filter_iterator_base<WrappedIteratorT, PredicateT, IterTag>, |
374 | WrappedIteratorT, |
375 | typename std::common_type< |
376 | IterTag, typename std::iterator_traits< |
377 | WrappedIteratorT>::iterator_category>::type> { |
378 | using BaseT = iterator_adaptor_base< |
379 | filter_iterator_base<WrappedIteratorT, PredicateT, IterTag>, |
380 | WrappedIteratorT, |
381 | typename std::common_type< |
382 | IterTag, typename std::iterator_traits< |
383 | WrappedIteratorT>::iterator_category>::type>; |
384 | |
385 | protected: |
386 | WrappedIteratorT End; |
387 | PredicateT Pred; |
388 | |
389 | void findNextValid() { |
390 | while (this->I != End && !Pred(*this->I)) |
391 | BaseT::operator++(); |
392 | } |
393 | |
394 | // Construct the iterator. The begin iterator needs to know where the end |
395 | // is, so that it can properly stop when it gets there. The end iterator only |
396 | // needs the predicate to support bidirectional iteration. |
397 | filter_iterator_base(WrappedIteratorT Begin, WrappedIteratorT End, |
398 | PredicateT Pred) |
399 | : BaseT(Begin), End(End), Pred(Pred) { |
400 | findNextValid(); |
401 | } |
402 | |
403 | public: |
404 | using BaseT::operator++; |
405 | |
406 | filter_iterator_base &operator++() { |
407 | BaseT::operator++(); |
408 | findNextValid(); |
409 | return *this; |
410 | } |
411 | }; |
412 | |
413 | /// Specialization of filter_iterator_base for forward iteration only. |
414 | template <typename WrappedIteratorT, typename PredicateT, |
415 | typename IterTag = std::forward_iterator_tag> |
416 | class filter_iterator_impl |
417 | : public filter_iterator_base<WrappedIteratorT, PredicateT, IterTag> { |
418 | using BaseT = filter_iterator_base<WrappedIteratorT, PredicateT, IterTag>; |
419 | |
420 | public: |
421 | filter_iterator_impl(WrappedIteratorT Begin, WrappedIteratorT End, |
422 | PredicateT Pred) |
423 | : BaseT(Begin, End, Pred) {} |
424 | }; |
425 | |
426 | /// Specialization of filter_iterator_base for bidirectional iteration. |
427 | template <typename WrappedIteratorT, typename PredicateT> |
428 | class filter_iterator_impl<WrappedIteratorT, PredicateT, |
429 | std::bidirectional_iterator_tag> |
430 | : public filter_iterator_base<WrappedIteratorT, PredicateT, |
431 | std::bidirectional_iterator_tag> { |
432 | using BaseT = filter_iterator_base<WrappedIteratorT, PredicateT, |
433 | std::bidirectional_iterator_tag>; |
434 | void findPrevValid() { |
435 | while (!this->Pred(*this->I)) |
436 | BaseT::operator--(); |
437 | } |
438 | |
439 | public: |
440 | using BaseT::operator--; |
441 | |
442 | filter_iterator_impl(WrappedIteratorT Begin, WrappedIteratorT End, |
443 | PredicateT Pred) |
444 | : BaseT(Begin, End, Pred) {} |
445 | |
446 | filter_iterator_impl &operator--() { |
447 | BaseT::operator--(); |
448 | findPrevValid(); |
449 | return *this; |
450 | } |
451 | }; |
452 | |
453 | namespace detail { |
454 | |
455 | template <bool is_bidirectional> struct fwd_or_bidi_tag_impl { |
456 | using type = std::forward_iterator_tag; |
457 | }; |
458 | |
459 | template <> struct fwd_or_bidi_tag_impl<true> { |
460 | using type = std::bidirectional_iterator_tag; |
461 | }; |
462 | |
463 | /// Helper which sets its type member to forward_iterator_tag if the category |
464 | /// of \p IterT does not derive from bidirectional_iterator_tag, and to |
465 | /// bidirectional_iterator_tag otherwise. |
466 | template <typename IterT> struct fwd_or_bidi_tag { |
467 | using type = typename fwd_or_bidi_tag_impl<std::is_base_of< |
468 | std::bidirectional_iterator_tag, |
469 | typename std::iterator_traits<IterT>::iterator_category>::value>::type; |
470 | }; |
471 | |
472 | } // namespace detail |
473 | |
474 | /// Defines filter_iterator to a suitable specialization of |
475 | /// filter_iterator_impl, based on the underlying iterator's category. |
476 | template <typename WrappedIteratorT, typename PredicateT> |
477 | using filter_iterator = filter_iterator_impl< |
478 | WrappedIteratorT, PredicateT, |
479 | typename detail::fwd_or_bidi_tag<WrappedIteratorT>::type>; |
480 | |
481 | /// Convenience function that takes a range of elements and a predicate, |
482 | /// and return a new filter_iterator range. |
483 | /// |
484 | /// FIXME: Currently if RangeT && is a rvalue reference to a temporary, the |
485 | /// lifetime of that temporary is not kept by the returned range object, and the |
486 | /// temporary is going to be dropped on the floor after the make_iterator_range |
487 | /// full expression that contains this function call. |
488 | template <typename RangeT, typename PredicateT> |
489 | iterator_range<filter_iterator<detail::IterOfRange<RangeT>, PredicateT>> |
490 | make_filter_range(RangeT &&Range, PredicateT Pred) { |
491 | using FilterIteratorT = |
492 | filter_iterator<detail::IterOfRange<RangeT>, PredicateT>; |
493 | return make_range( |
494 | FilterIteratorT(std::begin(std::forward<RangeT>(Range)), |
495 | std::end(std::forward<RangeT>(Range)), Pred), |
496 | FilterIteratorT(std::end(std::forward<RangeT>(Range)), |
497 | std::end(std::forward<RangeT>(Range)), Pred)); |
498 | } |
499 | |
500 | /// A pseudo-iterator adaptor that is designed to implement "early increment" |
501 | /// style loops. |
502 | /// |
503 | /// This is *not a normal iterator* and should almost never be used directly. It |
504 | /// is intended primarily to be used with range based for loops and some range |
505 | /// algorithms. |
506 | /// |
507 | /// The iterator isn't quite an `OutputIterator` or an `InputIterator` but |
508 | /// somewhere between them. The constraints of these iterators are: |
509 | /// |
510 | /// - On construction or after being incremented, it is comparable and |
511 | /// dereferencable. It is *not* incrementable. |
512 | /// - After being dereferenced, it is neither comparable nor dereferencable, it |
513 | /// is only incrementable. |
514 | /// |
515 | /// This means you can only dereference the iterator once, and you can only |
516 | /// increment it once between dereferences. |
517 | template <typename WrappedIteratorT> |
518 | class early_inc_iterator_impl |
519 | : public iterator_adaptor_base<early_inc_iterator_impl<WrappedIteratorT>, |
520 | WrappedIteratorT, std::input_iterator_tag> { |
521 | using BaseT = |
522 | iterator_adaptor_base<early_inc_iterator_impl<WrappedIteratorT>, |
523 | WrappedIteratorT, std::input_iterator_tag>; |
524 | |
525 | using PointerT = typename std::iterator_traits<WrappedIteratorT>::pointer; |
526 | |
527 | protected: |
528 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS1 |
529 | bool IsEarlyIncremented = false; |
530 | #endif |
531 | |
532 | public: |
533 | early_inc_iterator_impl(WrappedIteratorT I) : BaseT(I) {} |
534 | |
535 | using BaseT::operator*; |
536 | decltype(*std::declval<WrappedIteratorT>()) operator*() { |
537 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS1 |
538 | assert(!IsEarlyIncremented && "Cannot dereference twice!")(static_cast <bool> (!IsEarlyIncremented && "Cannot dereference twice!" ) ? void (0) : __assert_fail ("!IsEarlyIncremented && \"Cannot dereference twice!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 538, __extension__ __PRETTY_FUNCTION__)); |
539 | IsEarlyIncremented = true; |
540 | #endif |
541 | return *(this->I)++; |
542 | } |
543 | |
544 | using BaseT::operator++; |
545 | early_inc_iterator_impl &operator++() { |
546 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS1 |
547 | assert(IsEarlyIncremented && "Cannot increment before dereferencing!")(static_cast <bool> (IsEarlyIncremented && "Cannot increment before dereferencing!" ) ? void (0) : __assert_fail ("IsEarlyIncremented && \"Cannot increment before dereferencing!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 547, __extension__ __PRETTY_FUNCTION__)); |
548 | IsEarlyIncremented = false; |
549 | #endif |
550 | return *this; |
551 | } |
552 | |
553 | friend bool operator==(const early_inc_iterator_impl &LHS, |
554 | const early_inc_iterator_impl &RHS) { |
555 | #if LLVM_ENABLE_ABI_BREAKING_CHECKS1 |
556 | assert(!LHS.IsEarlyIncremented && "Cannot compare after dereferencing!")(static_cast <bool> (!LHS.IsEarlyIncremented && "Cannot compare after dereferencing!") ? void (0) : __assert_fail ("!LHS.IsEarlyIncremented && \"Cannot compare after dereferencing!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 556, __extension__ __PRETTY_FUNCTION__)); |
557 | #endif |
558 | return (const BaseT &)LHS == (const BaseT &)RHS; |
559 | } |
560 | }; |
561 | |
562 | /// Make a range that does early increment to allow mutation of the underlying |
563 | /// range without disrupting iteration. |
564 | /// |
565 | /// The underlying iterator will be incremented immediately after it is |
566 | /// dereferenced, allowing deletion of the current node or insertion of nodes to |
567 | /// not disrupt iteration provided they do not invalidate the *next* iterator -- |
568 | /// the current iterator can be invalidated. |
569 | /// |
570 | /// This requires a very exact pattern of use that is only really suitable to |
571 | /// range based for loops and other range algorithms that explicitly guarantee |
572 | /// to dereference exactly once each element, and to increment exactly once each |
573 | /// element. |
574 | template <typename RangeT> |
575 | iterator_range<early_inc_iterator_impl<detail::IterOfRange<RangeT>>> |
576 | make_early_inc_range(RangeT &&Range) { |
577 | using EarlyIncIteratorT = |
578 | early_inc_iterator_impl<detail::IterOfRange<RangeT>>; |
579 | return make_range(EarlyIncIteratorT(std::begin(std::forward<RangeT>(Range))), |
580 | EarlyIncIteratorT(std::end(std::forward<RangeT>(Range)))); |
581 | } |
582 | |
583 | // forward declarations required by zip_shortest/zip_first/zip_longest |
584 | template <typename R, typename UnaryPredicate> |
585 | bool all_of(R &&range, UnaryPredicate P); |
586 | template <typename R, typename UnaryPredicate> |
587 | bool any_of(R &&range, UnaryPredicate P); |
588 | |
589 | namespace detail { |
590 | |
591 | using std::declval; |
592 | |
593 | // We have to alias this since inlining the actual type at the usage site |
594 | // in the parameter list of iterator_facade_base<> below ICEs MSVC 2017. |
595 | template<typename... Iters> struct ZipTupleType { |
596 | using type = std::tuple<decltype(*declval<Iters>())...>; |
597 | }; |
598 | |
599 | template <typename ZipType, typename... Iters> |
600 | using zip_traits = iterator_facade_base< |
601 | ZipType, typename std::common_type<std::bidirectional_iterator_tag, |
602 | typename std::iterator_traits< |
603 | Iters>::iterator_category...>::type, |
604 | // ^ TODO: Implement random access methods. |
605 | typename ZipTupleType<Iters...>::type, |
606 | typename std::iterator_traits<typename std::tuple_element< |
607 | 0, std::tuple<Iters...>>::type>::difference_type, |
608 | // ^ FIXME: This follows boost::make_zip_iterator's assumption that all |
609 | // inner iterators have the same difference_type. It would fail if, for |
610 | // instance, the second field's difference_type were non-numeric while the |
611 | // first is. |
612 | typename ZipTupleType<Iters...>::type *, |
613 | typename ZipTupleType<Iters...>::type>; |
614 | |
615 | template <typename ZipType, typename... Iters> |
616 | struct zip_common : public zip_traits<ZipType, Iters...> { |
617 | using Base = zip_traits<ZipType, Iters...>; |
618 | using value_type = typename Base::value_type; |
619 | |
620 | std::tuple<Iters...> iterators; |
621 | |
622 | protected: |
623 | template <size_t... Ns> value_type deref(std::index_sequence<Ns...>) const { |
624 | return value_type(*std::get<Ns>(iterators)...); |
625 | } |
626 | |
627 | template <size_t... Ns> |
628 | decltype(iterators) tup_inc(std::index_sequence<Ns...>) const { |
629 | return std::tuple<Iters...>(std::next(std::get<Ns>(iterators))...); |
630 | } |
631 | |
632 | template <size_t... Ns> |
633 | decltype(iterators) tup_dec(std::index_sequence<Ns...>) const { |
634 | return std::tuple<Iters...>(std::prev(std::get<Ns>(iterators))...); |
635 | } |
636 | |
637 | template <size_t... Ns> |
638 | bool test_all_equals(const zip_common &other, |
639 | std::index_sequence<Ns...>) const { |
640 | return all_of(std::initializer_list<bool>{std::get<Ns>(this->iterators) == |
641 | std::get<Ns>(other.iterators)...}, |
642 | identity<bool>{}); |
643 | } |
644 | |
645 | public: |
646 | zip_common(Iters &&... ts) : iterators(std::forward<Iters>(ts)...) {} |
647 | |
648 | value_type operator*() { return deref(std::index_sequence_for<Iters...>{}); } |
649 | |
650 | const value_type operator*() const { |
651 | return deref(std::index_sequence_for<Iters...>{}); |
652 | } |
653 | |
654 | ZipType &operator++() { |
655 | iterators = tup_inc(std::index_sequence_for<Iters...>{}); |
656 | return *reinterpret_cast<ZipType *>(this); |
657 | } |
658 | |
659 | ZipType &operator--() { |
660 | static_assert(Base::IsBidirectional, |
661 | "All inner iterators must be at least bidirectional."); |
662 | iterators = tup_dec(std::index_sequence_for<Iters...>{}); |
663 | return *reinterpret_cast<ZipType *>(this); |
664 | } |
665 | |
666 | /// Return true if all the iterator are matching `other`'s iterators. |
667 | bool all_equals(zip_common &other) { |
668 | return test_all_equals(other, std::index_sequence_for<Iters...>{}); |
669 | } |
670 | }; |
671 | |
672 | template <typename... Iters> |
673 | struct zip_first : public zip_common<zip_first<Iters...>, Iters...> { |
674 | using Base = zip_common<zip_first<Iters...>, Iters...>; |
675 | |
676 | bool operator==(const zip_first<Iters...> &other) const { |
677 | return std::get<0>(this->iterators) == std::get<0>(other.iterators); |
678 | } |
679 | |
680 | zip_first(Iters &&... ts) : Base(std::forward<Iters>(ts)...) {} |
681 | }; |
682 | |
683 | template <typename... Iters> |
684 | class zip_shortest : public zip_common<zip_shortest<Iters...>, Iters...> { |
685 | template <size_t... Ns> |
686 | bool test(const zip_shortest<Iters...> &other, |
687 | std::index_sequence<Ns...>) const { |
688 | return all_of(std::initializer_list<bool>{std::get<Ns>(this->iterators) != |
689 | std::get<Ns>(other.iterators)...}, |
690 | identity<bool>{}); |
691 | } |
692 | |
693 | public: |
694 | using Base = zip_common<zip_shortest<Iters...>, Iters...>; |
695 | |
696 | zip_shortest(Iters &&... ts) : Base(std::forward<Iters>(ts)...) {} |
697 | |
698 | bool operator==(const zip_shortest<Iters...> &other) const { |
699 | return !test(other, std::index_sequence_for<Iters...>{}); |
700 | } |
701 | }; |
702 | |
703 | template <template <typename...> class ItType, typename... Args> class zippy { |
704 | public: |
705 | using iterator = ItType<decltype(std::begin(std::declval<Args>()))...>; |
706 | using iterator_category = typename iterator::iterator_category; |
707 | using value_type = typename iterator::value_type; |
708 | using difference_type = typename iterator::difference_type; |
709 | using pointer = typename iterator::pointer; |
710 | using reference = typename iterator::reference; |
711 | |
712 | private: |
713 | std::tuple<Args...> ts; |
714 | |
715 | template <size_t... Ns> |
716 | iterator begin_impl(std::index_sequence<Ns...>) const { |
717 | return iterator(std::begin(std::get<Ns>(ts))...); |
718 | } |
719 | template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) const { |
720 | return iterator(std::end(std::get<Ns>(ts))...); |
721 | } |
722 | |
723 | public: |
724 | zippy(Args &&... ts_) : ts(std::forward<Args>(ts_)...) {} |
725 | |
726 | iterator begin() const { |
727 | return begin_impl(std::index_sequence_for<Args...>{}); |
728 | } |
729 | iterator end() const { return end_impl(std::index_sequence_for<Args...>{}); } |
730 | }; |
731 | |
732 | } // end namespace detail |
733 | |
734 | /// zip iterator for two or more iteratable types. |
735 | template <typename T, typename U, typename... Args> |
736 | detail::zippy<detail::zip_shortest, T, U, Args...> zip(T &&t, U &&u, |
737 | Args &&... args) { |
738 | return detail::zippy<detail::zip_shortest, T, U, Args...>( |
739 | std::forward<T>(t), std::forward<U>(u), std::forward<Args>(args)...); |
740 | } |
741 | |
742 | /// zip iterator that, for the sake of efficiency, assumes the first iteratee to |
743 | /// be the shortest. |
744 | template <typename T, typename U, typename... Args> |
745 | detail::zippy<detail::zip_first, T, U, Args...> zip_first(T &&t, U &&u, |
746 | Args &&... args) { |
747 | return detail::zippy<detail::zip_first, T, U, Args...>( |
748 | std::forward<T>(t), std::forward<U>(u), std::forward<Args>(args)...); |
749 | } |
750 | |
751 | namespace detail { |
752 | template <typename Iter> |
753 | Iter next_or_end(const Iter &I, const Iter &End) { |
754 | if (I == End) |
755 | return End; |
756 | return std::next(I); |
757 | } |
758 | |
759 | template <typename Iter> |
760 | auto deref_or_none(const Iter &I, const Iter &End) -> llvm::Optional< |
761 | std::remove_const_t<std::remove_reference_t<decltype(*I)>>> { |
762 | if (I == End) |
763 | return None; |
764 | return *I; |
765 | } |
766 | |
767 | template <typename Iter> struct ZipLongestItemType { |
768 | using type = |
769 | llvm::Optional<typename std::remove_const<typename std::remove_reference< |
770 | decltype(*std::declval<Iter>())>::type>::type>; |
771 | }; |
772 | |
773 | template <typename... Iters> struct ZipLongestTupleType { |
774 | using type = std::tuple<typename ZipLongestItemType<Iters>::type...>; |
775 | }; |
776 | |
777 | template <typename... Iters> |
778 | class zip_longest_iterator |
779 | : public iterator_facade_base< |
780 | zip_longest_iterator<Iters...>, |
781 | typename std::common_type< |
782 | std::forward_iterator_tag, |
783 | typename std::iterator_traits<Iters>::iterator_category...>::type, |
784 | typename ZipLongestTupleType<Iters...>::type, |
785 | typename std::iterator_traits<typename std::tuple_element< |
786 | 0, std::tuple<Iters...>>::type>::difference_type, |
787 | typename ZipLongestTupleType<Iters...>::type *, |
788 | typename ZipLongestTupleType<Iters...>::type> { |
789 | public: |
790 | using value_type = typename ZipLongestTupleType<Iters...>::type; |
791 | |
792 | private: |
793 | std::tuple<Iters...> iterators; |
794 | std::tuple<Iters...> end_iterators; |
795 | |
796 | template <size_t... Ns> |
797 | bool test(const zip_longest_iterator<Iters...> &other, |
798 | std::index_sequence<Ns...>) const { |
799 | return llvm::any_of( |
800 | std::initializer_list<bool>{std::get<Ns>(this->iterators) != |
801 | std::get<Ns>(other.iterators)...}, |
802 | identity<bool>{}); |
803 | } |
804 | |
805 | template <size_t... Ns> value_type deref(std::index_sequence<Ns...>) const { |
806 | return value_type( |
807 | deref_or_none(std::get<Ns>(iterators), std::get<Ns>(end_iterators))...); |
808 | } |
809 | |
810 | template <size_t... Ns> |
811 | decltype(iterators) tup_inc(std::index_sequence<Ns...>) const { |
812 | return std::tuple<Iters...>( |
813 | next_or_end(std::get<Ns>(iterators), std::get<Ns>(end_iterators))...); |
814 | } |
815 | |
816 | public: |
817 | zip_longest_iterator(std::pair<Iters &&, Iters &&>... ts) |
818 | : iterators(std::forward<Iters>(ts.first)...), |
819 | end_iterators(std::forward<Iters>(ts.second)...) {} |
820 | |
821 | value_type operator*() { return deref(std::index_sequence_for<Iters...>{}); } |
822 | |
823 | value_type operator*() const { |
824 | return deref(std::index_sequence_for<Iters...>{}); |
825 | } |
826 | |
827 | zip_longest_iterator<Iters...> &operator++() { |
828 | iterators = tup_inc(std::index_sequence_for<Iters...>{}); |
829 | return *this; |
830 | } |
831 | |
832 | bool operator==(const zip_longest_iterator<Iters...> &other) const { |
833 | return !test(other, std::index_sequence_for<Iters...>{}); |
834 | } |
835 | }; |
836 | |
837 | template <typename... Args> class zip_longest_range { |
838 | public: |
839 | using iterator = |
840 | zip_longest_iterator<decltype(adl_begin(std::declval<Args>()))...>; |
841 | using iterator_category = typename iterator::iterator_category; |
842 | using value_type = typename iterator::value_type; |
843 | using difference_type = typename iterator::difference_type; |
844 | using pointer = typename iterator::pointer; |
845 | using reference = typename iterator::reference; |
846 | |
847 | private: |
848 | std::tuple<Args...> ts; |
849 | |
850 | template <size_t... Ns> |
851 | iterator begin_impl(std::index_sequence<Ns...>) const { |
852 | return iterator(std::make_pair(adl_begin(std::get<Ns>(ts)), |
853 | adl_end(std::get<Ns>(ts)))...); |
854 | } |
855 | |
856 | template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) const { |
857 | return iterator(std::make_pair(adl_end(std::get<Ns>(ts)), |
858 | adl_end(std::get<Ns>(ts)))...); |
859 | } |
860 | |
861 | public: |
862 | zip_longest_range(Args &&... ts_) : ts(std::forward<Args>(ts_)...) {} |
863 | |
864 | iterator begin() const { |
865 | return begin_impl(std::index_sequence_for<Args...>{}); |
866 | } |
867 | iterator end() const { return end_impl(std::index_sequence_for<Args...>{}); } |
868 | }; |
869 | } // namespace detail |
870 | |
871 | /// Iterate over two or more iterators at the same time. Iteration continues |
872 | /// until all iterators reach the end. The llvm::Optional only contains a value |
873 | /// if the iterator has not reached the end. |
874 | template <typename T, typename U, typename... Args> |
875 | detail::zip_longest_range<T, U, Args...> zip_longest(T &&t, U &&u, |
876 | Args &&... args) { |
877 | return detail::zip_longest_range<T, U, Args...>( |
878 | std::forward<T>(t), std::forward<U>(u), std::forward<Args>(args)...); |
879 | } |
880 | |
881 | /// Iterator wrapper that concatenates sequences together. |
882 | /// |
883 | /// This can concatenate different iterators, even with different types, into |
884 | /// a single iterator provided the value types of all the concatenated |
885 | /// iterators expose `reference` and `pointer` types that can be converted to |
886 | /// `ValueT &` and `ValueT *` respectively. It doesn't support more |
887 | /// interesting/customized pointer or reference types. |
888 | /// |
889 | /// Currently this only supports forward or higher iterator categories as |
890 | /// inputs and always exposes a forward iterator interface. |
891 | template <typename ValueT, typename... IterTs> |
892 | class concat_iterator |
893 | : public iterator_facade_base<concat_iterator<ValueT, IterTs...>, |
894 | std::forward_iterator_tag, ValueT> { |
895 | using BaseT = typename concat_iterator::iterator_facade_base; |
896 | |
897 | /// We store both the current and end iterators for each concatenated |
898 | /// sequence in a tuple of pairs. |
899 | /// |
900 | /// Note that something like iterator_range seems nice at first here, but the |
901 | /// range properties are of little benefit and end up getting in the way |
902 | /// because we need to do mutation on the current iterators. |
903 | std::tuple<IterTs...> Begins; |
904 | std::tuple<IterTs...> Ends; |
905 | |
906 | /// Attempts to increment a specific iterator. |
907 | /// |
908 | /// Returns true if it was able to increment the iterator. Returns false if |
909 | /// the iterator is already at the end iterator. |
910 | template <size_t Index> bool incrementHelper() { |
911 | auto &Begin = std::get<Index>(Begins); |
912 | auto &End = std::get<Index>(Ends); |
913 | if (Begin == End) |
914 | return false; |
915 | |
916 | ++Begin; |
917 | return true; |
918 | } |
919 | |
920 | /// Increments the first non-end iterator. |
921 | /// |
922 | /// It is an error to call this with all iterators at the end. |
923 | template <size_t... Ns> void increment(std::index_sequence<Ns...>) { |
924 | // Build a sequence of functions to increment each iterator if possible. |
925 | bool (concat_iterator::*IncrementHelperFns[])() = { |
926 | &concat_iterator::incrementHelper<Ns>...}; |
927 | |
928 | // Loop over them, and stop as soon as we succeed at incrementing one. |
929 | for (auto &IncrementHelperFn : IncrementHelperFns) |
930 | if ((this->*IncrementHelperFn)()) |
931 | return; |
932 | |
933 | llvm_unreachable("Attempted to increment an end concat iterator!")::llvm::llvm_unreachable_internal("Attempted to increment an end concat iterator!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 933); |
934 | } |
935 | |
936 | /// Returns null if the specified iterator is at the end. Otherwise, |
937 | /// dereferences the iterator and returns the address of the resulting |
938 | /// reference. |
939 | template <size_t Index> ValueT *getHelper() const { |
940 | auto &Begin = std::get<Index>(Begins); |
941 | auto &End = std::get<Index>(Ends); |
942 | if (Begin == End) |
943 | return nullptr; |
944 | |
945 | return &*Begin; |
946 | } |
947 | |
948 | /// Finds the first non-end iterator, dereferences, and returns the resulting |
949 | /// reference. |
950 | /// |
951 | /// It is an error to call this with all iterators at the end. |
952 | template <size_t... Ns> ValueT &get(std::index_sequence<Ns...>) const { |
953 | // Build a sequence of functions to get from iterator if possible. |
954 | ValueT *(concat_iterator::*GetHelperFns[])() const = { |
955 | &concat_iterator::getHelper<Ns>...}; |
956 | |
957 | // Loop over them, and return the first result we find. |
958 | for (auto &GetHelperFn : GetHelperFns) |
959 | if (ValueT *P = (this->*GetHelperFn)()) |
960 | return *P; |
961 | |
962 | llvm_unreachable("Attempted to get a pointer from an end concat iterator!")::llvm::llvm_unreachable_internal("Attempted to get a pointer from an end concat iterator!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 962); |
963 | } |
964 | |
965 | public: |
966 | /// Constructs an iterator from a sequence of ranges. |
967 | /// |
968 | /// We need the full range to know how to switch between each of the |
969 | /// iterators. |
970 | template <typename... RangeTs> |
971 | explicit concat_iterator(RangeTs &&... Ranges) |
972 | : Begins(std::begin(Ranges)...), Ends(std::end(Ranges)...) {} |
973 | |
974 | using BaseT::operator++; |
975 | |
976 | concat_iterator &operator++() { |
977 | increment(std::index_sequence_for<IterTs...>()); |
978 | return *this; |
979 | } |
980 | |
981 | ValueT &operator*() const { |
982 | return get(std::index_sequence_for<IterTs...>()); |
983 | } |
984 | |
985 | bool operator==(const concat_iterator &RHS) const { |
986 | return Begins == RHS.Begins && Ends == RHS.Ends; |
987 | } |
988 | }; |
989 | |
990 | namespace detail { |
991 | |
992 | /// Helper to store a sequence of ranges being concatenated and access them. |
993 | /// |
994 | /// This is designed to facilitate providing actual storage when temporaries |
995 | /// are passed into the constructor such that we can use it as part of range |
996 | /// based for loops. |
997 | template <typename ValueT, typename... RangeTs> class concat_range { |
998 | public: |
999 | using iterator = |
1000 | concat_iterator<ValueT, |
1001 | decltype(std::begin(std::declval<RangeTs &>()))...>; |
1002 | |
1003 | private: |
1004 | std::tuple<RangeTs...> Ranges; |
1005 | |
1006 | template <size_t... Ns> iterator begin_impl(std::index_sequence<Ns...>) { |
1007 | return iterator(std::get<Ns>(Ranges)...); |
1008 | } |
1009 | template <size_t... Ns> iterator end_impl(std::index_sequence<Ns...>) { |
1010 | return iterator(make_range(std::end(std::get<Ns>(Ranges)), |
1011 | std::end(std::get<Ns>(Ranges)))...); |
1012 | } |
1013 | |
1014 | public: |
1015 | concat_range(RangeTs &&... Ranges) |
1016 | : Ranges(std::forward<RangeTs>(Ranges)...) {} |
1017 | |
1018 | iterator begin() { return begin_impl(std::index_sequence_for<RangeTs...>{}); } |
1019 | iterator end() { return end_impl(std::index_sequence_for<RangeTs...>{}); } |
1020 | }; |
1021 | |
1022 | } // end namespace detail |
1023 | |
1024 | /// Concatenated range across two or more ranges. |
1025 | /// |
1026 | /// The desired value type must be explicitly specified. |
1027 | template <typename ValueT, typename... RangeTs> |
1028 | detail::concat_range<ValueT, RangeTs...> concat(RangeTs &&... Ranges) { |
1029 | static_assert(sizeof...(RangeTs) > 1, |
1030 | "Need more than one range to concatenate!"); |
1031 | return detail::concat_range<ValueT, RangeTs...>( |
1032 | std::forward<RangeTs>(Ranges)...); |
1033 | } |
1034 | |
1035 | /// A utility class used to implement an iterator that contains some base object |
1036 | /// and an index. The iterator moves the index but keeps the base constant. |
1037 | template <typename DerivedT, typename BaseT, typename T, |
1038 | typename PointerT = T *, typename ReferenceT = T &> |
1039 | class indexed_accessor_iterator |
1040 | : public llvm::iterator_facade_base<DerivedT, |
1041 | std::random_access_iterator_tag, T, |
1042 | std::ptrdiff_t, PointerT, ReferenceT> { |
1043 | public: |
1044 | ptrdiff_t operator-(const indexed_accessor_iterator &rhs) const { |
1045 | assert(base == rhs.base && "incompatible iterators")(static_cast <bool> (base == rhs.base && "incompatible iterators" ) ? void (0) : __assert_fail ("base == rhs.base && \"incompatible iterators\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1045, __extension__ __PRETTY_FUNCTION__)); |
1046 | return index - rhs.index; |
1047 | } |
1048 | bool operator==(const indexed_accessor_iterator &rhs) const { |
1049 | return base == rhs.base && index == rhs.index; |
1050 | } |
1051 | bool operator<(const indexed_accessor_iterator &rhs) const { |
1052 | assert(base == rhs.base && "incompatible iterators")(static_cast <bool> (base == rhs.base && "incompatible iterators" ) ? void (0) : __assert_fail ("base == rhs.base && \"incompatible iterators\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1052, __extension__ __PRETTY_FUNCTION__)); |
1053 | return index < rhs.index; |
1054 | } |
1055 | |
1056 | DerivedT &operator+=(ptrdiff_t offset) { |
1057 | this->index += offset; |
1058 | return static_cast<DerivedT &>(*this); |
1059 | } |
1060 | DerivedT &operator-=(ptrdiff_t offset) { |
1061 | this->index -= offset; |
1062 | return static_cast<DerivedT &>(*this); |
1063 | } |
1064 | |
1065 | /// Returns the current index of the iterator. |
1066 | ptrdiff_t getIndex() const { return index; } |
1067 | |
1068 | /// Returns the current base of the iterator. |
1069 | const BaseT &getBase() const { return base; } |
1070 | |
1071 | protected: |
1072 | indexed_accessor_iterator(BaseT base, ptrdiff_t index) |
1073 | : base(base), index(index) {} |
1074 | BaseT base; |
1075 | ptrdiff_t index; |
1076 | }; |
1077 | |
1078 | namespace detail { |
1079 | /// The class represents the base of a range of indexed_accessor_iterators. It |
1080 | /// provides support for many different range functionalities, e.g. |
1081 | /// drop_front/slice/etc.. Derived range classes must implement the following |
1082 | /// static methods: |
1083 | /// * ReferenceT dereference_iterator(const BaseT &base, ptrdiff_t index) |
1084 | /// - Dereference an iterator pointing to the base object at the given |
1085 | /// index. |
1086 | /// * BaseT offset_base(const BaseT &base, ptrdiff_t index) |
1087 | /// - Return a new base that is offset from the provide base by 'index' |
1088 | /// elements. |
1089 | template <typename DerivedT, typename BaseT, typename T, |
1090 | typename PointerT = T *, typename ReferenceT = T &> |
1091 | class indexed_accessor_range_base { |
1092 | public: |
1093 | using RangeBaseT = |
1094 | indexed_accessor_range_base<DerivedT, BaseT, T, PointerT, ReferenceT>; |
1095 | |
1096 | /// An iterator element of this range. |
1097 | class iterator : public indexed_accessor_iterator<iterator, BaseT, T, |
1098 | PointerT, ReferenceT> { |
1099 | public: |
1100 | // Index into this iterator, invoking a static method on the derived type. |
1101 | ReferenceT operator*() const { |
1102 | return DerivedT::dereference_iterator(this->getBase(), this->getIndex()); |
1103 | } |
1104 | |
1105 | private: |
1106 | iterator(BaseT owner, ptrdiff_t curIndex) |
1107 | : indexed_accessor_iterator<iterator, BaseT, T, PointerT, ReferenceT>( |
1108 | owner, curIndex) {} |
1109 | |
1110 | /// Allow access to the constructor. |
1111 | friend indexed_accessor_range_base<DerivedT, BaseT, T, PointerT, |
1112 | ReferenceT>; |
1113 | }; |
1114 | |
1115 | indexed_accessor_range_base(iterator begin, iterator end) |
1116 | : base(offset_base(begin.getBase(), begin.getIndex())), |
1117 | count(end.getIndex() - begin.getIndex()) {} |
1118 | indexed_accessor_range_base(const iterator_range<iterator> &range) |
1119 | : indexed_accessor_range_base(range.begin(), range.end()) {} |
1120 | indexed_accessor_range_base(BaseT base, ptrdiff_t count) |
1121 | : base(base), count(count) {} |
1122 | |
1123 | iterator begin() const { return iterator(base, 0); } |
1124 | iterator end() const { return iterator(base, count); } |
1125 | ReferenceT operator[](size_t Index) const { |
1126 | assert(Index < size() && "invalid index for value range")(static_cast <bool> (Index < size() && "invalid index for value range" ) ? void (0) : __assert_fail ("Index < size() && \"invalid index for value range\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1126, __extension__ __PRETTY_FUNCTION__)); |
1127 | return DerivedT::dereference_iterator(base, static_cast<ptrdiff_t>(Index)); |
1128 | } |
1129 | ReferenceT front() const { |
1130 | assert(!empty() && "expected non-empty range")(static_cast <bool> (!empty() && "expected non-empty range" ) ? void (0) : __assert_fail ("!empty() && \"expected non-empty range\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1130, __extension__ __PRETTY_FUNCTION__)); |
1131 | return (*this)[0]; |
1132 | } |
1133 | ReferenceT back() const { |
1134 | assert(!empty() && "expected non-empty range")(static_cast <bool> (!empty() && "expected non-empty range" ) ? void (0) : __assert_fail ("!empty() && \"expected non-empty range\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1134, __extension__ __PRETTY_FUNCTION__)); |
1135 | return (*this)[size() - 1]; |
1136 | } |
1137 | |
1138 | /// Compare this range with another. |
1139 | template <typename OtherT> bool operator==(const OtherT &other) const { |
1140 | return size() == |
1141 | static_cast<size_t>(std::distance(other.begin(), other.end())) && |
1142 | std::equal(begin(), end(), other.begin()); |
1143 | } |
1144 | template <typename OtherT> bool operator!=(const OtherT &other) const { |
1145 | return !(*this == other); |
1146 | } |
1147 | |
1148 | /// Return the size of this range. |
1149 | size_t size() const { return count; } |
1150 | |
1151 | /// Return if the range is empty. |
1152 | bool empty() const { return size() == 0; } |
1153 | |
1154 | /// Drop the first N elements, and keep M elements. |
1155 | DerivedT slice(size_t n, size_t m) const { |
1156 | assert(n + m <= size() && "invalid size specifiers")(static_cast <bool> (n + m <= size() && "invalid size specifiers" ) ? void (0) : __assert_fail ("n + m <= size() && \"invalid size specifiers\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1156, __extension__ __PRETTY_FUNCTION__)); |
1157 | return DerivedT(offset_base(base, n), m); |
1158 | } |
1159 | |
1160 | /// Drop the first n elements. |
1161 | DerivedT drop_front(size_t n = 1) const { |
1162 | assert(size() >= n && "Dropping more elements than exist")(static_cast <bool> (size() >= n && "Dropping more elements than exist" ) ? void (0) : __assert_fail ("size() >= n && \"Dropping more elements than exist\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1162, __extension__ __PRETTY_FUNCTION__)); |
1163 | return slice(n, size() - n); |
1164 | } |
1165 | /// Drop the last n elements. |
1166 | DerivedT drop_back(size_t n = 1) const { |
1167 | assert(size() >= n && "Dropping more elements than exist")(static_cast <bool> (size() >= n && "Dropping more elements than exist" ) ? void (0) : __assert_fail ("size() >= n && \"Dropping more elements than exist\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1167, __extension__ __PRETTY_FUNCTION__)); |
1168 | return DerivedT(base, size() - n); |
1169 | } |
1170 | |
1171 | /// Take the first n elements. |
1172 | DerivedT take_front(size_t n = 1) const { |
1173 | return n < size() ? drop_back(size() - n) |
1174 | : static_cast<const DerivedT &>(*this); |
1175 | } |
1176 | |
1177 | /// Take the last n elements. |
1178 | DerivedT take_back(size_t n = 1) const { |
1179 | return n < size() ? drop_front(size() - n) |
1180 | : static_cast<const DerivedT &>(*this); |
1181 | } |
1182 | |
1183 | /// Allow conversion to any type accepting an iterator_range. |
1184 | template <typename RangeT, typename = std::enable_if_t<std::is_constructible< |
1185 | RangeT, iterator_range<iterator>>::value>> |
1186 | operator RangeT() const { |
1187 | return RangeT(iterator_range<iterator>(*this)); |
1188 | } |
1189 | |
1190 | /// Returns the base of this range. |
1191 | const BaseT &getBase() const { return base; } |
1192 | |
1193 | private: |
1194 | /// Offset the given base by the given amount. |
1195 | static BaseT offset_base(const BaseT &base, size_t n) { |
1196 | return n == 0 ? base : DerivedT::offset_base(base, n); |
1197 | } |
1198 | |
1199 | protected: |
1200 | indexed_accessor_range_base(const indexed_accessor_range_base &) = default; |
1201 | indexed_accessor_range_base(indexed_accessor_range_base &&) = default; |
1202 | indexed_accessor_range_base & |
1203 | operator=(const indexed_accessor_range_base &) = default; |
1204 | |
1205 | /// The base that owns the provided range of values. |
1206 | BaseT base; |
1207 | /// The size from the owning range. |
1208 | ptrdiff_t count; |
1209 | }; |
1210 | } // end namespace detail |
1211 | |
1212 | /// This class provides an implementation of a range of |
1213 | /// indexed_accessor_iterators where the base is not indexable. Ranges with |
1214 | /// bases that are offsetable should derive from indexed_accessor_range_base |
1215 | /// instead. Derived range classes are expected to implement the following |
1216 | /// static method: |
1217 | /// * ReferenceT dereference(const BaseT &base, ptrdiff_t index) |
1218 | /// - Dereference an iterator pointing to a parent base at the given index. |
1219 | template <typename DerivedT, typename BaseT, typename T, |
1220 | typename PointerT = T *, typename ReferenceT = T &> |
1221 | class indexed_accessor_range |
1222 | : public detail::indexed_accessor_range_base< |
1223 | DerivedT, std::pair<BaseT, ptrdiff_t>, T, PointerT, ReferenceT> { |
1224 | public: |
1225 | indexed_accessor_range(BaseT base, ptrdiff_t startIndex, ptrdiff_t count) |
1226 | : detail::indexed_accessor_range_base< |
1227 | DerivedT, std::pair<BaseT, ptrdiff_t>, T, PointerT, ReferenceT>( |
1228 | std::make_pair(base, startIndex), count) {} |
1229 | using detail::indexed_accessor_range_base< |
1230 | DerivedT, std::pair<BaseT, ptrdiff_t>, T, PointerT, |
1231 | ReferenceT>::indexed_accessor_range_base; |
1232 | |
1233 | /// Returns the current base of the range. |
1234 | const BaseT &getBase() const { return this->base.first; } |
1235 | |
1236 | /// Returns the current start index of the range. |
1237 | ptrdiff_t getStartIndex() const { return this->base.second; } |
1238 | |
1239 | /// See `detail::indexed_accessor_range_base` for details. |
1240 | static std::pair<BaseT, ptrdiff_t> |
1241 | offset_base(const std::pair<BaseT, ptrdiff_t> &base, ptrdiff_t index) { |
1242 | // We encode the internal base as a pair of the derived base and a start |
1243 | // index into the derived base. |
1244 | return std::make_pair(base.first, base.second + index); |
1245 | } |
1246 | /// See `detail::indexed_accessor_range_base` for details. |
1247 | static ReferenceT |
1248 | dereference_iterator(const std::pair<BaseT, ptrdiff_t> &base, |
1249 | ptrdiff_t index) { |
1250 | return DerivedT::dereference(base.first, base.second + index); |
1251 | } |
1252 | }; |
1253 | |
1254 | namespace detail { |
1255 | /// Return a reference to the first or second member of a reference. Otherwise, |
1256 | /// return a copy of the member of a temporary. |
1257 | /// |
1258 | /// When passing a range whose iterators return values instead of references, |
1259 | /// the reference must be dropped from `decltype((elt.first))`, which will |
1260 | /// always be a reference, to avoid returning a reference to a temporary. |
1261 | template <typename EltTy, typename FirstTy> class first_or_second_type { |
1262 | public: |
1263 | using type = |
1264 | typename std::conditional_t<std::is_reference<EltTy>::value, FirstTy, |
1265 | std::remove_reference_t<FirstTy>>; |
1266 | }; |
1267 | } // end namespace detail |
1268 | |
1269 | /// Given a container of pairs, return a range over the first elements. |
1270 | template <typename ContainerTy> auto make_first_range(ContainerTy &&c) { |
1271 | using EltTy = decltype((*std::begin(c))); |
1272 | return llvm::map_range(std::forward<ContainerTy>(c), |
1273 | [](EltTy elt) -> typename detail::first_or_second_type< |
1274 | EltTy, decltype((elt.first))>::type { |
1275 | return elt.first; |
1276 | }); |
1277 | } |
1278 | |
1279 | /// Given a container of pairs, return a range over the second elements. |
1280 | template <typename ContainerTy> auto make_second_range(ContainerTy &&c) { |
1281 | using EltTy = decltype((*std::begin(c))); |
1282 | return llvm::map_range( |
1283 | std::forward<ContainerTy>(c), |
1284 | [](EltTy elt) -> |
1285 | typename detail::first_or_second_type<EltTy, |
1286 | decltype((elt.second))>::type { |
1287 | return elt.second; |
1288 | }); |
1289 | } |
1290 | |
1291 | //===----------------------------------------------------------------------===// |
1292 | // Extra additions to <utility> |
1293 | //===----------------------------------------------------------------------===// |
1294 | |
1295 | /// Function object to check whether the first component of a std::pair |
1296 | /// compares less than the first component of another std::pair. |
1297 | struct less_first { |
1298 | template <typename T> bool operator()(const T &lhs, const T &rhs) const { |
1299 | return std::less<>()(lhs.first, rhs.first); |
1300 | } |
1301 | }; |
1302 | |
1303 | /// Function object to check whether the second component of a std::pair |
1304 | /// compares less than the second component of another std::pair. |
1305 | struct less_second { |
1306 | template <typename T> bool operator()(const T &lhs, const T &rhs) const { |
1307 | return std::less<>()(lhs.second, rhs.second); |
1308 | } |
1309 | }; |
1310 | |
1311 | /// \brief Function object to apply a binary function to the first component of |
1312 | /// a std::pair. |
1313 | template<typename FuncTy> |
1314 | struct on_first { |
1315 | FuncTy func; |
1316 | |
1317 | template <typename T> |
1318 | decltype(auto) operator()(const T &lhs, const T &rhs) const { |
1319 | return func(lhs.first, rhs.first); |
1320 | } |
1321 | }; |
1322 | |
1323 | /// Utility type to build an inheritance chain that makes it easy to rank |
1324 | /// overload candidates. |
1325 | template <int N> struct rank : rank<N - 1> {}; |
1326 | template <> struct rank<0> {}; |
1327 | |
1328 | /// traits class for checking whether type T is one of any of the given |
1329 | /// types in the variadic list. |
1330 | template <typename T, typename... Ts> |
1331 | using is_one_of = disjunction<std::is_same<T, Ts>...>; |
1332 | |
1333 | /// traits class for checking whether type T is a base class for all |
1334 | /// the given types in the variadic list. |
1335 | template <typename T, typename... Ts> |
1336 | using are_base_of = conjunction<std::is_base_of<T, Ts>...>; |
1337 | |
1338 | namespace detail { |
1339 | template <typename... Ts> struct Visitor; |
1340 | |
1341 | template <typename HeadT, typename... TailTs> |
1342 | struct Visitor<HeadT, TailTs...> : remove_cvref_t<HeadT>, Visitor<TailTs...> { |
1343 | explicit constexpr Visitor(HeadT &&Head, TailTs &&...Tail) |
1344 | : remove_cvref_t<HeadT>(std::forward<HeadT>(Head)), |
1345 | Visitor<TailTs...>(std::forward<TailTs>(Tail)...) {} |
1346 | using remove_cvref_t<HeadT>::operator(); |
1347 | using Visitor<TailTs...>::operator(); |
1348 | }; |
1349 | |
1350 | template <typename HeadT> struct Visitor<HeadT> : remove_cvref_t<HeadT> { |
1351 | explicit constexpr Visitor(HeadT &&Head) |
1352 | : remove_cvref_t<HeadT>(std::forward<HeadT>(Head)) {} |
1353 | using remove_cvref_t<HeadT>::operator(); |
1354 | }; |
1355 | } // namespace detail |
1356 | |
1357 | /// Returns an opaquely-typed Callable object whose operator() overload set is |
1358 | /// the sum of the operator() overload sets of each CallableT in CallableTs. |
1359 | /// |
1360 | /// The type of the returned object derives from each CallableT in CallableTs. |
1361 | /// The returned object is constructed by invoking the appropriate copy or move |
1362 | /// constructor of each CallableT, as selected by overload resolution on the |
1363 | /// corresponding argument to makeVisitor. |
1364 | /// |
1365 | /// Example: |
1366 | /// |
1367 | /// \code |
1368 | /// auto visitor = makeVisitor([](auto) { return "unhandled type"; }, |
1369 | /// [](int i) { return "int"; }, |
1370 | /// [](std::string s) { return "str"; }); |
1371 | /// auto a = visitor(42); // `a` is now "int". |
1372 | /// auto b = visitor("foo"); // `b` is now "str". |
1373 | /// auto c = visitor(3.14f); // `c` is now "unhandled type". |
1374 | /// \endcode |
1375 | /// |
1376 | /// Example of making a visitor with a lambda which captures a move-only type: |
1377 | /// |
1378 | /// \code |
1379 | /// std::unique_ptr<FooHandler> FH = /* ... */; |
1380 | /// auto visitor = makeVisitor( |
1381 | /// [FH{std::move(FH)}](Foo F) { return FH->handle(F); }, |
1382 | /// [](int i) { return i; }, |
1383 | /// [](std::string s) { return atoi(s); }); |
1384 | /// \endcode |
1385 | template <typename... CallableTs> |
1386 | constexpr decltype(auto) makeVisitor(CallableTs &&...Callables) { |
1387 | return detail::Visitor<CallableTs...>(std::forward<CallableTs>(Callables)...); |
1388 | } |
1389 | |
1390 | //===----------------------------------------------------------------------===// |
1391 | // Extra additions for arrays |
1392 | //===----------------------------------------------------------------------===// |
1393 | |
1394 | // We have a copy here so that LLVM behaves the same when using different |
1395 | // standard libraries. |
1396 | template <class Iterator, class RNG> |
1397 | void shuffle(Iterator first, Iterator last, RNG &&g) { |
1398 | // It would be better to use a std::uniform_int_distribution, |
1399 | // but that would be stdlib dependent. |
1400 | typedef |
1401 | typename std::iterator_traits<Iterator>::difference_type difference_type; |
1402 | for (auto size = last - first; size > 1; ++first, (void)--size) { |
1403 | difference_type offset = g() % size; |
1404 | // Avoid self-assignment due to incorrect assertions in libstdc++ |
1405 | // containers (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85828). |
1406 | if (offset != difference_type(0)) |
1407 | std::iter_swap(first, first + offset); |
1408 | } |
1409 | } |
1410 | |
1411 | /// Find the length of an array. |
1412 | template <class T, std::size_t N> |
1413 | constexpr inline size_t array_lengthof(T (&)[N]) { |
1414 | return N; |
1415 | } |
1416 | |
1417 | /// Adapt std::less<T> for array_pod_sort. |
1418 | template<typename T> |
1419 | inline int array_pod_sort_comparator(const void *P1, const void *P2) { |
1420 | if (std::less<T>()(*reinterpret_cast<const T*>(P1), |
1421 | *reinterpret_cast<const T*>(P2))) |
1422 | return -1; |
1423 | if (std::less<T>()(*reinterpret_cast<const T*>(P2), |
1424 | *reinterpret_cast<const T*>(P1))) |
1425 | return 1; |
1426 | return 0; |
1427 | } |
1428 | |
1429 | /// get_array_pod_sort_comparator - This is an internal helper function used to |
1430 | /// get type deduction of T right. |
1431 | template<typename T> |
1432 | inline int (*get_array_pod_sort_comparator(const T &)) |
1433 | (const void*, const void*) { |
1434 | return array_pod_sort_comparator<T>; |
1435 | } |
1436 | |
1437 | #ifdef EXPENSIVE_CHECKS |
1438 | namespace detail { |
1439 | |
1440 | inline unsigned presortShuffleEntropy() { |
1441 | static unsigned Result(std::random_device{}()); |
1442 | return Result; |
1443 | } |
1444 | |
1445 | template <class IteratorTy> |
1446 | inline void presortShuffle(IteratorTy Start, IteratorTy End) { |
1447 | std::mt19937 Generator(presortShuffleEntropy()); |
1448 | llvm::shuffle(Start, End, Generator); |
1449 | } |
1450 | |
1451 | } // end namespace detail |
1452 | #endif |
1453 | |
1454 | /// array_pod_sort - This sorts an array with the specified start and end |
1455 | /// extent. This is just like std::sort, except that it calls qsort instead of |
1456 | /// using an inlined template. qsort is slightly slower than std::sort, but |
1457 | /// most sorts are not performance critical in LLVM and std::sort has to be |
1458 | /// template instantiated for each type, leading to significant measured code |
1459 | /// bloat. This function should generally be used instead of std::sort where |
1460 | /// possible. |
1461 | /// |
1462 | /// This function assumes that you have simple POD-like types that can be |
1463 | /// compared with std::less and can be moved with memcpy. If this isn't true, |
1464 | /// you should use std::sort. |
1465 | /// |
1466 | /// NOTE: If qsort_r were portable, we could allow a custom comparator and |
1467 | /// default to std::less. |
1468 | template<class IteratorTy> |
1469 | inline void array_pod_sort(IteratorTy Start, IteratorTy End) { |
1470 | // Don't inefficiently call qsort with one element or trigger undefined |
1471 | // behavior with an empty sequence. |
1472 | auto NElts = End - Start; |
1473 | if (NElts <= 1) return; |
1474 | #ifdef EXPENSIVE_CHECKS |
1475 | detail::presortShuffle<IteratorTy>(Start, End); |
1476 | #endif |
1477 | qsort(&*Start, NElts, sizeof(*Start), get_array_pod_sort_comparator(*Start)); |
1478 | } |
1479 | |
1480 | template <class IteratorTy> |
1481 | inline void array_pod_sort( |
1482 | IteratorTy Start, IteratorTy End, |
1483 | int (*Compare)( |
1484 | const typename std::iterator_traits<IteratorTy>::value_type *, |
1485 | const typename std::iterator_traits<IteratorTy>::value_type *)) { |
1486 | // Don't inefficiently call qsort with one element or trigger undefined |
1487 | // behavior with an empty sequence. |
1488 | auto NElts = End - Start; |
1489 | if (NElts <= 1) return; |
1490 | #ifdef EXPENSIVE_CHECKS |
1491 | detail::presortShuffle<IteratorTy>(Start, End); |
1492 | #endif |
1493 | qsort(&*Start, NElts, sizeof(*Start), |
1494 | reinterpret_cast<int (*)(const void *, const void *)>(Compare)); |
1495 | } |
1496 | |
1497 | namespace detail { |
1498 | template <typename T> |
1499 | // We can use qsort if the iterator type is a pointer and the underlying value |
1500 | // is trivially copyable. |
1501 | using sort_trivially_copyable = conjunction< |
1502 | std::is_pointer<T>, |
1503 | std::is_trivially_copyable<typename std::iterator_traits<T>::value_type>>; |
1504 | } // namespace detail |
1505 | |
1506 | // Provide wrappers to std::sort which shuffle the elements before sorting |
1507 | // to help uncover non-deterministic behavior (PR35135). |
1508 | template <typename IteratorTy, |
1509 | std::enable_if_t<!detail::sort_trivially_copyable<IteratorTy>::value, |
1510 | int> = 0> |
1511 | inline void sort(IteratorTy Start, IteratorTy End) { |
1512 | #ifdef EXPENSIVE_CHECKS |
1513 | detail::presortShuffle<IteratorTy>(Start, End); |
1514 | #endif |
1515 | std::sort(Start, End); |
1516 | } |
1517 | |
1518 | // Forward trivially copyable types to array_pod_sort. This avoids a large |
1519 | // amount of code bloat for a minor performance hit. |
1520 | template <typename IteratorTy, |
1521 | std::enable_if_t<detail::sort_trivially_copyable<IteratorTy>::value, |
1522 | int> = 0> |
1523 | inline void sort(IteratorTy Start, IteratorTy End) { |
1524 | array_pod_sort(Start, End); |
1525 | } |
1526 | |
1527 | template <typename Container> inline void sort(Container &&C) { |
1528 | llvm::sort(adl_begin(C), adl_end(C)); |
1529 | } |
1530 | |
1531 | template <typename IteratorTy, typename Compare> |
1532 | inline void sort(IteratorTy Start, IteratorTy End, Compare Comp) { |
1533 | #ifdef EXPENSIVE_CHECKS |
1534 | detail::presortShuffle<IteratorTy>(Start, End); |
1535 | #endif |
1536 | std::sort(Start, End, Comp); |
1537 | } |
1538 | |
1539 | template <typename Container, typename Compare> |
1540 | inline void sort(Container &&C, Compare Comp) { |
1541 | llvm::sort(adl_begin(C), adl_end(C), Comp); |
1542 | } |
1543 | |
1544 | //===----------------------------------------------------------------------===// |
1545 | // Extra additions to <algorithm> |
1546 | //===----------------------------------------------------------------------===// |
1547 | |
1548 | /// Get the size of a range. This is a wrapper function around std::distance |
1549 | /// which is only enabled when the operation is O(1). |
1550 | template <typename R> |
1551 | auto size(R &&Range, |
1552 | std::enable_if_t< |
1553 | std::is_base_of<std::random_access_iterator_tag, |
1554 | typename std::iterator_traits<decltype( |
1555 | Range.begin())>::iterator_category>::value, |
1556 | void> * = nullptr) { |
1557 | return std::distance(Range.begin(), Range.end()); |
1558 | } |
1559 | |
1560 | /// Provide wrappers to std::for_each which take ranges instead of having to |
1561 | /// pass begin/end explicitly. |
1562 | template <typename R, typename UnaryFunction> |
1563 | UnaryFunction for_each(R &&Range, UnaryFunction F) { |
1564 | return std::for_each(adl_begin(Range), adl_end(Range), F); |
1565 | } |
1566 | |
1567 | /// Provide wrappers to std::all_of which take ranges instead of having to pass |
1568 | /// begin/end explicitly. |
1569 | template <typename R, typename UnaryPredicate> |
1570 | bool all_of(R &&Range, UnaryPredicate P) { |
1571 | return std::all_of(adl_begin(Range), adl_end(Range), P); |
1572 | } |
1573 | |
1574 | /// Provide wrappers to std::any_of which take ranges instead of having to pass |
1575 | /// begin/end explicitly. |
1576 | template <typename R, typename UnaryPredicate> |
1577 | bool any_of(R &&Range, UnaryPredicate P) { |
1578 | return std::any_of(adl_begin(Range), adl_end(Range), P); |
1579 | } |
1580 | |
1581 | /// Provide wrappers to std::none_of which take ranges instead of having to pass |
1582 | /// begin/end explicitly. |
1583 | template <typename R, typename UnaryPredicate> |
1584 | bool none_of(R &&Range, UnaryPredicate P) { |
1585 | return std::none_of(adl_begin(Range), adl_end(Range), P); |
1586 | } |
1587 | |
1588 | /// Provide wrappers to std::find which take ranges instead of having to pass |
1589 | /// begin/end explicitly. |
1590 | template <typename R, typename T> auto find(R &&Range, const T &Val) { |
1591 | return std::find(adl_begin(Range), adl_end(Range), Val); |
1592 | } |
1593 | |
1594 | /// Provide wrappers to std::find_if which take ranges instead of having to pass |
1595 | /// begin/end explicitly. |
1596 | template <typename R, typename UnaryPredicate> |
1597 | auto find_if(R &&Range, UnaryPredicate P) { |
1598 | return std::find_if(adl_begin(Range), adl_end(Range), P); |
1599 | } |
1600 | |
1601 | template <typename R, typename UnaryPredicate> |
1602 | auto find_if_not(R &&Range, UnaryPredicate P) { |
1603 | return std::find_if_not(adl_begin(Range), adl_end(Range), P); |
1604 | } |
1605 | |
1606 | /// Provide wrappers to std::remove_if which take ranges instead of having to |
1607 | /// pass begin/end explicitly. |
1608 | template <typename R, typename UnaryPredicate> |
1609 | auto remove_if(R &&Range, UnaryPredicate P) { |
1610 | return std::remove_if(adl_begin(Range), adl_end(Range), P); |
1611 | } |
1612 | |
1613 | /// Provide wrappers to std::copy_if which take ranges instead of having to |
1614 | /// pass begin/end explicitly. |
1615 | template <typename R, typename OutputIt, typename UnaryPredicate> |
1616 | OutputIt copy_if(R &&Range, OutputIt Out, UnaryPredicate P) { |
1617 | return std::copy_if(adl_begin(Range), adl_end(Range), Out, P); |
1618 | } |
1619 | |
1620 | template <typename R, typename OutputIt> |
1621 | OutputIt copy(R &&Range, OutputIt Out) { |
1622 | return std::copy(adl_begin(Range), adl_end(Range), Out); |
1623 | } |
1624 | |
1625 | /// Provide wrappers to std::move which take ranges instead of having to |
1626 | /// pass begin/end explicitly. |
1627 | template <typename R, typename OutputIt> |
1628 | OutputIt move(R &&Range, OutputIt Out) { |
1629 | return std::move(adl_begin(Range), adl_end(Range), Out); |
1630 | } |
1631 | |
1632 | /// Wrapper function around std::find to detect if an element exists |
1633 | /// in a container. |
1634 | template <typename R, typename E> |
1635 | bool is_contained(R &&Range, const E &Element) { |
1636 | return std::find(adl_begin(Range), adl_end(Range), Element) != adl_end(Range); |
1637 | } |
1638 | |
1639 | /// Wrapper function around std::is_sorted to check if elements in a range \p R |
1640 | /// are sorted with respect to a comparator \p C. |
1641 | template <typename R, typename Compare> bool is_sorted(R &&Range, Compare C) { |
1642 | return std::is_sorted(adl_begin(Range), adl_end(Range), C); |
1643 | } |
1644 | |
1645 | /// Wrapper function around std::is_sorted to check if elements in a range \p R |
1646 | /// are sorted in non-descending order. |
1647 | template <typename R> bool is_sorted(R &&Range) { |
1648 | return std::is_sorted(adl_begin(Range), adl_end(Range)); |
1649 | } |
1650 | |
1651 | /// Wrapper function around std::count to count the number of times an element |
1652 | /// \p Element occurs in the given range \p Range. |
1653 | template <typename R, typename E> auto count(R &&Range, const E &Element) { |
1654 | return std::count(adl_begin(Range), adl_end(Range), Element); |
1655 | } |
1656 | |
1657 | /// Wrapper function around std::count_if to count the number of times an |
1658 | /// element satisfying a given predicate occurs in a range. |
1659 | template <typename R, typename UnaryPredicate> |
1660 | auto count_if(R &&Range, UnaryPredicate P) { |
1661 | return std::count_if(adl_begin(Range), adl_end(Range), P); |
1662 | } |
1663 | |
1664 | /// Wrapper function around std::transform to apply a function to a range and |
1665 | /// store the result elsewhere. |
1666 | template <typename R, typename OutputIt, typename UnaryFunction> |
1667 | OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F) { |
1668 | return std::transform(adl_begin(Range), adl_end(Range), d_first, F); |
1669 | } |
1670 | |
1671 | /// Provide wrappers to std::partition which take ranges instead of having to |
1672 | /// pass begin/end explicitly. |
1673 | template <typename R, typename UnaryPredicate> |
1674 | auto partition(R &&Range, UnaryPredicate P) { |
1675 | return std::partition(adl_begin(Range), adl_end(Range), P); |
1676 | } |
1677 | |
1678 | /// Provide wrappers to std::lower_bound which take ranges instead of having to |
1679 | /// pass begin/end explicitly. |
1680 | template <typename R, typename T> auto lower_bound(R &&Range, T &&Value) { |
1681 | return std::lower_bound(adl_begin(Range), adl_end(Range), |
1682 | std::forward<T>(Value)); |
1683 | } |
1684 | |
1685 | template <typename R, typename T, typename Compare> |
1686 | auto lower_bound(R &&Range, T &&Value, Compare C) { |
1687 | return std::lower_bound(adl_begin(Range), adl_end(Range), |
1688 | std::forward<T>(Value), C); |
1689 | } |
1690 | |
1691 | /// Provide wrappers to std::upper_bound which take ranges instead of having to |
1692 | /// pass begin/end explicitly. |
1693 | template <typename R, typename T> auto upper_bound(R &&Range, T &&Value) { |
1694 | return std::upper_bound(adl_begin(Range), adl_end(Range), |
1695 | std::forward<T>(Value)); |
1696 | } |
1697 | |
1698 | template <typename R, typename T, typename Compare> |
1699 | auto upper_bound(R &&Range, T &&Value, Compare C) { |
1700 | return std::upper_bound(adl_begin(Range), adl_end(Range), |
1701 | std::forward<T>(Value), C); |
1702 | } |
1703 | |
1704 | template <typename R> |
1705 | void stable_sort(R &&Range) { |
1706 | std::stable_sort(adl_begin(Range), adl_end(Range)); |
1707 | } |
1708 | |
1709 | template <typename R, typename Compare> |
1710 | void stable_sort(R &&Range, Compare C) { |
1711 | std::stable_sort(adl_begin(Range), adl_end(Range), C); |
1712 | } |
1713 | |
1714 | /// Binary search for the first iterator in a range where a predicate is false. |
1715 | /// Requires that C is always true below some limit, and always false above it. |
1716 | template <typename R, typename Predicate, |
1717 | typename Val = decltype(*adl_begin(std::declval<R>()))> |
1718 | auto partition_point(R &&Range, Predicate P) { |
1719 | return std::partition_point(adl_begin(Range), adl_end(Range), P); |
1720 | } |
1721 | |
1722 | template<typename Range, typename Predicate> |
1723 | auto unique(Range &&R, Predicate P) { |
1724 | return std::unique(adl_begin(R), adl_end(R), P); |
1725 | } |
1726 | |
1727 | /// Wrapper function around std::equal to detect if pair-wise elements between |
1728 | /// two ranges are the same. |
1729 | template <typename L, typename R> bool equal(L &&LRange, R &&RRange) { |
1730 | return std::equal(adl_begin(LRange), adl_end(LRange), adl_begin(RRange), |
1731 | adl_end(RRange)); |
1732 | } |
1733 | |
1734 | /// Wrapper function around std::equal to detect if all elements |
1735 | /// in a container are same. |
1736 | template <typename R> |
1737 | bool is_splat(R &&Range) { |
1738 | size_t range_size = size(Range); |
1739 | return range_size != 0 && (range_size == 1 || |
1740 | std::equal(adl_begin(Range) + 1, adl_end(Range), adl_begin(Range))); |
1741 | } |
1742 | |
1743 | /// Provide a container algorithm similar to C++ Library Fundamentals v2's |
1744 | /// `erase_if` which is equivalent to: |
1745 | /// |
1746 | /// C.erase(remove_if(C, pred), C.end()); |
1747 | /// |
1748 | /// This version works for any container with an erase method call accepting |
1749 | /// two iterators. |
1750 | template <typename Container, typename UnaryPredicate> |
1751 | void erase_if(Container &C, UnaryPredicate P) { |
1752 | C.erase(remove_if(C, P), C.end()); |
1753 | } |
1754 | |
1755 | /// Wrapper function to remove a value from a container: |
1756 | /// |
1757 | /// C.erase(remove(C.begin(), C.end(), V), C.end()); |
1758 | template <typename Container, typename ValueType> |
1759 | void erase_value(Container &C, ValueType V) { |
1760 | C.erase(std::remove(C.begin(), C.end(), V), C.end()); |
1761 | } |
1762 | |
1763 | /// Wrapper function to append a range to a container. |
1764 | /// |
1765 | /// C.insert(C.end(), R.begin(), R.end()); |
1766 | template <typename Container, typename Range> |
1767 | inline void append_range(Container &C, Range &&R) { |
1768 | C.insert(C.end(), R.begin(), R.end()); |
1769 | } |
1770 | |
1771 | /// Given a sequence container Cont, replace the range [ContIt, ContEnd) with |
1772 | /// the range [ValIt, ValEnd) (which is not from the same container). |
1773 | template<typename Container, typename RandomAccessIterator> |
1774 | void replace(Container &Cont, typename Container::iterator ContIt, |
1775 | typename Container::iterator ContEnd, RandomAccessIterator ValIt, |
1776 | RandomAccessIterator ValEnd) { |
1777 | while (true) { |
1778 | if (ValIt == ValEnd) { |
1779 | Cont.erase(ContIt, ContEnd); |
1780 | return; |
1781 | } else if (ContIt == ContEnd) { |
1782 | Cont.insert(ContIt, ValIt, ValEnd); |
1783 | return; |
1784 | } |
1785 | *ContIt++ = *ValIt++; |
1786 | } |
1787 | } |
1788 | |
1789 | /// Given a sequence container Cont, replace the range [ContIt, ContEnd) with |
1790 | /// the range R. |
1791 | template<typename Container, typename Range = std::initializer_list< |
1792 | typename Container::value_type>> |
1793 | void replace(Container &Cont, typename Container::iterator ContIt, |
1794 | typename Container::iterator ContEnd, Range R) { |
1795 | replace(Cont, ContIt, ContEnd, R.begin(), R.end()); |
1796 | } |
1797 | |
1798 | /// An STL-style algorithm similar to std::for_each that applies a second |
1799 | /// functor between every pair of elements. |
1800 | /// |
1801 | /// This provides the control flow logic to, for example, print a |
1802 | /// comma-separated list: |
1803 | /// \code |
1804 | /// interleave(names.begin(), names.end(), |
1805 | /// [&](StringRef name) { os << name; }, |
1806 | /// [&] { os << ", "; }); |
1807 | /// \endcode |
1808 | template <typename ForwardIterator, typename UnaryFunctor, |
1809 | typename NullaryFunctor, |
1810 | typename = typename std::enable_if< |
1811 | !std::is_constructible<StringRef, UnaryFunctor>::value && |
1812 | !std::is_constructible<StringRef, NullaryFunctor>::value>::type> |
1813 | inline void interleave(ForwardIterator begin, ForwardIterator end, |
1814 | UnaryFunctor each_fn, NullaryFunctor between_fn) { |
1815 | if (begin == end) |
1816 | return; |
1817 | each_fn(*begin); |
1818 | ++begin; |
1819 | for (; begin != end; ++begin) { |
1820 | between_fn(); |
1821 | each_fn(*begin); |
1822 | } |
1823 | } |
1824 | |
1825 | template <typename Container, typename UnaryFunctor, typename NullaryFunctor, |
1826 | typename = typename std::enable_if< |
1827 | !std::is_constructible<StringRef, UnaryFunctor>::value && |
1828 | !std::is_constructible<StringRef, NullaryFunctor>::value>::type> |
1829 | inline void interleave(const Container &c, UnaryFunctor each_fn, |
1830 | NullaryFunctor between_fn) { |
1831 | interleave(c.begin(), c.end(), each_fn, between_fn); |
1832 | } |
1833 | |
1834 | /// Overload of interleave for the common case of string separator. |
1835 | template <typename Container, typename UnaryFunctor, typename StreamT, |
1836 | typename T = detail::ValueOfRange<Container>> |
1837 | inline void interleave(const Container &c, StreamT &os, UnaryFunctor each_fn, |
1838 | const StringRef &separator) { |
1839 | interleave(c.begin(), c.end(), each_fn, [&] { os << separator; }); |
1840 | } |
1841 | template <typename Container, typename StreamT, |
1842 | typename T = detail::ValueOfRange<Container>> |
1843 | inline void interleave(const Container &c, StreamT &os, |
1844 | const StringRef &separator) { |
1845 | interleave( |
1846 | c, os, [&](const T &a) { os << a; }, separator); |
1847 | } |
1848 | |
1849 | template <typename Container, typename UnaryFunctor, typename StreamT, |
1850 | typename T = detail::ValueOfRange<Container>> |
1851 | inline void interleaveComma(const Container &c, StreamT &os, |
1852 | UnaryFunctor each_fn) { |
1853 | interleave(c, os, each_fn, ", "); |
1854 | } |
1855 | template <typename Container, typename StreamT, |
1856 | typename T = detail::ValueOfRange<Container>> |
1857 | inline void interleaveComma(const Container &c, StreamT &os) { |
1858 | interleaveComma(c, os, [&](const T &a) { os << a; }); |
1859 | } |
1860 | |
1861 | //===----------------------------------------------------------------------===// |
1862 | // Extra additions to <memory> |
1863 | //===----------------------------------------------------------------------===// |
1864 | |
1865 | struct FreeDeleter { |
1866 | void operator()(void* v) { |
1867 | ::free(v); |
1868 | } |
1869 | }; |
1870 | |
1871 | template<typename First, typename Second> |
1872 | struct pair_hash { |
1873 | size_t operator()(const std::pair<First, Second> &P) const { |
1874 | return std::hash<First>()(P.first) * 31 + std::hash<Second>()(P.second); |
1875 | } |
1876 | }; |
1877 | |
1878 | /// Binary functor that adapts to any other binary functor after dereferencing |
1879 | /// operands. |
1880 | template <typename T> struct deref { |
1881 | T func; |
1882 | |
1883 | // Could be further improved to cope with non-derivable functors and |
1884 | // non-binary functors (should be a variadic template member function |
1885 | // operator()). |
1886 | template <typename A, typename B> auto operator()(A &lhs, B &rhs) const { |
1887 | assert(lhs)(static_cast <bool> (lhs) ? void (0) : __assert_fail ("lhs" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1887, __extension__ __PRETTY_FUNCTION__)); |
1888 | assert(rhs)(static_cast <bool> (rhs) ? void (0) : __assert_fail ("rhs" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1888, __extension__ __PRETTY_FUNCTION__)); |
1889 | return func(*lhs, *rhs); |
1890 | } |
1891 | }; |
1892 | |
1893 | namespace detail { |
1894 | |
1895 | template <typename R> class enumerator_iter; |
1896 | |
1897 | template <typename R> struct result_pair { |
1898 | using value_reference = |
1899 | typename std::iterator_traits<IterOfRange<R>>::reference; |
1900 | |
1901 | friend class enumerator_iter<R>; |
1902 | |
1903 | result_pair() = default; |
1904 | result_pair(std::size_t Index, IterOfRange<R> Iter) |
1905 | : Index(Index), Iter(Iter) {} |
1906 | |
1907 | result_pair(const result_pair<R> &Other) |
1908 | : Index(Other.Index), Iter(Other.Iter) {} |
1909 | result_pair &operator=(const result_pair &Other) { |
1910 | Index = Other.Index; |
1911 | Iter = Other.Iter; |
1912 | return *this; |
1913 | } |
1914 | |
1915 | std::size_t index() const { return Index; } |
1916 | const value_reference value() const { return *Iter; } |
1917 | value_reference value() { return *Iter; } |
1918 | |
1919 | private: |
1920 | std::size_t Index = std::numeric_limits<std::size_t>::max(); |
1921 | IterOfRange<R> Iter; |
1922 | }; |
1923 | |
1924 | template <typename R> |
1925 | class enumerator_iter |
1926 | : public iterator_facade_base< |
1927 | enumerator_iter<R>, std::forward_iterator_tag, result_pair<R>, |
1928 | typename std::iterator_traits<IterOfRange<R>>::difference_type, |
1929 | typename std::iterator_traits<IterOfRange<R>>::pointer, |
1930 | typename std::iterator_traits<IterOfRange<R>>::reference> { |
1931 | using result_type = result_pair<R>; |
1932 | |
1933 | public: |
1934 | explicit enumerator_iter(IterOfRange<R> EndIter) |
1935 | : Result(std::numeric_limits<size_t>::max(), EndIter) {} |
1936 | |
1937 | enumerator_iter(std::size_t Index, IterOfRange<R> Iter) |
1938 | : Result(Index, Iter) {} |
1939 | |
1940 | result_type &operator*() { return Result; } |
1941 | const result_type &operator*() const { return Result; } |
1942 | |
1943 | enumerator_iter &operator++() { |
1944 | assert(Result.Index != std::numeric_limits<size_t>::max())(static_cast <bool> (Result.Index != std::numeric_limits <size_t>::max()) ? void (0) : __assert_fail ("Result.Index != std::numeric_limits<size_t>::max()" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 1944, __extension__ __PRETTY_FUNCTION__)); |
1945 | ++Result.Iter; |
1946 | ++Result.Index; |
1947 | return *this; |
1948 | } |
1949 | |
1950 | bool operator==(const enumerator_iter &RHS) const { |
1951 | // Don't compare indices here, only iterators. It's possible for an end |
1952 | // iterator to have different indices depending on whether it was created |
1953 | // by calling std::end() versus incrementing a valid iterator. |
1954 | return Result.Iter == RHS.Result.Iter; |
1955 | } |
1956 | |
1957 | enumerator_iter(const enumerator_iter &Other) : Result(Other.Result) {} |
1958 | enumerator_iter &operator=(const enumerator_iter &Other) { |
1959 | Result = Other.Result; |
1960 | return *this; |
1961 | } |
1962 | |
1963 | private: |
1964 | result_type Result; |
1965 | }; |
1966 | |
1967 | template <typename R> class enumerator { |
1968 | public: |
1969 | explicit enumerator(R &&Range) : TheRange(std::forward<R>(Range)) {} |
1970 | |
1971 | enumerator_iter<R> begin() { |
1972 | return enumerator_iter<R>(0, std::begin(TheRange)); |
1973 | } |
1974 | |
1975 | enumerator_iter<R> end() { |
1976 | return enumerator_iter<R>(std::end(TheRange)); |
1977 | } |
1978 | |
1979 | private: |
1980 | R TheRange; |
1981 | }; |
1982 | |
1983 | } // end namespace detail |
1984 | |
1985 | /// Given an input range, returns a new range whose values are are pair (A,B) |
1986 | /// such that A is the 0-based index of the item in the sequence, and B is |
1987 | /// the value from the original sequence. Example: |
1988 | /// |
1989 | /// std::vector<char> Items = {'A', 'B', 'C', 'D'}; |
1990 | /// for (auto X : enumerate(Items)) { |
1991 | /// printf("Item %d - %c\n", X.index(), X.value()); |
1992 | /// } |
1993 | /// |
1994 | /// Output: |
1995 | /// Item 0 - A |
1996 | /// Item 1 - B |
1997 | /// Item 2 - C |
1998 | /// Item 3 - D |
1999 | /// |
2000 | template <typename R> detail::enumerator<R> enumerate(R &&TheRange) { |
2001 | return detail::enumerator<R>(std::forward<R>(TheRange)); |
2002 | } |
2003 | |
2004 | namespace detail { |
2005 | |
2006 | template <typename F, typename Tuple, std::size_t... I> |
2007 | decltype(auto) apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>) { |
2008 | return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...); |
2009 | } |
2010 | |
2011 | } // end namespace detail |
2012 | |
2013 | /// Given an input tuple (a1, a2, ..., an), pass the arguments of the |
2014 | /// tuple variadically to f as if by calling f(a1, a2, ..., an) and |
2015 | /// return the result. |
2016 | template <typename F, typename Tuple> |
2017 | decltype(auto) apply_tuple(F &&f, Tuple &&t) { |
2018 | using Indices = std::make_index_sequence< |
2019 | std::tuple_size<typename std::decay<Tuple>::type>::value>; |
2020 | |
2021 | return detail::apply_tuple_impl(std::forward<F>(f), std::forward<Tuple>(t), |
2022 | Indices{}); |
2023 | } |
2024 | |
2025 | namespace detail { |
2026 | |
2027 | template <typename Predicate, typename... Args> |
2028 | bool all_of_zip_predicate_first(Predicate &&P, Args &&...args) { |
2029 | auto z = zip(args...); |
2030 | auto it = z.begin(); |
2031 | auto end = z.end(); |
2032 | while (it != end) { |
2033 | if (!apply_tuple([&](auto &&...args) { return P(args...); }, *it)) |
2034 | return false; |
2035 | ++it; |
2036 | } |
2037 | return it.all_equals(end); |
2038 | } |
2039 | |
2040 | // Just an adaptor to switch the order of argument and have the predicate before |
2041 | // the zipped inputs. |
2042 | template <typename... ArgsThenPredicate, size_t... InputIndexes> |
2043 | bool all_of_zip_predicate_last( |
2044 | std::tuple<ArgsThenPredicate...> argsThenPredicate, |
2045 | std::index_sequence<InputIndexes...>) { |
2046 | auto constexpr OutputIndex = |
2047 | std::tuple_size<decltype(argsThenPredicate)>::value - 1; |
2048 | return all_of_zip_predicate_first(std::get<OutputIndex>(argsThenPredicate), |
2049 | std::get<InputIndexes>(argsThenPredicate)...); |
2050 | } |
2051 | |
2052 | } // end namespace detail |
2053 | |
2054 | /// Compare two zipped ranges using the provided predicate (as last argument). |
2055 | /// Return true if all elements satisfy the predicate and false otherwise. |
2056 | // Return false if the zipped iterator aren't all at end (size mismatch). |
2057 | template <typename... ArgsAndPredicate> |
2058 | bool all_of_zip(ArgsAndPredicate &&...argsAndPredicate) { |
2059 | return detail::all_of_zip_predicate_last( |
2060 | std::forward_as_tuple(argsAndPredicate...), |
2061 | std::make_index_sequence<sizeof...(argsAndPredicate) - 1>{}); |
2062 | } |
2063 | |
2064 | /// Return true if the sequence [Begin, End) has exactly N items. Runs in O(N) |
2065 | /// time. Not meant for use with random-access iterators. |
2066 | /// Can optionally take a predicate to filter lazily some items. |
2067 | template <typename IterTy, |
2068 | typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)> |
2069 | bool hasNItems( |
2070 | IterTy &&Begin, IterTy &&End, unsigned N, |
2071 | Pred &&ShouldBeCounted = |
2072 | [](const decltype(*std::declval<IterTy>()) &) { return true; }, |
2073 | std::enable_if_t< |
2074 | !std::is_base_of<std::random_access_iterator_tag, |
2075 | typename std::iterator_traits<std::remove_reference_t< |
2076 | decltype(Begin)>>::iterator_category>::value, |
2077 | void> * = nullptr) { |
2078 | for (; N; ++Begin) { |
2079 | if (Begin == End) |
2080 | return false; // Too few. |
2081 | N -= ShouldBeCounted(*Begin); |
2082 | } |
2083 | for (; Begin != End; ++Begin) |
2084 | if (ShouldBeCounted(*Begin)) |
2085 | return false; // Too many. |
2086 | return true; |
2087 | } |
2088 | |
2089 | /// Return true if the sequence [Begin, End) has N or more items. Runs in O(N) |
2090 | /// time. Not meant for use with random-access iterators. |
2091 | /// Can optionally take a predicate to lazily filter some items. |
2092 | template <typename IterTy, |
2093 | typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)> |
2094 | bool hasNItemsOrMore( |
2095 | IterTy &&Begin, IterTy &&End, unsigned N, |
2096 | Pred &&ShouldBeCounted = |
2097 | [](const decltype(*std::declval<IterTy>()) &) { return true; }, |
2098 | std::enable_if_t< |
2099 | !std::is_base_of<std::random_access_iterator_tag, |
2100 | typename std::iterator_traits<std::remove_reference_t< |
2101 | decltype(Begin)>>::iterator_category>::value, |
2102 | void> * = nullptr) { |
2103 | for (; N; ++Begin) { |
2104 | if (Begin == End) |
2105 | return false; // Too few. |
2106 | N -= ShouldBeCounted(*Begin); |
2107 | } |
2108 | return true; |
2109 | } |
2110 | |
2111 | /// Returns true if the sequence [Begin, End) has N or less items. Can |
2112 | /// optionally take a predicate to lazily filter some items. |
2113 | template <typename IterTy, |
2114 | typename Pred = bool (*)(const decltype(*std::declval<IterTy>()) &)> |
2115 | bool hasNItemsOrLess( |
2116 | IterTy &&Begin, IterTy &&End, unsigned N, |
2117 | Pred &&ShouldBeCounted = [](const decltype(*std::declval<IterTy>()) &) { |
2118 | return true; |
2119 | }) { |
2120 | assert(N != std::numeric_limits<unsigned>::max())(static_cast <bool> (N != std::numeric_limits<unsigned >::max()) ? void (0) : __assert_fail ("N != std::numeric_limits<unsigned>::max()" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/ADT/STLExtras.h" , 2120, __extension__ __PRETTY_FUNCTION__)); |
2121 | return !hasNItemsOrMore(Begin, End, N + 1, ShouldBeCounted); |
2122 | } |
2123 | |
2124 | /// Returns true if the given container has exactly N items |
2125 | template <typename ContainerTy> bool hasNItems(ContainerTy &&C, unsigned N) { |
2126 | return hasNItems(std::begin(C), std::end(C), N); |
2127 | } |
2128 | |
2129 | /// Returns true if the given container has N or more items |
2130 | template <typename ContainerTy> |
2131 | bool hasNItemsOrMore(ContainerTy &&C, unsigned N) { |
2132 | return hasNItemsOrMore(std::begin(C), std::end(C), N); |
2133 | } |
2134 | |
2135 | /// Returns true if the given container has N or less items |
2136 | template <typename ContainerTy> |
2137 | bool hasNItemsOrLess(ContainerTy &&C, unsigned N) { |
2138 | return hasNItemsOrLess(std::begin(C), std::end(C), N); |
2139 | } |
2140 | |
2141 | /// Returns a raw pointer that represents the same address as the argument. |
2142 | /// |
2143 | /// This implementation can be removed once we move to C++20 where it's defined |
2144 | /// as std::to_address(). |
2145 | /// |
2146 | /// The std::pointer_traits<>::to_address(p) variations of these overloads has |
2147 | /// not been implemented. |
2148 | template <class Ptr> auto to_address(const Ptr &P) { return P.operator->(); } |
2149 | template <class T> constexpr T *to_address(T *P) { return P; } |
2150 | |
2151 | } // end namespace llvm |
2152 | |
2153 | #endif // LLVM_ADT_STLEXTRAS_H |
1 | //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the declaration of the MachineInstr class, which is the |
10 | // basic representation for all target dependent machine instructions used by |
11 | // the back end. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #ifndef LLVM_CODEGEN_MACHINEINSTR_H |
16 | #define LLVM_CODEGEN_MACHINEINSTR_H |
17 | |
18 | #include "llvm/ADT/DenseMapInfo.h" |
19 | #include "llvm/ADT/PointerSumType.h" |
20 | #include "llvm/ADT/SmallSet.h" |
21 | #include "llvm/ADT/ilist.h" |
22 | #include "llvm/ADT/ilist_node.h" |
23 | #include "llvm/ADT/iterator_range.h" |
24 | #include "llvm/CodeGen/MachineMemOperand.h" |
25 | #include "llvm/CodeGen/MachineOperand.h" |
26 | #include "llvm/CodeGen/TargetOpcodes.h" |
27 | #include "llvm/IR/DebugLoc.h" |
28 | #include "llvm/IR/InlineAsm.h" |
29 | #include "llvm/IR/PseudoProbe.h" |
30 | #include "llvm/MC/MCInstrDesc.h" |
31 | #include "llvm/MC/MCSymbol.h" |
32 | #include "llvm/Support/ArrayRecycler.h" |
33 | #include "llvm/Support/TrailingObjects.h" |
34 | #include <algorithm> |
35 | #include <cassert> |
36 | #include <cstdint> |
37 | #include <utility> |
38 | |
39 | namespace llvm { |
40 | |
41 | class AAResults; |
42 | template <typename T> class ArrayRef; |
43 | class DIExpression; |
44 | class DILocalVariable; |
45 | class MachineBasicBlock; |
46 | class MachineFunction; |
47 | class MachineRegisterInfo; |
48 | class ModuleSlotTracker; |
49 | class raw_ostream; |
50 | template <typename T> class SmallVectorImpl; |
51 | class SmallBitVector; |
52 | class StringRef; |
53 | class TargetInstrInfo; |
54 | class TargetRegisterClass; |
55 | class TargetRegisterInfo; |
56 | |
57 | //===----------------------------------------------------------------------===// |
58 | /// Representation of each machine instruction. |
59 | /// |
60 | /// This class isn't a POD type, but it must have a trivial destructor. When a |
61 | /// MachineFunction is deleted, all the contained MachineInstrs are deallocated |
62 | /// without having their destructor called. |
63 | /// |
64 | class MachineInstr |
65 | : public ilist_node_with_parent<MachineInstr, MachineBasicBlock, |
66 | ilist_sentinel_tracking<true>> { |
67 | public: |
68 | using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator; |
69 | |
70 | /// Flags to specify different kinds of comments to output in |
71 | /// assembly code. These flags carry semantic information not |
72 | /// otherwise easily derivable from the IR text. |
73 | /// |
74 | enum CommentFlag { |
75 | ReloadReuse = 0x1, // higher bits are reserved for target dep comments. |
76 | NoSchedComment = 0x2, |
77 | TAsmComments = 0x4 // Target Asm comments should start from this value. |
78 | }; |
79 | |
80 | enum MIFlag { |
81 | NoFlags = 0, |
82 | FrameSetup = 1 << 0, // Instruction is used as a part of |
83 | // function frame setup code. |
84 | FrameDestroy = 1 << 1, // Instruction is used as a part of |
85 | // function frame destruction code. |
86 | BundledPred = 1 << 2, // Instruction has bundled predecessors. |
87 | BundledSucc = 1 << 3, // Instruction has bundled successors. |
88 | FmNoNans = 1 << 4, // Instruction does not support Fast |
89 | // math nan values. |
90 | FmNoInfs = 1 << 5, // Instruction does not support Fast |
91 | // math infinity values. |
92 | FmNsz = 1 << 6, // Instruction is not required to retain |
93 | // signed zero values. |
94 | FmArcp = 1 << 7, // Instruction supports Fast math |
95 | // reciprocal approximations. |
96 | FmContract = 1 << 8, // Instruction supports Fast math |
97 | // contraction operations like fma. |
98 | FmAfn = 1 << 9, // Instruction may map to Fast math |
99 | // instrinsic approximation. |
100 | FmReassoc = 1 << 10, // Instruction supports Fast math |
101 | // reassociation of operand order. |
102 | NoUWrap = 1 << 11, // Instruction supports binary operator |
103 | // no unsigned wrap. |
104 | NoSWrap = 1 << 12, // Instruction supports binary operator |
105 | // no signed wrap. |
106 | IsExact = 1 << 13, // Instruction supports division is |
107 | // known to be exact. |
108 | NoFPExcept = 1 << 14, // Instruction does not raise |
109 | // floatint-point exceptions. |
110 | NoMerge = 1 << 15, // Passes that drop source location info |
111 | // (e.g. branch folding) should skip |
112 | // this instruction. |
113 | }; |
114 | |
115 | private: |
116 | const MCInstrDesc *MCID; // Instruction descriptor. |
117 | MachineBasicBlock *Parent = nullptr; // Pointer to the owning basic block. |
118 | |
119 | // Operands are allocated by an ArrayRecycler. |
120 | MachineOperand *Operands = nullptr; // Pointer to the first operand. |
121 | unsigned NumOperands = 0; // Number of operands on instruction. |
122 | |
123 | uint16_t Flags = 0; // Various bits of additional |
124 | // information about machine |
125 | // instruction. |
126 | |
127 | uint8_t AsmPrinterFlags = 0; // Various bits of information used by |
128 | // the AsmPrinter to emit helpful |
129 | // comments. This is *not* semantic |
130 | // information. Do not use this for |
131 | // anything other than to convey comment |
132 | // information to AsmPrinter. |
133 | |
134 | // OperandCapacity has uint8_t size, so it should be next to AsmPrinterFlags |
135 | // to properly pack. |
136 | using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity; |
137 | OperandCapacity CapOperands; // Capacity of the Operands array. |
138 | |
139 | /// Internal implementation detail class that provides out-of-line storage for |
140 | /// extra info used by the machine instruction when this info cannot be stored |
141 | /// in-line within the instruction itself. |
142 | /// |
143 | /// This has to be defined eagerly due to the implementation constraints of |
144 | /// `PointerSumType` where it is used. |
145 | class ExtraInfo final |
146 | : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> { |
147 | public: |
148 | static ExtraInfo *create(BumpPtrAllocator &Allocator, |
149 | ArrayRef<MachineMemOperand *> MMOs, |
150 | MCSymbol *PreInstrSymbol = nullptr, |
151 | MCSymbol *PostInstrSymbol = nullptr, |
152 | MDNode *HeapAllocMarker = nullptr) { |
153 | bool HasPreInstrSymbol = PreInstrSymbol != nullptr; |
154 | bool HasPostInstrSymbol = PostInstrSymbol != nullptr; |
155 | bool HasHeapAllocMarker = HeapAllocMarker != nullptr; |
156 | auto *Result = new (Allocator.Allocate( |
157 | totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>( |
158 | MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, |
159 | HasHeapAllocMarker), |
160 | alignof(ExtraInfo))) |
161 | ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, |
162 | HasHeapAllocMarker); |
163 | |
164 | // Copy the actual data into the trailing objects. |
165 | std::copy(MMOs.begin(), MMOs.end(), |
166 | Result->getTrailingObjects<MachineMemOperand *>()); |
167 | |
168 | if (HasPreInstrSymbol) |
169 | Result->getTrailingObjects<MCSymbol *>()[0] = PreInstrSymbol; |
170 | if (HasPostInstrSymbol) |
171 | Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] = |
172 | PostInstrSymbol; |
173 | if (HasHeapAllocMarker) |
174 | Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker; |
175 | |
176 | return Result; |
177 | } |
178 | |
179 | ArrayRef<MachineMemOperand *> getMMOs() const { |
180 | return makeArrayRef(getTrailingObjects<MachineMemOperand *>(), NumMMOs); |
181 | } |
182 | |
183 | MCSymbol *getPreInstrSymbol() const { |
184 | return HasPreInstrSymbol ? getTrailingObjects<MCSymbol *>()[0] : nullptr; |
185 | } |
186 | |
187 | MCSymbol *getPostInstrSymbol() const { |
188 | return HasPostInstrSymbol |
189 | ? getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] |
190 | : nullptr; |
191 | } |
192 | |
193 | MDNode *getHeapAllocMarker() const { |
194 | return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr; |
195 | } |
196 | |
197 | private: |
198 | friend TrailingObjects; |
199 | |
200 | // Description of the extra info, used to interpret the actual optional |
201 | // data appended. |
202 | // |
203 | // Note that this is not terribly space optimized. This leaves a great deal |
204 | // of flexibility to fit more in here later. |
205 | const int NumMMOs; |
206 | const bool HasPreInstrSymbol; |
207 | const bool HasPostInstrSymbol; |
208 | const bool HasHeapAllocMarker; |
209 | |
210 | // Implement the `TrailingObjects` internal API. |
211 | size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const { |
212 | return NumMMOs; |
213 | } |
214 | size_t numTrailingObjects(OverloadToken<MCSymbol *>) const { |
215 | return HasPreInstrSymbol + HasPostInstrSymbol; |
216 | } |
217 | size_t numTrailingObjects(OverloadToken<MDNode *>) const { |
218 | return HasHeapAllocMarker; |
219 | } |
220 | |
221 | // Just a boring constructor to allow us to initialize the sizes. Always use |
222 | // the `create` routine above. |
223 | ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, |
224 | bool HasHeapAllocMarker) |
225 | : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), |
226 | HasPostInstrSymbol(HasPostInstrSymbol), |
227 | HasHeapAllocMarker(HasHeapAllocMarker) {} |
228 | }; |
229 | |
230 | /// Enumeration of the kinds of inline extra info available. It is important |
231 | /// that the `MachineMemOperand` inline kind has a tag value of zero to make |
232 | /// it accessible as an `ArrayRef`. |
233 | enum ExtraInfoInlineKinds { |
234 | EIIK_MMO = 0, |
235 | EIIK_PreInstrSymbol, |
236 | EIIK_PostInstrSymbol, |
237 | EIIK_OutOfLine |
238 | }; |
239 | |
240 | // We store extra information about the instruction here. The common case is |
241 | // expected to be nothing or a single pointer (typically a MMO or a symbol). |
242 | // We work to optimize this common case by storing it inline here rather than |
243 | // requiring a separate allocation, but we fall back to an allocation when |
244 | // multiple pointers are needed. |
245 | PointerSumType<ExtraInfoInlineKinds, |
246 | PointerSumTypeMember<EIIK_MMO, MachineMemOperand *>, |
247 | PointerSumTypeMember<EIIK_PreInstrSymbol, MCSymbol *>, |
248 | PointerSumTypeMember<EIIK_PostInstrSymbol, MCSymbol *>, |
249 | PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>> |
250 | Info; |
251 | |
252 | DebugLoc debugLoc; // Source line information. |
253 | |
254 | /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values |
255 | /// defined by this instruction. |
256 | unsigned DebugInstrNum; |
257 | |
258 | // Intrusive list support |
259 | friend struct ilist_traits<MachineInstr>; |
260 | friend struct ilist_callback_traits<MachineBasicBlock>; |
261 | void setParent(MachineBasicBlock *P) { Parent = P; } |
262 | |
263 | /// This constructor creates a copy of the given |
264 | /// MachineInstr in the given MachineFunction. |
265 | MachineInstr(MachineFunction &, const MachineInstr &); |
266 | |
267 | /// This constructor create a MachineInstr and add the implicit operands. |
268 | /// It reserves space for number of operands specified by |
269 | /// MCInstrDesc. An explicit DebugLoc is supplied. |
270 | MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl, |
271 | bool NoImp = false); |
272 | |
273 | // MachineInstrs are pool-allocated and owned by MachineFunction. |
274 | friend class MachineFunction; |
275 | |
276 | void |
277 | dumprImpl(const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth, |
278 | SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const; |
279 | |
280 | public: |
281 | MachineInstr(const MachineInstr &) = delete; |
282 | MachineInstr &operator=(const MachineInstr &) = delete; |
283 | // Use MachineFunction::DeleteMachineInstr() instead. |
284 | ~MachineInstr() = delete; |
285 | |
286 | const MachineBasicBlock* getParent() const { return Parent; } |
287 | MachineBasicBlock* getParent() { return Parent; } |
288 | |
289 | /// Move the instruction before \p MovePos. |
290 | void moveBefore(MachineInstr *MovePos); |
291 | |
292 | /// Return the function that contains the basic block that this instruction |
293 | /// belongs to. |
294 | /// |
295 | /// Note: this is undefined behaviour if the instruction does not have a |
296 | /// parent. |
297 | const MachineFunction *getMF() const; |
298 | MachineFunction *getMF() { |
299 | return const_cast<MachineFunction *>( |
300 | static_cast<const MachineInstr *>(this)->getMF()); |
301 | } |
302 | |
303 | /// Return the asm printer flags bitvector. |
304 | uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } |
305 | |
306 | /// Clear the AsmPrinter bitvector. |
307 | void clearAsmPrinterFlags() { AsmPrinterFlags = 0; } |
308 | |
309 | /// Return whether an AsmPrinter flag is set. |
310 | bool getAsmPrinterFlag(CommentFlag Flag) const { |
311 | return AsmPrinterFlags & Flag; |
312 | } |
313 | |
314 | /// Set a flag for the AsmPrinter. |
315 | void setAsmPrinterFlag(uint8_t Flag) { |
316 | AsmPrinterFlags |= Flag; |
317 | } |
318 | |
319 | /// Clear specific AsmPrinter flags. |
320 | void clearAsmPrinterFlag(CommentFlag Flag) { |
321 | AsmPrinterFlags &= ~Flag; |
322 | } |
323 | |
324 | /// Return the MI flags bitvector. |
325 | uint16_t getFlags() const { |
326 | return Flags; |
327 | } |
328 | |
329 | /// Return whether an MI flag is set. |
330 | bool getFlag(MIFlag Flag) const { |
331 | return Flags & Flag; |
332 | } |
333 | |
334 | /// Set a MI flag. |
335 | void setFlag(MIFlag Flag) { |
336 | Flags |= (uint16_t)Flag; |
337 | } |
338 | |
339 | void setFlags(unsigned flags) { |
340 | // Filter out the automatically maintained flags. |
341 | unsigned Mask = BundledPred | BundledSucc; |
342 | Flags = (Flags & Mask) | (flags & ~Mask); |
343 | } |
344 | |
345 | /// clearFlag - Clear a MI flag. |
346 | void clearFlag(MIFlag Flag) { |
347 | Flags &= ~((uint16_t)Flag); |
348 | } |
349 | |
350 | /// Return true if MI is in a bundle (but not the first MI in a bundle). |
351 | /// |
352 | /// A bundle looks like this before it's finalized: |
353 | /// ---------------- |
354 | /// | MI | |
355 | /// ---------------- |
356 | /// | |
357 | /// ---------------- |
358 | /// | MI * | |
359 | /// ---------------- |
360 | /// | |
361 | /// ---------------- |
362 | /// | MI * | |
363 | /// ---------------- |
364 | /// In this case, the first MI starts a bundle but is not inside a bundle, the |
365 | /// next 2 MIs are considered "inside" the bundle. |
366 | /// |
367 | /// After a bundle is finalized, it looks like this: |
368 | /// ---------------- |
369 | /// | Bundle | |
370 | /// ---------------- |
371 | /// | |
372 | /// ---------------- |
373 | /// | MI * | |
374 | /// ---------------- |
375 | /// | |
376 | /// ---------------- |
377 | /// | MI * | |
378 | /// ---------------- |
379 | /// | |
380 | /// ---------------- |
381 | /// | MI * | |
382 | /// ---------------- |
383 | /// The first instruction has the special opcode "BUNDLE". It's not "inside" |
384 | /// a bundle, but the next three MIs are. |
385 | bool isInsideBundle() const { |
386 | return getFlag(BundledPred); |
387 | } |
388 | |
389 | /// Return true if this instruction part of a bundle. This is true |
390 | /// if either itself or its following instruction is marked "InsideBundle". |
391 | bool isBundled() const { |
392 | return isBundledWithPred() || isBundledWithSucc(); |
393 | } |
394 | |
395 | /// Return true if this instruction is part of a bundle, and it is not the |
396 | /// first instruction in the bundle. |
397 | bool isBundledWithPred() const { return getFlag(BundledPred); } |
398 | |
399 | /// Return true if this instruction is part of a bundle, and it is not the |
400 | /// last instruction in the bundle. |
401 | bool isBundledWithSucc() const { return getFlag(BundledSucc); } |
402 | |
403 | /// Bundle this instruction with its predecessor. This can be an unbundled |
404 | /// instruction, or it can be the first instruction in a bundle. |
405 | void bundleWithPred(); |
406 | |
407 | /// Bundle this instruction with its successor. This can be an unbundled |
408 | /// instruction, or it can be the last instruction in a bundle. |
409 | void bundleWithSucc(); |
410 | |
411 | /// Break bundle above this instruction. |
412 | void unbundleFromPred(); |
413 | |
414 | /// Break bundle below this instruction. |
415 | void unbundleFromSucc(); |
416 | |
417 | /// Returns the debug location id of this MachineInstr. |
418 | const DebugLoc &getDebugLoc() const { return debugLoc; } |
419 | |
420 | /// Return the operand containing the offset to be used if this DBG_VALUE |
421 | /// instruction is indirect; will be an invalid register if this value is |
422 | /// not indirect, and an immediate with value 0 otherwise. |
423 | const MachineOperand &getDebugOffset() const { |
424 | assert(isNonListDebugValue() && "not a DBG_VALUE")(static_cast <bool> (isNonListDebugValue() && "not a DBG_VALUE" ) ? void (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 424, __extension__ __PRETTY_FUNCTION__)); |
425 | return getOperand(1); |
426 | } |
427 | MachineOperand &getDebugOffset() { |
428 | assert(isNonListDebugValue() && "not a DBG_VALUE")(static_cast <bool> (isNonListDebugValue() && "not a DBG_VALUE" ) ? void (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 428, __extension__ __PRETTY_FUNCTION__)); |
429 | return getOperand(1); |
430 | } |
431 | |
432 | /// Return the operand for the debug variable referenced by |
433 | /// this DBG_VALUE instruction. |
434 | const MachineOperand &getDebugVariableOp() const; |
435 | MachineOperand &getDebugVariableOp(); |
436 | |
437 | /// Return the debug variable referenced by |
438 | /// this DBG_VALUE instruction. |
439 | const DILocalVariable *getDebugVariable() const; |
440 | |
441 | /// Return the operand for the complex address expression referenced by |
442 | /// this DBG_VALUE instruction. |
443 | const MachineOperand &getDebugExpressionOp() const; |
444 | MachineOperand &getDebugExpressionOp(); |
445 | |
446 | /// Return the complex address expression referenced by |
447 | /// this DBG_VALUE instruction. |
448 | const DIExpression *getDebugExpression() const; |
449 | |
450 | /// Return the debug label referenced by |
451 | /// this DBG_LABEL instruction. |
452 | const DILabel *getDebugLabel() const; |
453 | |
454 | /// Fetch the instruction number of this MachineInstr. If it does not have |
455 | /// one already, a new and unique number will be assigned. |
456 | unsigned getDebugInstrNum(); |
457 | |
458 | /// Fetch instruction number of this MachineInstr -- but before it's inserted |
459 | /// into \p MF. Needed for transformations that create an instruction but |
460 | /// don't immediately insert them. |
461 | unsigned getDebugInstrNum(MachineFunction &MF); |
462 | |
463 | /// Examine the instruction number of this MachineInstr. May be zero if |
464 | /// it hasn't been assigned a number yet. |
465 | unsigned peekDebugInstrNum() const { return DebugInstrNum; } |
466 | |
467 | /// Set instruction number of this MachineInstr. Avoid using unless you're |
468 | /// deserializing this information. |
469 | void setDebugInstrNum(unsigned Num) { DebugInstrNum = Num; } |
470 | |
471 | /// Drop any variable location debugging information associated with this |
472 | /// instruction. Use when an instruction is modified in such a way that it no |
473 | /// longer defines the value it used to. Variable locations using that value |
474 | /// will be dropped. |
475 | void dropDebugNumber() { DebugInstrNum = 0; } |
476 | |
477 | /// Emit an error referring to the source location of this instruction. |
478 | /// This should only be used for inline assembly that is somehow |
479 | /// impossible to compile. Other errors should have been handled much |
480 | /// earlier. |
481 | /// |
482 | /// If this method returns, the caller should try to recover from the error. |
483 | void emitError(StringRef Msg) const; |
484 | |
485 | /// Returns the target instruction descriptor of this MachineInstr. |
486 | const MCInstrDesc &getDesc() const { return *MCID; } |
487 | |
488 | /// Returns the opcode of this MachineInstr. |
489 | unsigned getOpcode() const { return MCID->Opcode; } |
490 | |
491 | /// Retuns the total number of operands. |
492 | unsigned getNumOperands() const { return NumOperands; } |
493 | |
494 | /// Returns the total number of operands which are debug locations. |
495 | unsigned getNumDebugOperands() const { |
496 | return std::distance(debug_operands().begin(), debug_operands().end()); |
497 | } |
498 | |
499 | const MachineOperand& getOperand(unsigned i) const { |
500 | assert(i < getNumOperands() && "getOperand() out of range!")(static_cast <bool> (i < getNumOperands() && "getOperand() out of range!") ? void (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 500, __extension__ __PRETTY_FUNCTION__)); |
501 | return Operands[i]; |
502 | } |
503 | MachineOperand& getOperand(unsigned i) { |
504 | assert(i < getNumOperands() && "getOperand() out of range!")(static_cast <bool> (i < getNumOperands() && "getOperand() out of range!") ? void (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 504, __extension__ __PRETTY_FUNCTION__)); |
505 | return Operands[i]; |
506 | } |
507 | |
508 | MachineOperand &getDebugOperand(unsigned Index) { |
509 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")(static_cast <bool> (Index < getNumDebugOperands() && "getDebugOperand() out of range!") ? void (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 509, __extension__ __PRETTY_FUNCTION__)); |
510 | return *(debug_operands().begin() + Index); |
511 | } |
512 | const MachineOperand &getDebugOperand(unsigned Index) const { |
513 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")(static_cast <bool> (Index < getNumDebugOperands() && "getDebugOperand() out of range!") ? void (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 513, __extension__ __PRETTY_FUNCTION__)); |
514 | return *(debug_operands().begin() + Index); |
515 | } |
516 | |
517 | SmallSet<Register, 4> getUsedDebugRegs() const { |
518 | assert(isDebugValue() && "not a DBG_VALUE*")(static_cast <bool> (isDebugValue() && "not a DBG_VALUE*" ) ? void (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE*\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 518, __extension__ __PRETTY_FUNCTION__)); |
519 | SmallSet<Register, 4> UsedRegs; |
520 | for (const auto &MO : debug_operands()) |
521 | if (MO.isReg() && MO.getReg()) |
522 | UsedRegs.insert(MO.getReg()); |
523 | return UsedRegs; |
524 | } |
525 | |
526 | /// Returns whether this debug value has at least one debug operand with the |
527 | /// register \p Reg. |
528 | bool hasDebugOperandForReg(Register Reg) const { |
529 | return any_of(debug_operands(), [Reg](const MachineOperand &Op) { |
530 | return Op.isReg() && Op.getReg() == Reg; |
531 | }); |
532 | } |
533 | |
534 | /// Returns a range of all of the operands that correspond to a debug use of |
535 | /// \p Reg. |
536 | template <typename Operand, typename Instruction> |
537 | static iterator_range< |
538 | filter_iterator<Operand *, std::function<bool(Operand &Op)>>> |
539 | getDebugOperandsForReg(Instruction *MI, Register Reg) { |
540 | std::function<bool(Operand & Op)> OpUsesReg( |
541 | [Reg](Operand &Op) { return Op.isReg() && Op.getReg() == Reg; }); |
542 | return make_filter_range(MI->debug_operands(), OpUsesReg); |
543 | } |
544 | iterator_range<filter_iterator<const MachineOperand *, |
545 | std::function<bool(const MachineOperand &Op)>>> |
546 | getDebugOperandsForReg(Register Reg) const { |
547 | return MachineInstr::getDebugOperandsForReg<const MachineOperand, |
548 | const MachineInstr>(this, Reg); |
549 | } |
550 | iterator_range<filter_iterator<MachineOperand *, |
551 | std::function<bool(MachineOperand &Op)>>> |
552 | getDebugOperandsForReg(Register Reg) { |
553 | return MachineInstr::getDebugOperandsForReg<MachineOperand, MachineInstr>( |
554 | this, Reg); |
555 | } |
556 | |
557 | bool isDebugOperand(const MachineOperand *Op) const { |
558 | return Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()); |
559 | } |
560 | |
561 | unsigned getDebugOperandIndex(const MachineOperand *Op) const { |
562 | assert(isDebugOperand(Op) && "Expected a debug operand.")(static_cast <bool> (isDebugOperand(Op) && "Expected a debug operand." ) ? void (0) : __assert_fail ("isDebugOperand(Op) && \"Expected a debug operand.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 562, __extension__ __PRETTY_FUNCTION__)); |
563 | return std::distance(adl_begin(debug_operands()), Op); |
564 | } |
565 | |
566 | /// Returns the total number of definitions. |
567 | unsigned getNumDefs() const { |
568 | return getNumExplicitDefs() + MCID->getNumImplicitDefs(); |
569 | } |
570 | |
571 | /// Returns true if the instruction has implicit definition. |
572 | bool hasImplicitDef() const { |
573 | for (unsigned I = getNumExplicitOperands(), E = getNumOperands(); |
574 | I != E; ++I) { |
575 | const MachineOperand &MO = getOperand(I); |
576 | if (MO.isDef() && MO.isImplicit()) |
577 | return true; |
578 | } |
579 | return false; |
580 | } |
581 | |
582 | /// Returns the implicit operands number. |
583 | unsigned getNumImplicitOperands() const { |
584 | return getNumOperands() - getNumExplicitOperands(); |
585 | } |
586 | |
587 | /// Return true if operand \p OpIdx is a subregister index. |
588 | bool isOperandSubregIdx(unsigned OpIdx) const { |
589 | assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate &&(static_cast <bool> (getOperand(OpIdx).getType() == MachineOperand ::MO_Immediate && "Expected MO_Immediate operand type." ) ? void (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 590, __extension__ __PRETTY_FUNCTION__)) |
590 | "Expected MO_Immediate operand type.")(static_cast <bool> (getOperand(OpIdx).getType() == MachineOperand ::MO_Immediate && "Expected MO_Immediate operand type." ) ? void (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 590, __extension__ __PRETTY_FUNCTION__)); |
591 | if (isExtractSubreg() && OpIdx == 2) |
592 | return true; |
593 | if (isInsertSubreg() && OpIdx == 3) |
594 | return true; |
595 | if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0) |
596 | return true; |
597 | if (isSubregToReg() && OpIdx == 3) |
598 | return true; |
599 | return false; |
600 | } |
601 | |
602 | /// Returns the number of non-implicit operands. |
603 | unsigned getNumExplicitOperands() const; |
604 | |
605 | /// Returns the number of non-implicit definitions. |
606 | unsigned getNumExplicitDefs() const; |
607 | |
608 | /// iterator/begin/end - Iterate over all operands of a machine instruction. |
609 | using mop_iterator = MachineOperand *; |
610 | using const_mop_iterator = const MachineOperand *; |
611 | |
612 | mop_iterator operands_begin() { return Operands; } |
613 | mop_iterator operands_end() { return Operands + NumOperands; } |
614 | |
615 | const_mop_iterator operands_begin() const { return Operands; } |
616 | const_mop_iterator operands_end() const { return Operands + NumOperands; } |
617 | |
618 | iterator_range<mop_iterator> operands() { |
619 | return make_range(operands_begin(), operands_end()); |
620 | } |
621 | iterator_range<const_mop_iterator> operands() const { |
622 | return make_range(operands_begin(), operands_end()); |
623 | } |
624 | iterator_range<mop_iterator> explicit_operands() { |
625 | return make_range(operands_begin(), |
626 | operands_begin() + getNumExplicitOperands()); |
627 | } |
628 | iterator_range<const_mop_iterator> explicit_operands() const { |
629 | return make_range(operands_begin(), |
630 | operands_begin() + getNumExplicitOperands()); |
631 | } |
632 | iterator_range<mop_iterator> implicit_operands() { |
633 | return make_range(explicit_operands().end(), operands_end()); |
634 | } |
635 | iterator_range<const_mop_iterator> implicit_operands() const { |
636 | return make_range(explicit_operands().end(), operands_end()); |
637 | } |
638 | /// Returns a range over all operands that are used to determine the variable |
639 | /// location for this DBG_VALUE instruction. |
640 | iterator_range<mop_iterator> debug_operands() { |
641 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 641, __extension__ __PRETTY_FUNCTION__)); |
642 | return isDebugValueList() |
643 | ? make_range(operands_begin() + 2, operands_end()) |
644 | : make_range(operands_begin(), operands_begin() + 1); |
645 | } |
646 | /// \copydoc debug_operands() |
647 | iterator_range<const_mop_iterator> debug_operands() const { |
648 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 648, __extension__ __PRETTY_FUNCTION__)); |
649 | return isDebugValueList() |
650 | ? make_range(operands_begin() + 2, operands_end()) |
651 | : make_range(operands_begin(), operands_begin() + 1); |
652 | } |
653 | /// Returns a range over all explicit operands that are register definitions. |
654 | /// Implicit definition are not included! |
655 | iterator_range<mop_iterator> defs() { |
656 | return make_range(operands_begin(), |
657 | operands_begin() + getNumExplicitDefs()); |
658 | } |
659 | /// \copydoc defs() |
660 | iterator_range<const_mop_iterator> defs() const { |
661 | return make_range(operands_begin(), |
662 | operands_begin() + getNumExplicitDefs()); |
663 | } |
664 | /// Returns a range that includes all operands that are register uses. |
665 | /// This may include unrelated operands which are not register uses. |
666 | iterator_range<mop_iterator> uses() { |
667 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
668 | } |
669 | /// \copydoc uses() |
670 | iterator_range<const_mop_iterator> uses() const { |
671 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
672 | } |
673 | iterator_range<mop_iterator> explicit_uses() { |
674 | return make_range(operands_begin() + getNumExplicitDefs(), |
675 | operands_begin() + getNumExplicitOperands()); |
676 | } |
677 | iterator_range<const_mop_iterator> explicit_uses() const { |
678 | return make_range(operands_begin() + getNumExplicitDefs(), |
679 | operands_begin() + getNumExplicitOperands()); |
680 | } |
681 | |
682 | /// Returns the number of the operand iterator \p I points to. |
683 | unsigned getOperandNo(const_mop_iterator I) const { |
684 | return I - operands_begin(); |
685 | } |
686 | |
687 | /// Access to memory operands of the instruction. If there are none, that does |
688 | /// not imply anything about whether the function accesses memory. Instead, |
689 | /// the caller must behave conservatively. |
690 | ArrayRef<MachineMemOperand *> memoperands() const { |
691 | if (!Info) |
692 | return {}; |
693 | |
694 | if (Info.is<EIIK_MMO>()) |
695 | return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); |
696 | |
697 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
698 | return EI->getMMOs(); |
699 | |
700 | return {}; |
701 | } |
702 | |
703 | /// Access to memory operands of the instruction. |
704 | /// |
705 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
706 | /// anything about whether the function accesses memory. Instead, the caller |
707 | /// must behave conservatively. |
708 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } |
709 | |
710 | /// Access to memory operands of the instruction. |
711 | /// |
712 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
713 | /// anything about whether the function accesses memory. Instead, the caller |
714 | /// must behave conservatively. |
715 | mmo_iterator memoperands_end() const { return memoperands().end(); } |
716 | |
717 | /// Return true if we don't have any memory operands which described the |
718 | /// memory access done by this instruction. If this is true, calling code |
719 | /// must be conservative. |
720 | bool memoperands_empty() const { return memoperands().empty(); } |
721 | |
722 | /// Return true if this instruction has exactly one MachineMemOperand. |
723 | bool hasOneMemOperand() const { return memoperands().size() == 1; } |
724 | |
725 | /// Return the number of memory operands. |
726 | unsigned getNumMemOperands() const { return memoperands().size(); } |
727 | |
728 | /// Helper to extract a pre-instruction symbol if one has been added. |
729 | MCSymbol *getPreInstrSymbol() const { |
730 | if (!Info) |
731 | return nullptr; |
732 | if (MCSymbol *S = Info.get<EIIK_PreInstrSymbol>()) |
733 | return S; |
734 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
735 | return EI->getPreInstrSymbol(); |
736 | |
737 | return nullptr; |
738 | } |
739 | |
740 | /// Helper to extract a post-instruction symbol if one has been added. |
741 | MCSymbol *getPostInstrSymbol() const { |
742 | if (!Info) |
743 | return nullptr; |
744 | if (MCSymbol *S = Info.get<EIIK_PostInstrSymbol>()) |
745 | return S; |
746 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
747 | return EI->getPostInstrSymbol(); |
748 | |
749 | return nullptr; |
750 | } |
751 | |
752 | /// Helper to extract a heap alloc marker if one has been added. |
753 | MDNode *getHeapAllocMarker() const { |
754 | if (!Info) |
755 | return nullptr; |
756 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
757 | return EI->getHeapAllocMarker(); |
758 | |
759 | return nullptr; |
760 | } |
761 | |
762 | /// API for querying MachineInstr properties. They are the same as MCInstrDesc |
763 | /// queries but they are bundle aware. |
764 | |
765 | enum QueryType { |
766 | IgnoreBundle, // Ignore bundles |
767 | AnyInBundle, // Return true if any instruction in bundle has property |
768 | AllInBundle // Return true if all instructions in bundle have property |
769 | }; |
770 | |
771 | /// Return true if the instruction (or in the case of a bundle, |
772 | /// the instructions inside the bundle) has the specified property. |
773 | /// The first argument is the property being queried. |
774 | /// The second argument indicates whether the query should look inside |
775 | /// instruction bundles. |
776 | bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { |
777 | assert(MCFlag < 64 &&(static_cast <bool> (MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? void (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 778, __extension__ __PRETTY_FUNCTION__)) |
778 | "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.")(static_cast <bool> (MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? void (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 778, __extension__ __PRETTY_FUNCTION__)); |
779 | // Inline the fast path for unbundled or bundle-internal instructions. |
780 | if (Type == IgnoreBundle || !isBundled() || isBundledWithPred()) |
781 | return getDesc().getFlags() & (1ULL << MCFlag); |
782 | |
783 | // If this is the first instruction in a bundle, take the slow path. |
784 | return hasPropertyInBundle(1ULL << MCFlag, Type); |
785 | } |
786 | |
787 | /// Return true if this is an instruction that should go through the usual |
788 | /// legalization steps. |
789 | bool isPreISelOpcode(QueryType Type = IgnoreBundle) const { |
790 | return hasProperty(MCID::PreISelOpcode, Type); |
791 | } |
792 | |
793 | /// Return true if this instruction can have a variable number of operands. |
794 | /// In this case, the variable operands will be after the normal |
795 | /// operands but before the implicit definitions and uses (if any are |
796 | /// present). |
797 | bool isVariadic(QueryType Type = IgnoreBundle) const { |
798 | return hasProperty(MCID::Variadic, Type); |
799 | } |
800 | |
801 | /// Set if this instruction has an optional definition, e.g. |
802 | /// ARM instructions which can set condition code if 's' bit is set. |
803 | bool hasOptionalDef(QueryType Type = IgnoreBundle) const { |
804 | return hasProperty(MCID::HasOptionalDef, Type); |
805 | } |
806 | |
807 | /// Return true if this is a pseudo instruction that doesn't |
808 | /// correspond to a real machine instruction. |
809 | bool isPseudo(QueryType Type = IgnoreBundle) const { |
810 | return hasProperty(MCID::Pseudo, Type); |
811 | } |
812 | |
813 | bool isReturn(QueryType Type = AnyInBundle) const { |
814 | return hasProperty(MCID::Return, Type); |
815 | } |
816 | |
817 | /// Return true if this is an instruction that marks the end of an EH scope, |
818 | /// i.e., a catchpad or a cleanuppad instruction. |
819 | bool isEHScopeReturn(QueryType Type = AnyInBundle) const { |
820 | return hasProperty(MCID::EHScopeReturn, Type); |
821 | } |
822 | |
823 | bool isCall(QueryType Type = AnyInBundle) const { |
824 | return hasProperty(MCID::Call, Type); |
825 | } |
826 | |
827 | /// Return true if this is a call instruction that may have an associated |
828 | /// call site entry in the debug info. |
829 | bool isCandidateForCallSiteEntry(QueryType Type = IgnoreBundle) const; |
830 | /// Return true if copying, moving, or erasing this instruction requires |
831 | /// updating Call Site Info (see \ref copyCallSiteInfo, \ref moveCallSiteInfo, |
832 | /// \ref eraseCallSiteInfo). |
833 | bool shouldUpdateCallSiteInfo() const; |
834 | |
835 | /// Returns true if the specified instruction stops control flow |
836 | /// from executing the instruction immediately following it. Examples include |
837 | /// unconditional branches and return instructions. |
838 | bool isBarrier(QueryType Type = AnyInBundle) const { |
839 | return hasProperty(MCID::Barrier, Type); |
840 | } |
841 | |
842 | /// Returns true if this instruction part of the terminator for a basic block. |
843 | /// Typically this is things like return and branch instructions. |
844 | /// |
845 | /// Various passes use this to insert code into the bottom of a basic block, |
846 | /// but before control flow occurs. |
847 | bool isTerminator(QueryType Type = AnyInBundle) const { |
848 | return hasProperty(MCID::Terminator, Type); |
849 | } |
850 | |
851 | /// Returns true if this is a conditional, unconditional, or indirect branch. |
852 | /// Predicates below can be used to discriminate between |
853 | /// these cases, and the TargetInstrInfo::analyzeBranch method can be used to |
854 | /// get more information. |
855 | bool isBranch(QueryType Type = AnyInBundle) const { |
856 | return hasProperty(MCID::Branch, Type); |
857 | } |
858 | |
859 | /// Return true if this is an indirect branch, such as a |
860 | /// branch through a register. |
861 | bool isIndirectBranch(QueryType Type = AnyInBundle) const { |
862 | return hasProperty(MCID::IndirectBranch, Type); |
863 | } |
864 | |
865 | /// Return true if this is a branch which may fall |
866 | /// through to the next instruction or may transfer control flow to some other |
867 | /// block. The TargetInstrInfo::analyzeBranch method can be used to get more |
868 | /// information about this branch. |
869 | bool isConditionalBranch(QueryType Type = AnyInBundle) const { |
870 | return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type); |
871 | } |
872 | |
873 | /// Return true if this is a branch which always |
874 | /// transfers control flow to some other block. The |
875 | /// TargetInstrInfo::analyzeBranch method can be used to get more information |
876 | /// about this branch. |
877 | bool isUnconditionalBranch(QueryType Type = AnyInBundle) const { |
878 | return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type); |
879 | } |
880 | |
881 | /// Return true if this instruction has a predicate operand that |
882 | /// controls execution. It may be set to 'always', or may be set to other |
883 | /// values. There are various methods in TargetInstrInfo that can be used to |
884 | /// control and modify the predicate in this instruction. |
885 | bool isPredicable(QueryType Type = AllInBundle) const { |
886 | // If it's a bundle than all bundled instructions must be predicable for this |
887 | // to return true. |
888 | return hasProperty(MCID::Predicable, Type); |
889 | } |
890 | |
891 | /// Return true if this instruction is a comparison. |
892 | bool isCompare(QueryType Type = IgnoreBundle) const { |
893 | return hasProperty(MCID::Compare, Type); |
894 | } |
895 | |
896 | /// Return true if this instruction is a move immediate |
897 | /// (including conditional moves) instruction. |
898 | bool isMoveImmediate(QueryType Type = IgnoreBundle) const { |
899 | return hasProperty(MCID::MoveImm, Type); |
900 | } |
901 | |
902 | /// Return true if this instruction is a register move. |
903 | /// (including moving values from subreg to reg) |
904 | bool isMoveReg(QueryType Type = IgnoreBundle) const { |
905 | return hasProperty(MCID::MoveReg, Type); |
906 | } |
907 | |
908 | /// Return true if this instruction is a bitcast instruction. |
909 | bool isBitcast(QueryType Type = IgnoreBundle) const { |
910 | return hasProperty(MCID::Bitcast, Type); |
911 | } |
912 | |
913 | /// Return true if this instruction is a select instruction. |
914 | bool isSelect(QueryType Type = IgnoreBundle) const { |
915 | return hasProperty(MCID::Select, Type); |
916 | } |
917 | |
918 | /// Return true if this instruction cannot be safely duplicated. |
919 | /// For example, if the instruction has a unique labels attached |
920 | /// to it, duplicating it would cause multiple definition errors. |
921 | bool isNotDuplicable(QueryType Type = AnyInBundle) const { |
922 | return hasProperty(MCID::NotDuplicable, Type); |
923 | } |
924 | |
925 | /// Return true if this instruction is convergent. |
926 | /// Convergent instructions can not be made control-dependent on any |
927 | /// additional values. |
928 | bool isConvergent(QueryType Type = AnyInBundle) const { |
929 | if (isInlineAsm()) { |
930 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
931 | if (ExtraInfo & InlineAsm::Extra_IsConvergent) |
932 | return true; |
933 | } |
934 | return hasProperty(MCID::Convergent, Type); |
935 | } |
936 | |
937 | /// Returns true if the specified instruction has a delay slot |
938 | /// which must be filled by the code generator. |
939 | bool hasDelaySlot(QueryType Type = AnyInBundle) const { |
940 | return hasProperty(MCID::DelaySlot, Type); |
941 | } |
942 | |
943 | /// Return true for instructions that can be folded as |
944 | /// memory operands in other instructions. The most common use for this |
945 | /// is instructions that are simple loads from memory that don't modify |
946 | /// the loaded value in any way, but it can also be used for instructions |
947 | /// that can be expressed as constant-pool loads, such as V_SETALLONES |
948 | /// on x86, to allow them to be folded when it is beneficial. |
949 | /// This should only be set on instructions that return a value in their |
950 | /// only virtual register definition. |
951 | bool canFoldAsLoad(QueryType Type = IgnoreBundle) const { |
952 | return hasProperty(MCID::FoldableAsLoad, Type); |
953 | } |
954 | |
955 | /// Return true if this instruction behaves |
956 | /// the same way as the generic REG_SEQUENCE instructions. |
957 | /// E.g., on ARM, |
958 | /// dX VMOVDRR rY, rZ |
959 | /// is equivalent to |
960 | /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. |
961 | /// |
962 | /// Note that for the optimizers to be able to take advantage of |
963 | /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be |
964 | /// override accordingly. |
965 | bool isRegSequenceLike(QueryType Type = IgnoreBundle) const { |
966 | return hasProperty(MCID::RegSequence, Type); |
967 | } |
968 | |
969 | /// Return true if this instruction behaves |
970 | /// the same way as the generic EXTRACT_SUBREG instructions. |
971 | /// E.g., on ARM, |
972 | /// rX, rY VMOVRRD dZ |
973 | /// is equivalent to two EXTRACT_SUBREG: |
974 | /// rX = EXTRACT_SUBREG dZ, ssub_0 |
975 | /// rY = EXTRACT_SUBREG dZ, ssub_1 |
976 | /// |
977 | /// Note that for the optimizers to be able to take advantage of |
978 | /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be |
979 | /// override accordingly. |
980 | bool isExtractSubregLike(QueryType Type = IgnoreBundle) const { |
981 | return hasProperty(MCID::ExtractSubreg, Type); |
982 | } |
983 | |
984 | /// Return true if this instruction behaves |
985 | /// the same way as the generic INSERT_SUBREG instructions. |
986 | /// E.g., on ARM, |
987 | /// dX = VSETLNi32 dY, rZ, Imm |
988 | /// is equivalent to a INSERT_SUBREG: |
989 | /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) |
990 | /// |
991 | /// Note that for the optimizers to be able to take advantage of |
992 | /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be |
993 | /// override accordingly. |
994 | bool isInsertSubregLike(QueryType Type = IgnoreBundle) const { |
995 | return hasProperty(MCID::InsertSubreg, Type); |
996 | } |
997 | |
998 | //===--------------------------------------------------------------------===// |
999 | // Side Effect Analysis |
1000 | //===--------------------------------------------------------------------===// |
1001 | |
1002 | /// Return true if this instruction could possibly read memory. |
1003 | /// Instructions with this flag set are not necessarily simple load |
1004 | /// instructions, they may load a value and modify it, for example. |
1005 | bool mayLoad(QueryType Type = AnyInBundle) const { |
1006 | if (isInlineAsm()) { |
1007 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
1008 | if (ExtraInfo & InlineAsm::Extra_MayLoad) |
1009 | return true; |
1010 | } |
1011 | return hasProperty(MCID::MayLoad, Type); |
1012 | } |
1013 | |
1014 | /// Return true if this instruction could possibly modify memory. |
1015 | /// Instructions with this flag set are not necessarily simple store |
1016 | /// instructions, they may store a modified value based on their operands, or |
1017 | /// may not actually modify anything, for example. |
1018 | bool mayStore(QueryType Type = AnyInBundle) const { |
1019 | if (isInlineAsm()) { |
1020 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
1021 | if (ExtraInfo & InlineAsm::Extra_MayStore) |
1022 | return true; |
1023 | } |
1024 | return hasProperty(MCID::MayStore, Type); |
1025 | } |
1026 | |
1027 | /// Return true if this instruction could possibly read or modify memory. |
1028 | bool mayLoadOrStore(QueryType Type = AnyInBundle) const { |
1029 | return mayLoad(Type) || mayStore(Type); |
1030 | } |
1031 | |
1032 | /// Return true if this instruction could possibly raise a floating-point |
1033 | /// exception. This is the case if the instruction is a floating-point |
1034 | /// instruction that can in principle raise an exception, as indicated |
1035 | /// by the MCID::MayRaiseFPException property, *and* at the same time, |
1036 | /// the instruction is used in a context where we expect floating-point |
1037 | /// exceptions are not disabled, as indicated by the NoFPExcept MI flag. |
1038 | bool mayRaiseFPException() const { |
1039 | return hasProperty(MCID::MayRaiseFPException) && |
1040 | !getFlag(MachineInstr::MIFlag::NoFPExcept); |
1041 | } |
1042 | |
1043 | //===--------------------------------------------------------------------===// |
1044 | // Flags that indicate whether an instruction can be modified by a method. |
1045 | //===--------------------------------------------------------------------===// |
1046 | |
1047 | /// Return true if this may be a 2- or 3-address |
1048 | /// instruction (of the form "X = op Y, Z, ..."), which produces the same |
1049 | /// result if Y and Z are exchanged. If this flag is set, then the |
1050 | /// TargetInstrInfo::commuteInstruction method may be used to hack on the |
1051 | /// instruction. |
1052 | /// |
1053 | /// Note that this flag may be set on instructions that are only commutable |
1054 | /// sometimes. In these cases, the call to commuteInstruction will fail. |
1055 | /// Also note that some instructions require non-trivial modification to |
1056 | /// commute them. |
1057 | bool isCommutable(QueryType Type = IgnoreBundle) const { |
1058 | return hasProperty(MCID::Commutable, Type); |
1059 | } |
1060 | |
1061 | /// Return true if this is a 2-address instruction |
1062 | /// which can be changed into a 3-address instruction if needed. Doing this |
1063 | /// transformation can be profitable in the register allocator, because it |
1064 | /// means that the instruction can use a 2-address form if possible, but |
1065 | /// degrade into a less efficient form if the source and dest register cannot |
1066 | /// be assigned to the same register. For example, this allows the x86 |
1067 | /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which |
1068 | /// is the same speed as the shift but has bigger code size. |
1069 | /// |
1070 | /// If this returns true, then the target must implement the |
1071 | /// TargetInstrInfo::convertToThreeAddress method for this instruction, which |
1072 | /// is allowed to fail if the transformation isn't valid for this specific |
1073 | /// instruction (e.g. shl reg, 4 on x86). |
1074 | /// |
1075 | bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const { |
1076 | return hasProperty(MCID::ConvertibleTo3Addr, Type); |
1077 | } |
1078 | |
1079 | /// Return true if this instruction requires |
1080 | /// custom insertion support when the DAG scheduler is inserting it into a |
1081 | /// machine basic block. If this is true for the instruction, it basically |
1082 | /// means that it is a pseudo instruction used at SelectionDAG time that is |
1083 | /// expanded out into magic code by the target when MachineInstrs are formed. |
1084 | /// |
1085 | /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method |
1086 | /// is used to insert this into the MachineBasicBlock. |
1087 | bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const { |
1088 | return hasProperty(MCID::UsesCustomInserter, Type); |
1089 | } |
1090 | |
1091 | /// Return true if this instruction requires *adjustment* |
1092 | /// after instruction selection by calling a target hook. For example, this |
1093 | /// can be used to fill in ARM 's' optional operand depending on whether |
1094 | /// the conditional flag register is used. |
1095 | bool hasPostISelHook(QueryType Type = IgnoreBundle) const { |
1096 | return hasProperty(MCID::HasPostISelHook, Type); |
1097 | } |
1098 | |
1099 | /// Returns true if this instruction is a candidate for remat. |
1100 | /// This flag is deprecated, please don't use it anymore. If this |
1101 | /// flag is set, the isReallyTriviallyReMaterializable() method is called to |
1102 | /// verify the instruction is really rematable. |
1103 | bool isRematerializable(QueryType Type = AllInBundle) const { |
1104 | // It's only possible to re-mat a bundle if all bundled instructions are |
1105 | // re-materializable. |
1106 | return hasProperty(MCID::Rematerializable, Type); |
1107 | } |
1108 | |
1109 | /// Returns true if this instruction has the same cost (or less) than a move |
1110 | /// instruction. This is useful during certain types of optimizations |
1111 | /// (e.g., remat during two-address conversion or machine licm) |
1112 | /// where we would like to remat or hoist the instruction, but not if it costs |
1113 | /// more than moving the instruction into the appropriate register. Note, we |
1114 | /// are not marking copies from and to the same register class with this flag. |
1115 | bool isAsCheapAsAMove(QueryType Type = AllInBundle) const { |
1116 | // Only returns true for a bundle if all bundled instructions are cheap. |
1117 | return hasProperty(MCID::CheapAsAMove, Type); |
1118 | } |
1119 | |
1120 | /// Returns true if this instruction source operands |
1121 | /// have special register allocation requirements that are not captured by the |
1122 | /// operand register classes. e.g. ARM::STRD's two source registers must be an |
1123 | /// even / odd pair, ARM::STM registers have to be in ascending order. |
1124 | /// Post-register allocation passes should not attempt to change allocations |
1125 | /// for sources of instructions with this flag. |
1126 | bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const { |
1127 | return hasProperty(MCID::ExtraSrcRegAllocReq, Type); |
1128 | } |
1129 | |
1130 | /// Returns true if this instruction def operands |
1131 | /// have special register allocation requirements that are not captured by the |
1132 | /// operand register classes. e.g. ARM::LDRD's two def registers must be an |
1133 | /// even / odd pair, ARM::LDM registers have to be in ascending order. |
1134 | /// Post-register allocation passes should not attempt to change allocations |
1135 | /// for definitions of instructions with this flag. |
1136 | bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const { |
1137 | return hasProperty(MCID::ExtraDefRegAllocReq, Type); |
1138 | } |
1139 | |
1140 | enum MICheckType { |
1141 | CheckDefs, // Check all operands for equality |
1142 | CheckKillDead, // Check all operands including kill / dead markers |
1143 | IgnoreDefs, // Ignore all definitions |
1144 | IgnoreVRegDefs // Ignore virtual register definitions |
1145 | }; |
1146 | |
1147 | /// Return true if this instruction is identical to \p Other. |
1148 | /// Two instructions are identical if they have the same opcode and all their |
1149 | /// operands are identical (with respect to MachineOperand::isIdenticalTo()). |
1150 | /// Note that this means liveness related flags (dead, undef, kill) do not |
1151 | /// affect the notion of identical. |
1152 | bool isIdenticalTo(const MachineInstr &Other, |
1153 | MICheckType Check = CheckDefs) const; |
1154 | |
1155 | /// Unlink 'this' from the containing basic block, and return it without |
1156 | /// deleting it. |
1157 | /// |
1158 | /// This function can not be used on bundled instructions, use |
1159 | /// removeFromBundle() to remove individual instructions from a bundle. |
1160 | MachineInstr *removeFromParent(); |
1161 | |
1162 | /// Unlink this instruction from its basic block and return it without |
1163 | /// deleting it. |
1164 | /// |
1165 | /// If the instruction is part of a bundle, the other instructions in the |
1166 | /// bundle remain bundled. |
1167 | MachineInstr *removeFromBundle(); |
1168 | |
1169 | /// Unlink 'this' from the containing basic block and delete it. |
1170 | /// |
1171 | /// If this instruction is the header of a bundle, the whole bundle is erased. |
1172 | /// This function can not be used for instructions inside a bundle, use |
1173 | /// eraseFromBundle() to erase individual bundled instructions. |
1174 | void eraseFromParent(); |
1175 | |
1176 | /// Unlink 'this' from the containing basic block and delete it. |
1177 | /// |
1178 | /// For all definitions mark their uses in DBG_VALUE nodes |
1179 | /// as undefined. Otherwise like eraseFromParent(). |
1180 | void eraseFromParentAndMarkDBGValuesForRemoval(); |
1181 | |
1182 | /// Unlink 'this' form its basic block and delete it. |
1183 | /// |
1184 | /// If the instruction is part of a bundle, the other instructions in the |
1185 | /// bundle remain bundled. |
1186 | void eraseFromBundle(); |
1187 | |
1188 | bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } |
1189 | bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } |
1190 | bool isAnnotationLabel() const { |
1191 | return getOpcode() == TargetOpcode::ANNOTATION_LABEL; |
1192 | } |
1193 | |
1194 | /// Returns true if the MachineInstr represents a label. |
1195 | bool isLabel() const { |
1196 | return isEHLabel() || isGCLabel() || isAnnotationLabel(); |
1197 | } |
1198 | |
1199 | bool isCFIInstruction() const { |
1200 | return getOpcode() == TargetOpcode::CFI_INSTRUCTION; |
1201 | } |
1202 | |
1203 | bool isPseudoProbe() const { |
1204 | return getOpcode() == TargetOpcode::PSEUDO_PROBE; |
1205 | } |
1206 | |
1207 | // True if the instruction represents a position in the function. |
1208 | bool isPosition() const { return isLabel() || isCFIInstruction(); } |
1209 | |
1210 | bool isNonListDebugValue() const { |
1211 | return getOpcode() == TargetOpcode::DBG_VALUE; |
1212 | } |
1213 | bool isDebugValueList() const { |
1214 | return getOpcode() == TargetOpcode::DBG_VALUE_LIST; |
1215 | } |
1216 | bool isDebugValue() const { |
1217 | return isNonListDebugValue() || isDebugValueList(); |
1218 | } |
1219 | bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; } |
1220 | bool isDebugRef() const { return getOpcode() == TargetOpcode::DBG_INSTR_REF; } |
1221 | bool isDebugPHI() const { return getOpcode() == TargetOpcode::DBG_PHI; } |
1222 | bool isDebugInstr() const { |
1223 | return isDebugValue() || isDebugLabel() || isDebugRef() || isDebugPHI(); |
1224 | } |
1225 | bool isDebugOrPseudoInstr() const { |
1226 | return isDebugInstr() || isPseudoProbe(); |
1227 | } |
1228 | |
1229 | bool isDebugOffsetImm() const { |
1230 | return isNonListDebugValue() && getDebugOffset().isImm(); |
1231 | } |
1232 | |
1233 | /// A DBG_VALUE is indirect iff the location operand is a register and |
1234 | /// the offset operand is an immediate. |
1235 | bool isIndirectDebugValue() const { |
1236 | return isDebugOffsetImm() && getDebugOperand(0).isReg(); |
1237 | } |
1238 | |
1239 | /// A DBG_VALUE is an entry value iff its debug expression contains the |
1240 | /// DW_OP_LLVM_entry_value operation. |
1241 | bool isDebugEntryValue() const; |
1242 | |
1243 | /// Return true if the instruction is a debug value which describes a part of |
1244 | /// a variable as unavailable. |
1245 | bool isUndefDebugValue() const { |
1246 | if (!isDebugValue()) |
1247 | return false; |
1248 | // If any $noreg locations are given, this DV is undef. |
1249 | for (const MachineOperand &Op : debug_operands()) |
1250 | if (Op.isReg() && !Op.getReg().isValid()) |
1251 | return true; |
1252 | return false; |
1253 | } |
1254 | |
1255 | bool isPHI() const { |
1256 | return getOpcode() == TargetOpcode::PHI || |
1257 | getOpcode() == TargetOpcode::G_PHI; |
1258 | } |
1259 | bool isKill() const { return getOpcode() == TargetOpcode::KILL; } |
1260 | bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } |
1261 | bool isInlineAsm() const { |
1262 | return getOpcode() == TargetOpcode::INLINEASM || |
1263 | getOpcode() == TargetOpcode::INLINEASM_BR; |
1264 | } |
1265 | |
1266 | /// FIXME: Seems like a layering violation that the AsmDialect, which is X86 |
1267 | /// specific, be attached to a generic MachineInstr. |
1268 | bool isMSInlineAsm() const { |
1269 | return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; |
1270 | } |
1271 | |
1272 | bool isStackAligningInlineAsm() const; |
1273 | InlineAsm::AsmDialect getInlineAsmDialect() const; |
1274 | |
1275 | bool isInsertSubreg() const { |
1276 | return getOpcode() == TargetOpcode::INSERT_SUBREG; |
1277 | } |
1278 | |
1279 | bool isSubregToReg() const { |
1280 | return getOpcode() == TargetOpcode::SUBREG_TO_REG; |
1281 | } |
1282 | |
1283 | bool isRegSequence() const { |
1284 | return getOpcode() == TargetOpcode::REG_SEQUENCE; |
1285 | } |
1286 | |
1287 | bool isBundle() const { |
1288 | return getOpcode() == TargetOpcode::BUNDLE; |
1289 | } |
1290 | |
1291 | bool isCopy() const { |
1292 | return getOpcode() == TargetOpcode::COPY; |
1293 | } |
1294 | |
1295 | bool isFullCopy() const { |
1296 | return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg(); |
1297 | } |
1298 | |
1299 | bool isExtractSubreg() const { |
1300 | return getOpcode() == TargetOpcode::EXTRACT_SUBREG; |
1301 | } |
1302 | |
1303 | /// Return true if the instruction behaves like a copy. |
1304 | /// This does not include native copy instructions. |
1305 | bool isCopyLike() const { |
1306 | return isCopy() || isSubregToReg(); |
1307 | } |
1308 | |
1309 | /// Return true is the instruction is an identity copy. |
1310 | bool isIdentityCopy() const { |
1311 | return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() && |
1312 | getOperand(0).getSubReg() == getOperand(1).getSubReg(); |
1313 | } |
1314 | |
1315 | /// Return true if this instruction doesn't produce any output in the form of |
1316 | /// executable instructions. |
1317 | bool isMetaInstruction() const { |
1318 | switch (getOpcode()) { |
1319 | default: |
1320 | return false; |
1321 | case TargetOpcode::IMPLICIT_DEF: |
1322 | case TargetOpcode::KILL: |
1323 | case TargetOpcode::CFI_INSTRUCTION: |
1324 | case TargetOpcode::EH_LABEL: |
1325 | case TargetOpcode::GC_LABEL: |
1326 | case TargetOpcode::DBG_VALUE: |
1327 | case TargetOpcode::DBG_VALUE_LIST: |
1328 | case TargetOpcode::DBG_INSTR_REF: |
1329 | case TargetOpcode::DBG_PHI: |
1330 | case TargetOpcode::DBG_LABEL: |
1331 | case TargetOpcode::LIFETIME_START: |
1332 | case TargetOpcode::LIFETIME_END: |
1333 | case TargetOpcode::PSEUDO_PROBE: |
1334 | case TargetOpcode::ARITH_FENCE: |
1335 | return true; |
1336 | } |
1337 | } |
1338 | |
1339 | /// Return true if this is a transient instruction that is either very likely |
1340 | /// to be eliminated during register allocation (such as copy-like |
1341 | /// instructions), or if this instruction doesn't have an execution-time cost. |
1342 | bool isTransient() const { |
1343 | switch (getOpcode()) { |
1344 | default: |
1345 | return isMetaInstruction(); |
1346 | // Copy-like instructions are usually eliminated during register allocation. |
1347 | case TargetOpcode::PHI: |
1348 | case TargetOpcode::G_PHI: |
1349 | case TargetOpcode::COPY: |
1350 | case TargetOpcode::INSERT_SUBREG: |
1351 | case TargetOpcode::SUBREG_TO_REG: |
1352 | case TargetOpcode::REG_SEQUENCE: |
1353 | return true; |
1354 | } |
1355 | } |
1356 | |
1357 | /// Return the number of instructions inside the MI bundle, excluding the |
1358 | /// bundle header. |
1359 | /// |
1360 | /// This is the number of instructions that MachineBasicBlock::iterator |
1361 | /// skips, 0 for unbundled instructions. |
1362 | unsigned getBundleSize() const; |
1363 | |
1364 | /// Return true if the MachineInstr reads the specified register. |
1365 | /// If TargetRegisterInfo is passed, then it also checks if there |
1366 | /// is a read of a super-register. |
1367 | /// This does not count partial redefines of virtual registers as reads: |
1368 | /// %reg1024:6 = OP. |
1369 | bool readsRegister(Register Reg, |
1370 | const TargetRegisterInfo *TRI = nullptr) const { |
1371 | return findRegisterUseOperandIdx(Reg, false, TRI) != -1; |
1372 | } |
1373 | |
1374 | /// Return true if the MachineInstr reads the specified virtual register. |
1375 | /// Take into account that a partial define is a |
1376 | /// read-modify-write operation. |
1377 | bool readsVirtualRegister(Register Reg) const { |
1378 | return readsWritesVirtualRegister(Reg).first; |
1379 | } |
1380 | |
1381 | /// Return a pair of bools (reads, writes) indicating if this instruction |
1382 | /// reads or writes Reg. This also considers partial defines. |
1383 | /// If Ops is not null, all operand indices for Reg are added. |
1384 | std::pair<bool,bool> readsWritesVirtualRegister(Register Reg, |
1385 | SmallVectorImpl<unsigned> *Ops = nullptr) const; |
1386 | |
1387 | /// Return true if the MachineInstr kills the specified register. |
1388 | /// If TargetRegisterInfo is passed, then it also checks if there is |
1389 | /// a kill of a super-register. |
1390 | bool killsRegister(Register Reg, |
1391 | const TargetRegisterInfo *TRI = nullptr) const { |
1392 | return findRegisterUseOperandIdx(Reg, true, TRI) != -1; |
1393 | } |
1394 | |
1395 | /// Return true if the MachineInstr fully defines the specified register. |
1396 | /// If TargetRegisterInfo is passed, then it also checks |
1397 | /// if there is a def of a super-register. |
1398 | /// NOTE: It's ignoring subreg indices on virtual registers. |
1399 | bool definesRegister(Register Reg, |
1400 | const TargetRegisterInfo *TRI = nullptr) const { |
1401 | return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; |
1402 | } |
1403 | |
1404 | /// Return true if the MachineInstr modifies (fully define or partially |
1405 | /// define) the specified register. |
1406 | /// NOTE: It's ignoring subreg indices on virtual registers. |
1407 | bool modifiesRegister(Register Reg, |
1408 | const TargetRegisterInfo *TRI = nullptr) const { |
1409 | return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; |
1410 | } |
1411 | |
1412 | /// Returns true if the register is dead in this machine instruction. |
1413 | /// If TargetRegisterInfo is passed, then it also checks |
1414 | /// if there is a dead def of a super-register. |
1415 | bool registerDefIsDead(Register Reg, |
1416 | const TargetRegisterInfo *TRI = nullptr) const { |
1417 | return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; |
1418 | } |
1419 | |
1420 | /// Returns true if the MachineInstr has an implicit-use operand of exactly |
1421 | /// the given register (not considering sub/super-registers). |
1422 | bool hasRegisterImplicitUseOperand(Register Reg) const; |
1423 | |
1424 | /// Returns the operand index that is a use of the specific register or -1 |
1425 | /// if it is not found. It further tightens the search criteria to a use |
1426 | /// that kills the register if isKill is true. |
1427 | int findRegisterUseOperandIdx(Register Reg, bool isKill = false, |
1428 | const TargetRegisterInfo *TRI = nullptr) const; |
1429 | |
1430 | /// Wrapper for findRegisterUseOperandIdx, it returns |
1431 | /// a pointer to the MachineOperand rather than an index. |
1432 | MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false, |
1433 | const TargetRegisterInfo *TRI = nullptr) { |
1434 | int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI); |
1435 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
1436 | } |
1437 | |
1438 | const MachineOperand *findRegisterUseOperand( |
1439 | Register Reg, bool isKill = false, |
1440 | const TargetRegisterInfo *TRI = nullptr) const { |
1441 | return const_cast<MachineInstr *>(this)-> |
1442 | findRegisterUseOperand(Reg, isKill, TRI); |
1443 | } |
1444 | |
1445 | /// Returns the operand index that is a def of the specified register or |
1446 | /// -1 if it is not found. If isDead is true, defs that are not dead are |
1447 | /// skipped. If Overlap is true, then it also looks for defs that merely |
1448 | /// overlap the specified register. If TargetRegisterInfo is non-null, |
1449 | /// then it also checks if there is a def of a super-register. |
1450 | /// This may also return a register mask operand when Overlap is true. |
1451 | int findRegisterDefOperandIdx(Register Reg, |
1452 | bool isDead = false, bool Overlap = false, |
1453 | const TargetRegisterInfo *TRI = nullptr) const; |
1454 | |
1455 | /// Wrapper for findRegisterDefOperandIdx, it returns |
1456 | /// a pointer to the MachineOperand rather than an index. |
1457 | MachineOperand * |
1458 | findRegisterDefOperand(Register Reg, bool isDead = false, |
1459 | bool Overlap = false, |
1460 | const TargetRegisterInfo *TRI = nullptr) { |
1461 | int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); |
1462 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
1463 | } |
1464 | |
1465 | const MachineOperand * |
1466 | findRegisterDefOperand(Register Reg, bool isDead = false, |
1467 | bool Overlap = false, |
1468 | const TargetRegisterInfo *TRI = nullptr) const { |
1469 | return const_cast<MachineInstr *>(this)->findRegisterDefOperand( |
1470 | Reg, isDead, Overlap, TRI); |
1471 | } |
1472 | |
1473 | /// Find the index of the first operand in the |
1474 | /// operand list that is used to represent the predicate. It returns -1 if |
1475 | /// none is found. |
1476 | int findFirstPredOperandIdx() const; |
1477 | |
1478 | /// Find the index of the flag word operand that |
1479 | /// corresponds to operand OpIdx on an inline asm instruction. Returns -1 if |
1480 | /// getOperand(OpIdx) does not belong to an inline asm operand group. |
1481 | /// |
1482 | /// If GroupNo is not NULL, it will receive the number of the operand group |
1483 | /// containing OpIdx. |
1484 | int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const; |
1485 | |
1486 | /// Compute the static register class constraint for operand OpIdx. |
1487 | /// For normal instructions, this is derived from the MCInstrDesc. |
1488 | /// For inline assembly it is derived from the flag words. |
1489 | /// |
1490 | /// Returns NULL if the static register class constraint cannot be |
1491 | /// determined. |
1492 | const TargetRegisterClass* |
1493 | getRegClassConstraint(unsigned OpIdx, |
1494 | const TargetInstrInfo *TII, |
1495 | const TargetRegisterInfo *TRI) const; |
1496 | |
1497 | /// Applies the constraints (def/use) implied by this MI on \p Reg to |
1498 | /// the given \p CurRC. |
1499 | /// If \p ExploreBundle is set and MI is part of a bundle, all the |
1500 | /// instructions inside the bundle will be taken into account. In other words, |
1501 | /// this method accumulates all the constraints of the operand of this MI and |
1502 | /// the related bundle if MI is a bundle or inside a bundle. |
1503 | /// |
1504 | /// Returns the register class that satisfies both \p CurRC and the |
1505 | /// constraints set by MI. Returns NULL if such a register class does not |
1506 | /// exist. |
1507 | /// |
1508 | /// \pre CurRC must not be NULL. |
1509 | const TargetRegisterClass *getRegClassConstraintEffectForVReg( |
1510 | Register Reg, const TargetRegisterClass *CurRC, |
1511 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, |
1512 | bool ExploreBundle = false) const; |
1513 | |
1514 | /// Applies the constraints (def/use) implied by the \p OpIdx operand |
1515 | /// to the given \p CurRC. |
1516 | /// |
1517 | /// Returns the register class that satisfies both \p CurRC and the |
1518 | /// constraints set by \p OpIdx MI. Returns NULL if such a register class |
1519 | /// does not exist. |
1520 | /// |
1521 | /// \pre CurRC must not be NULL. |
1522 | /// \pre The operand at \p OpIdx must be a register. |
1523 | const TargetRegisterClass * |
1524 | getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC, |
1525 | const TargetInstrInfo *TII, |
1526 | const TargetRegisterInfo *TRI) const; |
1527 | |
1528 | /// Add a tie between the register operands at DefIdx and UseIdx. |
1529 | /// The tie will cause the register allocator to ensure that the two |
1530 | /// operands are assigned the same physical register. |
1531 | /// |
1532 | /// Tied operands are managed automatically for explicit operands in the |
1533 | /// MCInstrDesc. This method is for exceptional cases like inline asm. |
1534 | void tieOperands(unsigned DefIdx, unsigned UseIdx); |
1535 | |
1536 | /// Given the index of a tied register operand, find the |
1537 | /// operand it is tied to. Defs are tied to uses and vice versa. Returns the |
1538 | /// index of the tied operand which must exist. |
1539 | unsigned findTiedOperandIdx(unsigned OpIdx) const; |
1540 | |
1541 | /// Given the index of a register def operand, |
1542 | /// check if the register def is tied to a source operand, due to either |
1543 | /// two-address elimination or inline assembly constraints. Returns the |
1544 | /// first tied use operand index by reference if UseOpIdx is not null. |
1545 | bool isRegTiedToUseOperand(unsigned DefOpIdx, |
1546 | unsigned *UseOpIdx = nullptr) const { |
1547 | const MachineOperand &MO = getOperand(DefOpIdx); |
1548 | if (!MO.isReg() || !MO.isDef() || !MO.isTied()) |
1549 | return false; |
1550 | if (UseOpIdx) |
1551 | *UseOpIdx = findTiedOperandIdx(DefOpIdx); |
1552 | return true; |
1553 | } |
1554 | |
1555 | /// Return true if the use operand of the specified index is tied to a def |
1556 | /// operand. It also returns the def operand index by reference if DefOpIdx |
1557 | /// is not null. |
1558 | bool isRegTiedToDefOperand(unsigned UseOpIdx, |
1559 | unsigned *DefOpIdx = nullptr) const { |
1560 | const MachineOperand &MO = getOperand(UseOpIdx); |
1561 | if (!MO.isReg() || !MO.isUse() || !MO.isTied()) |
1562 | return false; |
1563 | if (DefOpIdx) |
1564 | *DefOpIdx = findTiedOperandIdx(UseOpIdx); |
1565 | return true; |
1566 | } |
1567 | |
1568 | /// Clears kill flags on all operands. |
1569 | void clearKillInfo(); |
1570 | |
1571 | /// Replace all occurrences of FromReg with ToReg:SubIdx, |
1572 | /// properly composing subreg indices where necessary. |
1573 | void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, |
1574 | const TargetRegisterInfo &RegInfo); |
1575 | |
1576 | /// We have determined MI kills a register. Look for the |
1577 | /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, |
1578 | /// add a implicit operand if it's not found. Returns true if the operand |
1579 | /// exists / is added. |
1580 | bool addRegisterKilled(Register IncomingReg, |
1581 | const TargetRegisterInfo *RegInfo, |
1582 | bool AddIfNotFound = false); |
1583 | |
1584 | /// Clear all kill flags affecting Reg. If RegInfo is provided, this includes |
1585 | /// all aliasing registers. |
1586 | void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo); |
1587 | |
1588 | /// We have determined MI defined a register without a use. |
1589 | /// Look for the operand that defines it and mark it as IsDead. If |
1590 | /// AddIfNotFound is true, add a implicit operand if it's not found. Returns |
1591 | /// true if the operand exists / is added. |
1592 | bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, |
1593 | bool AddIfNotFound = false); |
1594 | |
1595 | /// Clear all dead flags on operands defining register @p Reg. |
1596 | void clearRegisterDeads(Register Reg); |
1597 | |
1598 | /// Mark all subregister defs of register @p Reg with the undef flag. |
1599 | /// This function is used when we determined to have a subregister def in an |
1600 | /// otherwise undefined super register. |
1601 | void setRegisterDefReadUndef(Register Reg, bool IsUndef = true); |
1602 | |
1603 | /// We have determined MI defines a register. Make sure there is an operand |
1604 | /// defining Reg. |
1605 | void addRegisterDefined(Register Reg, |
1606 | const TargetRegisterInfo *RegInfo = nullptr); |
1607 | |
1608 | /// Mark every physreg used by this instruction as |
1609 | /// dead except those in the UsedRegs list. |
1610 | /// |
1611 | /// On instructions with register mask operands, also add implicit-def |
1612 | /// operands for all registers in UsedRegs. |
1613 | void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs, |
1614 | const TargetRegisterInfo &TRI); |
1615 | |
1616 | /// Return true if it is safe to move this instruction. If |
1617 | /// SawStore is set to true, it means that there is a store (or call) between |
1618 | /// the instruction's location and its intended destination. |
1619 | bool isSafeToMove(AAResults *AA, bool &SawStore) const; |
1620 | |
1621 | /// Returns true if this instruction's memory access aliases the memory |
1622 | /// access of Other. |
1623 | // |
1624 | /// Assumes any physical registers used to compute addresses |
1625 | /// have the same value for both instructions. Returns false if neither |
1626 | /// instruction writes to memory. |
1627 | /// |
1628 | /// @param AA Optional alias analysis, used to compare memory operands. |
1629 | /// @param Other MachineInstr to check aliasing against. |
1630 | /// @param UseTBAA Whether to pass TBAA information to alias analysis. |
1631 | bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; |
1632 | |
1633 | /// Return true if this instruction may have an ordered |
1634 | /// or volatile memory reference, or if the information describing the memory |
1635 | /// reference is not available. Return false if it is known to have no |
1636 | /// ordered or volatile memory references. |
1637 | bool hasOrderedMemoryRef() const; |
1638 | |
1639 | /// Return true if this load instruction never traps and points to a memory |
1640 | /// location whose value doesn't change during the execution of this function. |
1641 | /// |
1642 | /// Examples include loading a value from the constant pool or from the |
1643 | /// argument area of a function (if it does not change). If the instruction |
1644 | /// does multiple loads, this returns true only if all of the loads are |
1645 | /// dereferenceable and invariant. |
1646 | bool isDereferenceableInvariantLoad(AAResults *AA) const; |
1647 | |
1648 | /// If the specified instruction is a PHI that always merges together the |
1649 | /// same virtual register, return the register, otherwise return 0. |
1650 | unsigned isConstantValuePHI() const; |
1651 | |
1652 | /// Return true if this instruction has side effects that are not modeled |
1653 | /// by mayLoad / mayStore, etc. |
1654 | /// For all instructions, the property is encoded in MCInstrDesc::Flags |
1655 | /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is |
1656 | /// INLINEASM instruction, in which case the side effect property is encoded |
1657 | /// in one of its operands (see InlineAsm::Extra_HasSideEffect). |
1658 | /// |
1659 | bool hasUnmodeledSideEffects() const; |
1660 | |
1661 | /// Returns true if it is illegal to fold a load across this instruction. |
1662 | bool isLoadFoldBarrier() const; |
1663 | |
1664 | /// Return true if all the defs of this instruction are dead. |
1665 | bool allDefsAreDead() const; |
1666 | |
1667 | /// Return a valid size if the instruction is a spill instruction. |
1668 | Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const; |
1669 | |
1670 | /// Return a valid size if the instruction is a folded spill instruction. |
1671 | Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const; |
1672 | |
1673 | /// Return a valid size if the instruction is a restore instruction. |
1674 | Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const; |
1675 | |
1676 | /// Return a valid size if the instruction is a folded restore instruction. |
1677 | Optional<unsigned> |
1678 | getFoldedRestoreSize(const TargetInstrInfo *TII) const; |
1679 | |
1680 | /// Copy implicit register operands from specified |
1681 | /// instruction to this instruction. |
1682 | void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI); |
1683 | |
1684 | /// Debugging support |
1685 | /// @{ |
1686 | /// Determine the generic type to be printed (if needed) on uses and defs. |
1687 | LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes, |
1688 | const MachineRegisterInfo &MRI) const; |
1689 | |
1690 | /// Return true when an instruction has tied register that can't be determined |
1691 | /// by the instruction's descriptor. This is useful for MIR printing, to |
1692 | /// determine whether we need to print the ties or not. |
1693 | bool hasComplexRegisterTies() const; |
1694 | |
1695 | /// Print this MI to \p OS. |
1696 | /// Don't print information that can be inferred from other instructions if |
1697 | /// \p IsStandalone is false. It is usually true when only a fragment of the |
1698 | /// function is printed. |
1699 | /// Only print the defs and the opcode if \p SkipOpers is true. |
1700 | /// Otherwise, also print operands if \p SkipDebugLoc is true. |
1701 | /// Otherwise, also print the debug loc, with a terminating newline. |
1702 | /// \p TII is used to print the opcode name. If it's not present, but the |
1703 | /// MI is in a function, the opcode will be printed using the function's TII. |
1704 | void print(raw_ostream &OS, bool IsStandalone = true, bool SkipOpers = false, |
1705 | bool SkipDebugLoc = false, bool AddNewLine = true, |
1706 | const TargetInstrInfo *TII = nullptr) const; |
1707 | void print(raw_ostream &OS, ModuleSlotTracker &MST, bool IsStandalone = true, |
1708 | bool SkipOpers = false, bool SkipDebugLoc = false, |
1709 | bool AddNewLine = true, |
1710 | const TargetInstrInfo *TII = nullptr) const; |
1711 | void dump() const; |
1712 | /// Print on dbgs() the current instruction and the instructions defining its |
1713 | /// operands and so on until we reach \p MaxDepth. |
1714 | void dumpr(const MachineRegisterInfo &MRI, |
1715 | unsigned MaxDepth = UINT_MAX(2147483647 *2U +1U)) const; |
1716 | /// @} |
1717 | |
1718 | //===--------------------------------------------------------------------===// |
1719 | // Accessors used to build up machine instructions. |
1720 | |
1721 | /// Add the specified operand to the instruction. If it is an implicit |
1722 | /// operand, it is added to the end of the operand list. If it is an |
1723 | /// explicit operand it is added at the end of the explicit operand list |
1724 | /// (before the first implicit operand). |
1725 | /// |
1726 | /// MF must be the machine function that was used to allocate this |
1727 | /// instruction. |
1728 | /// |
1729 | /// MachineInstrBuilder provides a more convenient interface for creating |
1730 | /// instructions and adding operands. |
1731 | void addOperand(MachineFunction &MF, const MachineOperand &Op); |
1732 | |
1733 | /// Add an operand without providing an MF reference. This only works for |
1734 | /// instructions that are inserted in a basic block. |
1735 | /// |
1736 | /// MachineInstrBuilder and the two-argument addOperand(MF, MO) should be |
1737 | /// preferred. |
1738 | void addOperand(const MachineOperand &Op); |
1739 | |
1740 | /// Replace the instruction descriptor (thus opcode) of |
1741 | /// the current instruction with a new one. |
1742 | void setDesc(const MCInstrDesc &tid) { MCID = &tid; } |
1743 | |
1744 | /// Replace current source information with new such. |
1745 | /// Avoid using this, the constructor argument is preferable. |
1746 | void setDebugLoc(DebugLoc dl) { |
1747 | debugLoc = std::move(dl); |
1748 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 1748, __extension__ __PRETTY_FUNCTION__)); |
1749 | } |
1750 | |
1751 | /// Erase an operand from an instruction, leaving it with one |
1752 | /// fewer operand than it started with. |
1753 | void RemoveOperand(unsigned OpNo); |
1754 | |
1755 | /// Clear this MachineInstr's memory reference descriptor list. This resets |
1756 | /// the memrefs to their most conservative state. This should be used only |
1757 | /// as a last resort since it greatly pessimizes our knowledge of the memory |
1758 | /// access performed by the instruction. |
1759 | void dropMemRefs(MachineFunction &MF); |
1760 | |
1761 | /// Assign this MachineInstr's memory reference descriptor list. |
1762 | /// |
1763 | /// Unlike other methods, this *will* allocate them into a new array |
1764 | /// associated with the provided `MachineFunction`. |
1765 | void setMemRefs(MachineFunction &MF, ArrayRef<MachineMemOperand *> MemRefs); |
1766 | |
1767 | /// Add a MachineMemOperand to the machine instruction. |
1768 | /// This function should be used only occasionally. The setMemRefs function |
1769 | /// is the primary method for setting up a MachineInstr's MemRefs list. |
1770 | void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); |
1771 | |
1772 | /// Clone another MachineInstr's memory reference descriptor list and replace |
1773 | /// ours with it. |
1774 | /// |
1775 | /// Note that `*this` may be the incoming MI! |
1776 | /// |
1777 | /// Prefer this API whenever possible as it can avoid allocations in common |
1778 | /// cases. |
1779 | void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); |
1780 | |
1781 | /// Clone the merge of multiple MachineInstrs' memory reference descriptors |
1782 | /// list and replace ours with it. |
1783 | /// |
1784 | /// Note that `*this` may be one of the incoming MIs! |
1785 | /// |
1786 | /// Prefer this API whenever possible as it can avoid allocations in common |
1787 | /// cases. |
1788 | void cloneMergedMemRefs(MachineFunction &MF, |
1789 | ArrayRef<const MachineInstr *> MIs); |
1790 | |
1791 | /// Set a symbol that will be emitted just prior to the instruction itself. |
1792 | /// |
1793 | /// Setting this to a null pointer will remove any such symbol. |
1794 | /// |
1795 | /// FIXME: This is not fully implemented yet. |
1796 | void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
1797 | |
1798 | /// Set a symbol that will be emitted just after the instruction itself. |
1799 | /// |
1800 | /// Setting this to a null pointer will remove any such symbol. |
1801 | /// |
1802 | /// FIXME: This is not fully implemented yet. |
1803 | void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
1804 | |
1805 | /// Clone another MachineInstr's pre- and post- instruction symbols and |
1806 | /// replace ours with it. |
1807 | void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); |
1808 | |
1809 | /// Set a marker on instructions that denotes where we should create and emit |
1810 | /// heap alloc site labels. This waits until after instruction selection and |
1811 | /// optimizations to create the label, so it should still work if the |
1812 | /// instruction is removed or duplicated. |
1813 | void setHeapAllocMarker(MachineFunction &MF, MDNode *MD); |
1814 | |
1815 | /// Return the MIFlags which represent both MachineInstrs. This |
1816 | /// should be used when merging two MachineInstrs into one. This routine does |
1817 | /// not modify the MIFlags of this MachineInstr. |
1818 | uint16_t mergeFlagsWith(const MachineInstr& Other) const; |
1819 | |
1820 | static uint16_t copyFlagsFromInstruction(const Instruction &I); |
1821 | |
1822 | /// Copy all flags to MachineInst MIFlags |
1823 | void copyIRFlags(const Instruction &I); |
1824 | |
1825 | /// Break any tie involving OpIdx. |
1826 | void untieRegOperand(unsigned OpIdx) { |
1827 | MachineOperand &MO = getOperand(OpIdx); |
1828 | if (MO.isReg() && MO.isTied()) { |
1829 | getOperand(findTiedOperandIdx(OpIdx)).TiedTo = 0; |
1830 | MO.TiedTo = 0; |
1831 | } |
1832 | } |
1833 | |
1834 | /// Add all implicit def and use operands to this instruction. |
1835 | void addImplicitDefUseOperands(MachineFunction &MF); |
1836 | |
1837 | /// Scan instructions immediately following MI and collect any matching |
1838 | /// DBG_VALUEs. |
1839 | void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); |
1840 | |
1841 | /// Find all DBG_VALUEs that point to the register def in this instruction |
1842 | /// and point them to \p Reg instead. |
1843 | void changeDebugValuesDefReg(Register Reg); |
1844 | |
1845 | /// Returns the Intrinsic::ID for this instruction. |
1846 | /// \pre Must have an intrinsic ID operand. |
1847 | unsigned getIntrinsicID() const { |
1848 | return getOperand(getNumExplicitDefs()).getIntrinsicID(); |
1849 | } |
1850 | |
1851 | /// Sets all register debug operands in this debug value instruction to be |
1852 | /// undef. |
1853 | void setDebugValueUndef() { |
1854 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/MachineInstr.h" , 1854, __extension__ __PRETTY_FUNCTION__)); |
1855 | for (MachineOperand &MO : debug_operands()) { |
1856 | if (MO.isReg()) { |
1857 | MO.setReg(0); |
1858 | MO.setSubReg(0); |
1859 | } |
1860 | } |
1861 | } |
1862 | |
1863 | private: |
1864 | /// If this instruction is embedded into a MachineFunction, return the |
1865 | /// MachineRegisterInfo object for the current function, otherwise |
1866 | /// return null. |
1867 | MachineRegisterInfo *getRegInfo(); |
1868 | |
1869 | /// Unlink all of the register operands in this instruction from their |
1870 | /// respective use lists. This requires that the operands already be on their |
1871 | /// use lists. |
1872 | void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); |
1873 | |
1874 | /// Add all of the register operands in this instruction from their |
1875 | /// respective use lists. This requires that the operands not be on their |
1876 | /// use lists yet. |
1877 | void AddRegOperandsToUseLists(MachineRegisterInfo&); |
1878 | |
1879 | /// Slow path for hasProperty when we're dealing with a bundle. |
1880 | bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; |
1881 | |
1882 | /// Implements the logic of getRegClassConstraintEffectForVReg for the |
1883 | /// this MI and the given operand index \p OpIdx. |
1884 | /// If the related operand does not constrained Reg, this returns CurRC. |
1885 | const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( |
1886 | unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, |
1887 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; |
1888 | |
1889 | /// Stores extra instruction information inline or allocates as ExtraInfo |
1890 | /// based on the number of pointers. |
1891 | void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs, |
1892 | MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, |
1893 | MDNode *HeapAllocMarker); |
1894 | }; |
1895 | |
1896 | /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the |
1897 | /// instruction rather than by pointer value. |
1898 | /// The hashing and equality testing functions ignore definitions so this is |
1899 | /// useful for CSE, etc. |
1900 | struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> { |
1901 | static inline MachineInstr *getEmptyKey() { |
1902 | return nullptr; |
1903 | } |
1904 | |
1905 | static inline MachineInstr *getTombstoneKey() { |
1906 | return reinterpret_cast<MachineInstr*>(-1); |
1907 | } |
1908 | |
1909 | static unsigned getHashValue(const MachineInstr* const &MI); |
1910 | |
1911 | static bool isEqual(const MachineInstr* const &LHS, |
1912 | const MachineInstr* const &RHS) { |
1913 | if (RHS == getEmptyKey() || RHS == getTombstoneKey() || |
1914 | LHS == getEmptyKey() || LHS == getTombstoneKey()) |
1915 | return LHS == RHS; |
1916 | return LHS->isIdenticalTo(*RHS, MachineInstr::IgnoreVRegDefs); |
1917 | } |
1918 | }; |
1919 | |
1920 | //===----------------------------------------------------------------------===// |
1921 | // Debugging Support |
1922 | |
1923 | inline raw_ostream& operator<<(raw_ostream &OS, const MachineInstr &MI) { |
1924 | MI.print(OS); |
1925 | return OS; |
1926 | } |
1927 | |
1928 | } // end namespace llvm |
1929 | |
1930 | #endif // LLVM_CODEGEN_MACHINEINSTR_H |
1 | //===-- llvm/CodeGen/Register.h ---------------------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #ifndef LLVM_CODEGEN_REGISTER_H |
10 | #define LLVM_CODEGEN_REGISTER_H |
11 | |
12 | #include "llvm/MC/MCRegister.h" |
13 | #include <cassert> |
14 | |
15 | namespace llvm { |
16 | |
17 | /// Wrapper class representing virtual and physical registers. Should be passed |
18 | /// by value. |
19 | class Register { |
20 | unsigned Reg; |
21 | |
22 | public: |
23 | constexpr Register(unsigned Val = 0): Reg(Val) {} |
24 | constexpr Register(MCRegister Val): Reg(Val) {} |
25 | |
26 | // Register numbers can represent physical registers, virtual registers, and |
27 | // sometimes stack slots. The unsigned values are divided into these ranges: |
28 | // |
29 | // 0 Not a register, can be used as a sentinel. |
30 | // [1;2^30) Physical registers assigned by TableGen. |
31 | // [2^30;2^31) Stack slots. (Rarely used.) |
32 | // [2^31;2^32) Virtual registers assigned by MachineRegisterInfo. |
33 | // |
34 | // Further sentinels can be allocated from the small negative integers. |
35 | // DenseMapInfo<unsigned> uses -1u and -2u. |
36 | static_assert(std::numeric_limits<decltype(Reg)>::max() >= 0xFFFFFFFF, |
37 | "Reg isn't large enough to hold full range."); |
38 | |
39 | /// isStackSlot - Sometimes it is useful the be able to store a non-negative |
40 | /// frame index in a variable that normally holds a register. isStackSlot() |
41 | /// returns true if Reg is in the range used for stack slots. |
42 | /// |
43 | /// FIXME: remove in favor of member. |
44 | static bool isStackSlot(unsigned Reg) { |
45 | return MCRegister::isStackSlot(Reg); |
46 | } |
47 | |
48 | /// Return true if this is a stack slot. |
49 | bool isStack() const { return MCRegister::isStackSlot(Reg); } |
50 | |
51 | /// Compute the frame index from a register value representing a stack slot. |
52 | static int stackSlot2Index(Register Reg) { |
53 | assert(Reg.isStack() && "Not a stack slot")(static_cast <bool> (Reg.isStack() && "Not a stack slot" ) ? void (0) : __assert_fail ("Reg.isStack() && \"Not a stack slot\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/Register.h" , 53, __extension__ __PRETTY_FUNCTION__)); |
54 | return int(Reg - MCRegister::FirstStackSlot); |
55 | } |
56 | |
57 | /// Convert a non-negative frame index to a stack slot register value. |
58 | static Register index2StackSlot(int FI) { |
59 | assert(FI >= 0 && "Cannot hold a negative frame index.")(static_cast <bool> (FI >= 0 && "Cannot hold a negative frame index." ) ? void (0) : __assert_fail ("FI >= 0 && \"Cannot hold a negative frame index.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/Register.h" , 59, __extension__ __PRETTY_FUNCTION__)); |
60 | return Register(FI + MCRegister::FirstStackSlot); |
61 | } |
62 | |
63 | /// Return true if the specified register number is in |
64 | /// the physical register namespace. |
65 | static bool isPhysicalRegister(unsigned Reg) { |
66 | return MCRegister::isPhysicalRegister(Reg); |
67 | } |
68 | |
69 | /// Return true if the specified register number is in |
70 | /// the virtual register namespace. |
71 | static bool isVirtualRegister(unsigned Reg) { |
72 | return Reg & MCRegister::VirtualRegFlag && !isStackSlot(Reg); |
73 | } |
74 | |
75 | /// Convert a virtual register number to a 0-based index. |
76 | /// The first virtual register in a function will get the index 0. |
77 | static unsigned virtReg2Index(Register Reg) { |
78 | assert(isVirtualRegister(Reg) && "Not a virtual register")(static_cast <bool> (isVirtualRegister(Reg) && "Not a virtual register" ) ? void (0) : __assert_fail ("isVirtualRegister(Reg) && \"Not a virtual register\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/Register.h" , 78, __extension__ __PRETTY_FUNCTION__)); |
79 | return Reg & ~MCRegister::VirtualRegFlag; |
80 | } |
81 | |
82 | /// Convert a 0-based index to a virtual register number. |
83 | /// This is the inverse operation of VirtReg2IndexFunctor below. |
84 | static Register index2VirtReg(unsigned Index) { |
85 | assert(Index < (1u << 31) && "Index too large for virtual register range.")(static_cast <bool> (Index < (1u << 31) && "Index too large for virtual register range.") ? void (0) : __assert_fail ("Index < (1u << 31) && \"Index too large for virtual register range.\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/Register.h" , 85, __extension__ __PRETTY_FUNCTION__)); |
86 | return Index | MCRegister::VirtualRegFlag; |
87 | } |
88 | |
89 | /// Return true if the specified register number is in the virtual register |
90 | /// namespace. |
91 | bool isVirtual() const { |
92 | return isVirtualRegister(Reg); |
93 | } |
94 | |
95 | /// Return true if the specified register number is in the physical register |
96 | /// namespace. |
97 | bool isPhysical() const { |
98 | return isPhysicalRegister(Reg); |
99 | } |
100 | |
101 | /// Convert a virtual register number to a 0-based index. The first virtual |
102 | /// register in a function will get the index 0. |
103 | unsigned virtRegIndex() const { |
104 | return virtReg2Index(Reg); |
105 | } |
106 | |
107 | constexpr operator unsigned() const { |
108 | return Reg; |
109 | } |
110 | |
111 | unsigned id() const { return Reg; } |
112 | |
113 | operator MCRegister() const { |
114 | return MCRegister(Reg); |
115 | } |
116 | |
117 | /// Utility to check-convert this value to a MCRegister. The caller is |
118 | /// expected to have already validated that this Register is, indeed, |
119 | /// physical. |
120 | MCRegister asMCReg() const { |
121 | assert(Reg == MCRegister::NoRegister ||(static_cast <bool> (Reg == MCRegister::NoRegister || MCRegister ::isPhysicalRegister(Reg)) ? void (0) : __assert_fail ("Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister(Reg)" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/Register.h" , 122, __extension__ __PRETTY_FUNCTION__)) |
122 | MCRegister::isPhysicalRegister(Reg))(static_cast <bool> (Reg == MCRegister::NoRegister || MCRegister ::isPhysicalRegister(Reg)) ? void (0) : __assert_fail ("Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister(Reg)" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/Register.h" , 122, __extension__ __PRETTY_FUNCTION__)); |
123 | return MCRegister(Reg); |
124 | } |
125 | |
126 | bool isValid() const { return Reg != MCRegister::NoRegister; } |
127 | |
128 | /// Comparisons between register objects |
129 | bool operator==(const Register &Other) const { return Reg == Other.Reg; } |
130 | bool operator!=(const Register &Other) const { return Reg != Other.Reg; } |
131 | bool operator==(const MCRegister &Other) const { return Reg == Other.id(); } |
132 | bool operator!=(const MCRegister &Other) const { return Reg != Other.id(); } |
133 | |
134 | /// Comparisons against register constants. E.g. |
135 | /// * R == AArch64::WZR |
136 | /// * R == 0 |
137 | /// * R == VirtRegMap::NO_PHYS_REG |
138 | bool operator==(unsigned Other) const { return Reg == Other; } |
139 | bool operator!=(unsigned Other) const { return Reg != Other; } |
140 | bool operator==(int Other) const { return Reg == unsigned(Other); } |
141 | bool operator!=(int Other) const { return Reg != unsigned(Other); } |
142 | // MSVC requires that we explicitly declare these two as well. |
143 | bool operator==(MCPhysReg Other) const { return Reg == unsigned(Other); } |
144 | bool operator!=(MCPhysReg Other) const { return Reg != unsigned(Other); } |
145 | }; |
146 | |
147 | // Provide DenseMapInfo for Register |
148 | template<> struct DenseMapInfo<Register> { |
149 | static inline unsigned getEmptyKey() { |
150 | return DenseMapInfo<unsigned>::getEmptyKey(); |
151 | } |
152 | static inline unsigned getTombstoneKey() { |
153 | return DenseMapInfo<unsigned>::getTombstoneKey(); |
154 | } |
155 | static unsigned getHashValue(const Register &Val) { |
156 | return DenseMapInfo<unsigned>::getHashValue(Val.id()); |
157 | } |
158 | static bool isEqual(const Register &LHS, const Register &RHS) { |
159 | return DenseMapInfo<unsigned>::isEqual(LHS.id(), RHS.id()); |
160 | } |
161 | }; |
162 | |
163 | } |
164 | |
165 | #endif // LLVM_CODEGEN_REGISTER_H |
1 | //===- llvm/CodeGen/TargetInstrInfo.h - Instruction Info --------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file describes the target machine instruction set to the code generator. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #ifndef LLVM_CODEGEN_TARGETINSTRINFO_H |
14 | #define LLVM_CODEGEN_TARGETINSTRINFO_H |
15 | |
16 | #include "llvm/ADT/ArrayRef.h" |
17 | #include "llvm/ADT/DenseMap.h" |
18 | #include "llvm/ADT/DenseMapInfo.h" |
19 | #include "llvm/ADT/None.h" |
20 | #include "llvm/CodeGen/MIRFormatter.h" |
21 | #include "llvm/CodeGen/MachineBasicBlock.h" |
22 | #include "llvm/CodeGen/MachineCombinerPattern.h" |
23 | #include "llvm/CodeGen/MachineFunction.h" |
24 | #include "llvm/CodeGen/MachineInstr.h" |
25 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
26 | #include "llvm/CodeGen/MachineOperand.h" |
27 | #include "llvm/CodeGen/MachineOutliner.h" |
28 | #include "llvm/CodeGen/RegisterClassInfo.h" |
29 | #include "llvm/CodeGen/VirtRegMap.h" |
30 | #include "llvm/MC/MCInstrInfo.h" |
31 | #include "llvm/Support/BranchProbability.h" |
32 | #include "llvm/Support/ErrorHandling.h" |
33 | #include <cassert> |
34 | #include <cstddef> |
35 | #include <cstdint> |
36 | #include <utility> |
37 | #include <vector> |
38 | |
39 | namespace llvm { |
40 | |
41 | class AAResults; |
42 | class DFAPacketizer; |
43 | class InstrItineraryData; |
44 | class LiveIntervals; |
45 | class LiveVariables; |
46 | class MachineLoop; |
47 | class MachineMemOperand; |
48 | class MachineRegisterInfo; |
49 | class MCAsmInfo; |
50 | class MCInst; |
51 | struct MCSchedModel; |
52 | class Module; |
53 | class ScheduleDAG; |
54 | class ScheduleDAGMI; |
55 | class ScheduleHazardRecognizer; |
56 | class SDNode; |
57 | class SelectionDAG; |
58 | class RegScavenger; |
59 | class TargetRegisterClass; |
60 | class TargetRegisterInfo; |
61 | class TargetSchedModel; |
62 | class TargetSubtargetInfo; |
63 | |
64 | template <class T> class SmallVectorImpl; |
65 | |
66 | using ParamLoadedValue = std::pair<MachineOperand, DIExpression*>; |
67 | |
68 | struct DestSourcePair { |
69 | const MachineOperand *Destination; |
70 | const MachineOperand *Source; |
71 | |
72 | DestSourcePair(const MachineOperand &Dest, const MachineOperand &Src) |
73 | : Destination(&Dest), Source(&Src) {} |
74 | }; |
75 | |
76 | /// Used to describe a register and immediate addition. |
77 | struct RegImmPair { |
78 | Register Reg; |
79 | int64_t Imm; |
80 | |
81 | RegImmPair(Register Reg, int64_t Imm) : Reg(Reg), Imm(Imm) {} |
82 | }; |
83 | |
84 | /// Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare. |
85 | /// It holds the register values, the scale value and the displacement. |
86 | struct ExtAddrMode { |
87 | Register BaseReg; |
88 | Register ScaledReg; |
89 | int64_t Scale; |
90 | int64_t Displacement; |
91 | }; |
92 | |
93 | //--------------------------------------------------------------------------- |
94 | /// |
95 | /// TargetInstrInfo - Interface to description of machine instruction set |
96 | /// |
97 | class TargetInstrInfo : public MCInstrInfo { |
98 | public: |
99 | TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u, |
100 | unsigned CatchRetOpcode = ~0u, unsigned ReturnOpcode = ~0u) |
101 | : CallFrameSetupOpcode(CFSetupOpcode), |
102 | CallFrameDestroyOpcode(CFDestroyOpcode), CatchRetOpcode(CatchRetOpcode), |
103 | ReturnOpcode(ReturnOpcode) {} |
104 | TargetInstrInfo(const TargetInstrInfo &) = delete; |
105 | TargetInstrInfo &operator=(const TargetInstrInfo &) = delete; |
106 | virtual ~TargetInstrInfo(); |
107 | |
108 | static bool isGenericOpcode(unsigned Opc) { |
109 | return Opc <= TargetOpcode::GENERIC_OP_END; |
110 | } |
111 | |
112 | /// Given a machine instruction descriptor, returns the register |
113 | /// class constraint for OpNum, or NULL. |
114 | virtual |
115 | const TargetRegisterClass *getRegClass(const MCInstrDesc &MCID, unsigned OpNum, |
116 | const TargetRegisterInfo *TRI, |
117 | const MachineFunction &MF) const; |
118 | |
119 | /// Return true if the instruction is trivially rematerializable, meaning it |
120 | /// has no side effects and requires no operands that aren't always available. |
121 | /// This means the only allowed uses are constants and unallocatable physical |
122 | /// registers so that the instructions result is independent of the place |
123 | /// in the function. |
124 | bool isTriviallyReMaterializable(const MachineInstr &MI, |
125 | AAResults *AA = nullptr) const { |
126 | return MI.getOpcode() == TargetOpcode::IMPLICIT_DEF || |
127 | (MI.getDesc().isRematerializable() && |
128 | (isReallyTriviallyReMaterializable(MI, AA) || |
129 | isReallyTriviallyReMaterializableGeneric(MI, AA))); |
130 | } |
131 | |
132 | /// Given \p MO is a PhysReg use return if it can be ignored for the purpose |
133 | /// of instruction rematerialization. |
134 | virtual bool isIgnorableUse(const MachineOperand &MO) const { |
135 | return false; |
136 | } |
137 | |
138 | protected: |
139 | /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is |
140 | /// set, this hook lets the target specify whether the instruction is actually |
141 | /// trivially rematerializable, taking into consideration its operands. This |
142 | /// predicate must return false if the instruction has any side effects other |
143 | /// than producing a value, or if it requres any address registers that are |
144 | /// not always available. |
145 | /// Requirements must be check as stated in isTriviallyReMaterializable() . |
146 | virtual bool isReallyTriviallyReMaterializable(const MachineInstr &MI, |
147 | AAResults *AA) const { |
148 | return false; |
149 | } |
150 | |
151 | /// This method commutes the operands of the given machine instruction MI. |
152 | /// The operands to be commuted are specified by their indices OpIdx1 and |
153 | /// OpIdx2. |
154 | /// |
155 | /// If a target has any instructions that are commutable but require |
156 | /// converting to different instructions or making non-trivial changes |
157 | /// to commute them, this method can be overloaded to do that. |
158 | /// The default implementation simply swaps the commutable operands. |
159 | /// |
160 | /// If NewMI is false, MI is modified in place and returned; otherwise, a |
161 | /// new machine instruction is created and returned. |
162 | /// |
163 | /// Do not call this method for a non-commutable instruction. |
164 | /// Even though the instruction is commutable, the method may still |
165 | /// fail to commute the operands, null pointer is returned in such cases. |
166 | virtual MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI, |
167 | unsigned OpIdx1, |
168 | unsigned OpIdx2) const; |
169 | |
170 | /// Assigns the (CommutableOpIdx1, CommutableOpIdx2) pair of commutable |
171 | /// operand indices to (ResultIdx1, ResultIdx2). |
172 | /// One or both input values of the pair: (ResultIdx1, ResultIdx2) may be |
173 | /// predefined to some indices or be undefined (designated by the special |
174 | /// value 'CommuteAnyOperandIndex'). |
175 | /// The predefined result indices cannot be re-defined. |
176 | /// The function returns true iff after the result pair redefinition |
177 | /// the fixed result pair is equal to or equivalent to the source pair of |
178 | /// indices: (CommutableOpIdx1, CommutableOpIdx2). It is assumed here that |
179 | /// the pairs (x,y) and (y,x) are equivalent. |
180 | static bool fixCommutedOpIndices(unsigned &ResultIdx1, unsigned &ResultIdx2, |
181 | unsigned CommutableOpIdx1, |
182 | unsigned CommutableOpIdx2); |
183 | |
184 | private: |
185 | /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is |
186 | /// set and the target hook isReallyTriviallyReMaterializable returns false, |
187 | /// this function does target-independent tests to determine if the |
188 | /// instruction is really trivially rematerializable. |
189 | bool isReallyTriviallyReMaterializableGeneric(const MachineInstr &MI, |
190 | AAResults *AA) const; |
191 | |
192 | public: |
193 | /// These methods return the opcode of the frame setup/destroy instructions |
194 | /// if they exist (-1 otherwise). Some targets use pseudo instructions in |
195 | /// order to abstract away the difference between operating with a frame |
196 | /// pointer and operating without, through the use of these two instructions. |
197 | /// |
198 | unsigned getCallFrameSetupOpcode() const { return CallFrameSetupOpcode; } |
199 | unsigned getCallFrameDestroyOpcode() const { return CallFrameDestroyOpcode; } |
200 | |
201 | /// Returns true if the argument is a frame pseudo instruction. |
202 | bool isFrameInstr(const MachineInstr &I) const { |
203 | return I.getOpcode() == getCallFrameSetupOpcode() || |
204 | I.getOpcode() == getCallFrameDestroyOpcode(); |
205 | } |
206 | |
207 | /// Returns true if the argument is a frame setup pseudo instruction. |
208 | bool isFrameSetup(const MachineInstr &I) const { |
209 | return I.getOpcode() == getCallFrameSetupOpcode(); |
210 | } |
211 | |
212 | /// Returns size of the frame associated with the given frame instruction. |
213 | /// For frame setup instruction this is frame that is set up space set up |
214 | /// after the instruction. For frame destroy instruction this is the frame |
215 | /// freed by the caller. |
216 | /// Note, in some cases a call frame (or a part of it) may be prepared prior |
217 | /// to the frame setup instruction. It occurs in the calls that involve |
218 | /// inalloca arguments. This function reports only the size of the frame part |
219 | /// that is set up between the frame setup and destroy pseudo instructions. |
220 | int64_t getFrameSize(const MachineInstr &I) const { |
221 | assert(isFrameInstr(I) && "Not a frame instruction")(static_cast <bool> (isFrameInstr(I) && "Not a frame instruction" ) ? void (0) : __assert_fail ("isFrameInstr(I) && \"Not a frame instruction\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 221, __extension__ __PRETTY_FUNCTION__)); |
222 | assert(I.getOperand(0).getImm() >= 0)(static_cast <bool> (I.getOperand(0).getImm() >= 0) ? void (0) : __assert_fail ("I.getOperand(0).getImm() >= 0" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 222, __extension__ __PRETTY_FUNCTION__)); |
223 | return I.getOperand(0).getImm(); |
224 | } |
225 | |
226 | /// Returns the total frame size, which is made up of the space set up inside |
227 | /// the pair of frame start-stop instructions and the space that is set up |
228 | /// prior to the pair. |
229 | int64_t getFrameTotalSize(const MachineInstr &I) const { |
230 | if (isFrameSetup(I)) { |
231 | assert(I.getOperand(1).getImm() >= 0 &&(static_cast <bool> (I.getOperand(1).getImm() >= 0 && "Frame size must not be negative") ? void (0) : __assert_fail ("I.getOperand(1).getImm() >= 0 && \"Frame size must not be negative\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 232, __extension__ __PRETTY_FUNCTION__)) |
232 | "Frame size must not be negative")(static_cast <bool> (I.getOperand(1).getImm() >= 0 && "Frame size must not be negative") ? void (0) : __assert_fail ("I.getOperand(1).getImm() >= 0 && \"Frame size must not be negative\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 232, __extension__ __PRETTY_FUNCTION__)); |
233 | return getFrameSize(I) + I.getOperand(1).getImm(); |
234 | } |
235 | return getFrameSize(I); |
236 | } |
237 | |
238 | unsigned getCatchReturnOpcode() const { return CatchRetOpcode; } |
239 | unsigned getReturnOpcode() const { return ReturnOpcode; } |
240 | |
241 | /// Returns the actual stack pointer adjustment made by an instruction |
242 | /// as part of a call sequence. By default, only call frame setup/destroy |
243 | /// instructions adjust the stack, but targets may want to override this |
244 | /// to enable more fine-grained adjustment, or adjust by a different value. |
245 | virtual int getSPAdjust(const MachineInstr &MI) const; |
246 | |
247 | /// Return true if the instruction is a "coalescable" extension instruction. |
248 | /// That is, it's like a copy where it's legal for the source to overlap the |
249 | /// destination. e.g. X86::MOVSX64rr32. If this returns true, then it's |
250 | /// expected the pre-extension value is available as a subreg of the result |
251 | /// register. This also returns the sub-register index in SubIdx. |
252 | virtual bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, |
253 | Register &DstReg, unsigned &SubIdx) const { |
254 | return false; |
255 | } |
256 | |
257 | /// If the specified machine instruction is a direct |
258 | /// load from a stack slot, return the virtual or physical register number of |
259 | /// the destination along with the FrameIndex of the loaded stack slot. If |
260 | /// not, return 0. This predicate must return 0 if the instruction has |
261 | /// any side effects other than loading from the stack slot. |
262 | virtual unsigned isLoadFromStackSlot(const MachineInstr &MI, |
263 | int &FrameIndex) const { |
264 | return 0; |
265 | } |
266 | |
267 | /// Optional extension of isLoadFromStackSlot that returns the number of |
268 | /// bytes loaded from the stack. This must be implemented if a backend |
269 | /// supports partial stack slot spills/loads to further disambiguate |
270 | /// what the load does. |
271 | virtual unsigned isLoadFromStackSlot(const MachineInstr &MI, |
272 | int &FrameIndex, |
273 | unsigned &MemBytes) const { |
274 | MemBytes = 0; |
275 | return isLoadFromStackSlot(MI, FrameIndex); |
276 | } |
277 | |
278 | /// Check for post-frame ptr elimination stack locations as well. |
279 | /// This uses a heuristic so it isn't reliable for correctness. |
280 | virtual unsigned isLoadFromStackSlotPostFE(const MachineInstr &MI, |
281 | int &FrameIndex) const { |
282 | return 0; |
283 | } |
284 | |
285 | /// If the specified machine instruction has a load from a stack slot, |
286 | /// return true along with the FrameIndices of the loaded stack slot and the |
287 | /// machine mem operands containing the reference. |
288 | /// If not, return false. Unlike isLoadFromStackSlot, this returns true for |
289 | /// any instructions that loads from the stack. This is just a hint, as some |
290 | /// cases may be missed. |
291 | virtual bool hasLoadFromStackSlot( |
292 | const MachineInstr &MI, |
293 | SmallVectorImpl<const MachineMemOperand *> &Accesses) const; |
294 | |
295 | /// If the specified machine instruction is a direct |
296 | /// store to a stack slot, return the virtual or physical register number of |
297 | /// the source reg along with the FrameIndex of the loaded stack slot. If |
298 | /// not, return 0. This predicate must return 0 if the instruction has |
299 | /// any side effects other than storing to the stack slot. |
300 | virtual unsigned isStoreToStackSlot(const MachineInstr &MI, |
301 | int &FrameIndex) const { |
302 | return 0; |
303 | } |
304 | |
305 | /// Optional extension of isStoreToStackSlot that returns the number of |
306 | /// bytes stored to the stack. This must be implemented if a backend |
307 | /// supports partial stack slot spills/loads to further disambiguate |
308 | /// what the store does. |
309 | virtual unsigned isStoreToStackSlot(const MachineInstr &MI, |
310 | int &FrameIndex, |
311 | unsigned &MemBytes) const { |
312 | MemBytes = 0; |
313 | return isStoreToStackSlot(MI, FrameIndex); |
314 | } |
315 | |
316 | /// Check for post-frame ptr elimination stack locations as well. |
317 | /// This uses a heuristic, so it isn't reliable for correctness. |
318 | virtual unsigned isStoreToStackSlotPostFE(const MachineInstr &MI, |
319 | int &FrameIndex) const { |
320 | return 0; |
321 | } |
322 | |
323 | /// If the specified machine instruction has a store to a stack slot, |
324 | /// return true along with the FrameIndices of the loaded stack slot and the |
325 | /// machine mem operands containing the reference. |
326 | /// If not, return false. Unlike isStoreToStackSlot, |
327 | /// this returns true for any instructions that stores to the |
328 | /// stack. This is just a hint, as some cases may be missed. |
329 | virtual bool hasStoreToStackSlot( |
330 | const MachineInstr &MI, |
331 | SmallVectorImpl<const MachineMemOperand *> &Accesses) const; |
332 | |
333 | /// Return true if the specified machine instruction |
334 | /// is a copy of one stack slot to another and has no other effect. |
335 | /// Provide the identity of the two frame indices. |
336 | virtual bool isStackSlotCopy(const MachineInstr &MI, int &DestFrameIndex, |
337 | int &SrcFrameIndex) const { |
338 | return false; |
339 | } |
340 | |
341 | /// Compute the size in bytes and offset within a stack slot of a spilled |
342 | /// register or subregister. |
343 | /// |
344 | /// \param [out] Size in bytes of the spilled value. |
345 | /// \param [out] Offset in bytes within the stack slot. |
346 | /// \returns true if both Size and Offset are successfully computed. |
347 | /// |
348 | /// Not all subregisters have computable spill slots. For example, |
349 | /// subregisters registers may not be byte-sized, and a pair of discontiguous |
350 | /// subregisters has no single offset. |
351 | /// |
352 | /// Targets with nontrivial bigendian implementations may need to override |
353 | /// this, particularly to support spilled vector registers. |
354 | virtual bool getStackSlotRange(const TargetRegisterClass *RC, unsigned SubIdx, |
355 | unsigned &Size, unsigned &Offset, |
356 | const MachineFunction &MF) const; |
357 | |
358 | /// Return true if the given instruction is terminator that is unspillable, |
359 | /// according to isUnspillableTerminatorImpl. |
360 | bool isUnspillableTerminator(const MachineInstr *MI) const { |
361 | return MI->isTerminator() && isUnspillableTerminatorImpl(MI); |
362 | } |
363 | |
364 | /// Returns the size in bytes of the specified MachineInstr, or ~0U |
365 | /// when this function is not implemented by a target. |
366 | virtual unsigned getInstSizeInBytes(const MachineInstr &MI) const { |
367 | return ~0U; |
368 | } |
369 | |
370 | /// Return true if the instruction is as cheap as a move instruction. |
371 | /// |
372 | /// Targets for different archs need to override this, and different |
373 | /// micro-architectures can also be finely tuned inside. |
374 | virtual bool isAsCheapAsAMove(const MachineInstr &MI) const { |
375 | return MI.isAsCheapAsAMove(); |
376 | } |
377 | |
378 | /// Return true if the instruction should be sunk by MachineSink. |
379 | /// |
380 | /// MachineSink determines on its own whether the instruction is safe to sink; |
381 | /// this gives the target a hook to override the default behavior with regards |
382 | /// to which instructions should be sunk. |
383 | virtual bool shouldSink(const MachineInstr &MI) const { return true; } |
384 | |
385 | /// Re-issue the specified 'original' instruction at the |
386 | /// specific location targeting a new destination register. |
387 | /// The register in Orig->getOperand(0).getReg() will be substituted by |
388 | /// DestReg:SubIdx. Any existing subreg index is preserved or composed with |
389 | /// SubIdx. |
390 | virtual void reMaterialize(MachineBasicBlock &MBB, |
391 | MachineBasicBlock::iterator MI, Register DestReg, |
392 | unsigned SubIdx, const MachineInstr &Orig, |
393 | const TargetRegisterInfo &TRI) const; |
394 | |
395 | /// Clones instruction or the whole instruction bundle \p Orig and |
396 | /// insert into \p MBB before \p InsertBefore. The target may update operands |
397 | /// that are required to be unique. |
398 | /// |
399 | /// \p Orig must not return true for MachineInstr::isNotDuplicable(). |
400 | virtual MachineInstr &duplicate(MachineBasicBlock &MBB, |
401 | MachineBasicBlock::iterator InsertBefore, |
402 | const MachineInstr &Orig) const; |
403 | |
404 | /// This method must be implemented by targets that |
405 | /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target |
406 | /// may be able to convert a two-address instruction into one or more true |
407 | /// three-address instructions on demand. This allows the X86 target (for |
408 | /// example) to convert ADD and SHL instructions into LEA instructions if they |
409 | /// would require register copies due to two-addressness. |
410 | /// |
411 | /// This method returns a null pointer if the transformation cannot be |
412 | /// performed, otherwise it returns the last new instruction. |
413 | /// |
414 | virtual MachineInstr *convertToThreeAddress(MachineInstr &MI, |
415 | LiveVariables *LV) const { |
416 | return nullptr; |
417 | } |
418 | |
419 | // This constant can be used as an input value of operand index passed to |
420 | // the method findCommutedOpIndices() to tell the method that the |
421 | // corresponding operand index is not pre-defined and that the method |
422 | // can pick any commutable operand. |
423 | static const unsigned CommuteAnyOperandIndex = ~0U; |
424 | |
425 | /// This method commutes the operands of the given machine instruction MI. |
426 | /// |
427 | /// The operands to be commuted are specified by their indices OpIdx1 and |
428 | /// OpIdx2. OpIdx1 and OpIdx2 arguments may be set to a special value |
429 | /// 'CommuteAnyOperandIndex', which means that the method is free to choose |
430 | /// any arbitrarily chosen commutable operand. If both arguments are set to |
431 | /// 'CommuteAnyOperandIndex' then the method looks for 2 different commutable |
432 | /// operands; then commutes them if such operands could be found. |
433 | /// |
434 | /// If NewMI is false, MI is modified in place and returned; otherwise, a |
435 | /// new machine instruction is created and returned. |
436 | /// |
437 | /// Do not call this method for a non-commutable instruction or |
438 | /// for non-commuable operands. |
439 | /// Even though the instruction is commutable, the method may still |
440 | /// fail to commute the operands, null pointer is returned in such cases. |
441 | MachineInstr * |
442 | commuteInstruction(MachineInstr &MI, bool NewMI = false, |
443 | unsigned OpIdx1 = CommuteAnyOperandIndex, |
444 | unsigned OpIdx2 = CommuteAnyOperandIndex) const; |
445 | |
446 | /// Returns true iff the routine could find two commutable operands in the |
447 | /// given machine instruction. |
448 | /// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. |
449 | /// If any of the INPUT values is set to the special value |
450 | /// 'CommuteAnyOperandIndex' then the method arbitrarily picks a commutable |
451 | /// operand, then returns its index in the corresponding argument. |
452 | /// If both of INPUT values are set to 'CommuteAnyOperandIndex' then method |
453 | /// looks for 2 commutable operands. |
454 | /// If INPUT values refer to some operands of MI, then the method simply |
455 | /// returns true if the corresponding operands are commutable and returns |
456 | /// false otherwise. |
457 | /// |
458 | /// For example, calling this method this way: |
459 | /// unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex; |
460 | /// findCommutedOpIndices(MI, Op1, Op2); |
461 | /// can be interpreted as a query asking to find an operand that would be |
462 | /// commutable with the operand#1. |
463 | virtual bool findCommutedOpIndices(const MachineInstr &MI, |
464 | unsigned &SrcOpIdx1, |
465 | unsigned &SrcOpIdx2) const; |
466 | |
467 | /// Returns true if the target has a preference on the operands order of |
468 | /// the given machine instruction. And specify if \p Commute is required to |
469 | /// get the desired operands order. |
470 | virtual bool hasCommutePreference(MachineInstr &MI, bool &Commute) const { |
471 | return false; |
472 | } |
473 | |
474 | /// A pair composed of a register and a sub-register index. |
475 | /// Used to give some type checking when modeling Reg:SubReg. |
476 | struct RegSubRegPair { |
477 | Register Reg; |
478 | unsigned SubReg; |
479 | |
480 | RegSubRegPair(Register Reg = Register(), unsigned SubReg = 0) |
481 | : Reg(Reg), SubReg(SubReg) {} |
482 | |
483 | bool operator==(const RegSubRegPair& P) const { |
484 | return Reg == P.Reg && SubReg == P.SubReg; |
485 | } |
486 | bool operator!=(const RegSubRegPair& P) const { |
487 | return !(*this == P); |
488 | } |
489 | }; |
490 | |
491 | /// A pair composed of a pair of a register and a sub-register index, |
492 | /// and another sub-register index. |
493 | /// Used to give some type checking when modeling Reg:SubReg1, SubReg2. |
494 | struct RegSubRegPairAndIdx : RegSubRegPair { |
495 | unsigned SubIdx; |
496 | |
497 | RegSubRegPairAndIdx(Register Reg = Register(), unsigned SubReg = 0, |
498 | unsigned SubIdx = 0) |
499 | : RegSubRegPair(Reg, SubReg), SubIdx(SubIdx) {} |
500 | }; |
501 | |
502 | /// Build the equivalent inputs of a REG_SEQUENCE for the given \p MI |
503 | /// and \p DefIdx. |
504 | /// \p [out] InputRegs of the equivalent REG_SEQUENCE. Each element of |
505 | /// the list is modeled as <Reg:SubReg, SubIdx>. Operands with the undef |
506 | /// flag are not added to this list. |
507 | /// E.g., REG_SEQUENCE %1:sub1, sub0, %2, sub1 would produce |
508 | /// two elements: |
509 | /// - %1:sub1, sub0 |
510 | /// - %2<:0>, sub1 |
511 | /// |
512 | /// \returns true if it is possible to build such an input sequence |
513 | /// with the pair \p MI, \p DefIdx. False otherwise. |
514 | /// |
515 | /// \pre MI.isRegSequence() or MI.isRegSequenceLike(). |
516 | /// |
517 | /// \note The generic implementation does not provide any support for |
518 | /// MI.isRegSequenceLike(). In other words, one has to override |
519 | /// getRegSequenceLikeInputs for target specific instructions. |
520 | bool |
521 | getRegSequenceInputs(const MachineInstr &MI, unsigned DefIdx, |
522 | SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const; |
523 | |
524 | /// Build the equivalent inputs of a EXTRACT_SUBREG for the given \p MI |
525 | /// and \p DefIdx. |
526 | /// \p [out] InputReg of the equivalent EXTRACT_SUBREG. |
527 | /// E.g., EXTRACT_SUBREG %1:sub1, sub0, sub1 would produce: |
528 | /// - %1:sub1, sub0 |
529 | /// |
530 | /// \returns true if it is possible to build such an input sequence |
531 | /// with the pair \p MI, \p DefIdx and the operand has no undef flag set. |
532 | /// False otherwise. |
533 | /// |
534 | /// \pre MI.isExtractSubreg() or MI.isExtractSubregLike(). |
535 | /// |
536 | /// \note The generic implementation does not provide any support for |
537 | /// MI.isExtractSubregLike(). In other words, one has to override |
538 | /// getExtractSubregLikeInputs for target specific instructions. |
539 | bool getExtractSubregInputs(const MachineInstr &MI, unsigned DefIdx, |
540 | RegSubRegPairAndIdx &InputReg) const; |
541 | |
542 | /// Build the equivalent inputs of a INSERT_SUBREG for the given \p MI |
543 | /// and \p DefIdx. |
544 | /// \p [out] BaseReg and \p [out] InsertedReg contain |
545 | /// the equivalent inputs of INSERT_SUBREG. |
546 | /// E.g., INSERT_SUBREG %0:sub0, %1:sub1, sub3 would produce: |
547 | /// - BaseReg: %0:sub0 |
548 | /// - InsertedReg: %1:sub1, sub3 |
549 | /// |
550 | /// \returns true if it is possible to build such an input sequence |
551 | /// with the pair \p MI, \p DefIdx and the operand has no undef flag set. |
552 | /// False otherwise. |
553 | /// |
554 | /// \pre MI.isInsertSubreg() or MI.isInsertSubregLike(). |
555 | /// |
556 | /// \note The generic implementation does not provide any support for |
557 | /// MI.isInsertSubregLike(). In other words, one has to override |
558 | /// getInsertSubregLikeInputs for target specific instructions. |
559 | bool getInsertSubregInputs(const MachineInstr &MI, unsigned DefIdx, |
560 | RegSubRegPair &BaseReg, |
561 | RegSubRegPairAndIdx &InsertedReg) const; |
562 | |
563 | /// Return true if two machine instructions would produce identical values. |
564 | /// By default, this is only true when the two instructions |
565 | /// are deemed identical except for defs. If this function is called when the |
566 | /// IR is still in SSA form, the caller can pass the MachineRegisterInfo for |
567 | /// aggressive checks. |
568 | virtual bool produceSameValue(const MachineInstr &MI0, |
569 | const MachineInstr &MI1, |
570 | const MachineRegisterInfo *MRI = nullptr) const; |
571 | |
572 | /// \returns true if a branch from an instruction with opcode \p BranchOpc |
573 | /// bytes is capable of jumping to a position \p BrOffset bytes away. |
574 | virtual bool isBranchOffsetInRange(unsigned BranchOpc, |
575 | int64_t BrOffset) const { |
576 | llvm_unreachable("target did not implement")::llvm::llvm_unreachable_internal("target did not implement", "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 576); |
577 | } |
578 | |
579 | /// \returns The block that branch instruction \p MI jumps to. |
580 | virtual MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const { |
581 | llvm_unreachable("target did not implement")::llvm::llvm_unreachable_internal("target did not implement", "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 581); |
582 | } |
583 | |
584 | /// Insert an unconditional indirect branch at the end of \p MBB to \p |
585 | /// NewDestBB. Optionally, insert the clobbered register restoring in \p |
586 | /// RestoreBB. \p BrOffset indicates the offset of \p NewDestBB relative to |
587 | /// the offset of the position to insert the new branch. |
588 | virtual void insertIndirectBranch(MachineBasicBlock &MBB, |
589 | MachineBasicBlock &NewDestBB, |
590 | MachineBasicBlock &RestoreBB, |
591 | const DebugLoc &DL, int64_t BrOffset = 0, |
592 | RegScavenger *RS = nullptr) const { |
593 | llvm_unreachable("target did not implement")::llvm::llvm_unreachable_internal("target did not implement", "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 593); |
594 | } |
595 | |
596 | /// Analyze the branching code at the end of MBB, returning |
597 | /// true if it cannot be understood (e.g. it's a switch dispatch or isn't |
598 | /// implemented for a target). Upon success, this returns false and returns |
599 | /// with the following information in various cases: |
600 | /// |
601 | /// 1. If this block ends with no branches (it just falls through to its succ) |
602 | /// just return false, leaving TBB/FBB null. |
603 | /// 2. If this block ends with only an unconditional branch, it sets TBB to be |
604 | /// the destination block. |
605 | /// 3. If this block ends with a conditional branch and it falls through to a |
606 | /// successor block, it sets TBB to be the branch destination block and a |
607 | /// list of operands that evaluate the condition. These operands can be |
608 | /// passed to other TargetInstrInfo methods to create new branches. |
609 | /// 4. If this block ends with a conditional branch followed by an |
610 | /// unconditional branch, it returns the 'true' destination in TBB, the |
611 | /// 'false' destination in FBB, and a list of operands that evaluate the |
612 | /// condition. These operands can be passed to other TargetInstrInfo |
613 | /// methods to create new branches. |
614 | /// |
615 | /// Note that removeBranch and insertBranch must be implemented to support |
616 | /// cases where this method returns success. |
617 | /// |
618 | /// If AllowModify is true, then this routine is allowed to modify the basic |
619 | /// block (e.g. delete instructions after the unconditional branch). |
620 | /// |
621 | /// The CFG information in MBB.Predecessors and MBB.Successors must be valid |
622 | /// before calling this function. |
623 | virtual bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, |
624 | MachineBasicBlock *&FBB, |
625 | SmallVectorImpl<MachineOperand> &Cond, |
626 | bool AllowModify = false) const { |
627 | return true; |
628 | } |
629 | |
630 | /// Represents a predicate at the MachineFunction level. The control flow a |
631 | /// MachineBranchPredicate represents is: |
632 | /// |
633 | /// Reg = LHS `Predicate` RHS == ConditionDef |
634 | /// if Reg then goto TrueDest else goto FalseDest |
635 | /// |
636 | struct MachineBranchPredicate { |
637 | enum ComparePredicate { |
638 | PRED_EQ, // True if two values are equal |
639 | PRED_NE, // True if two values are not equal |
640 | PRED_INVALID // Sentinel value |
641 | }; |
642 | |
643 | ComparePredicate Predicate = PRED_INVALID; |
644 | MachineOperand LHS = MachineOperand::CreateImm(0); |
645 | MachineOperand RHS = MachineOperand::CreateImm(0); |
646 | MachineBasicBlock *TrueDest = nullptr; |
647 | MachineBasicBlock *FalseDest = nullptr; |
648 | MachineInstr *ConditionDef = nullptr; |
649 | |
650 | /// SingleUseCondition is true if ConditionDef is dead except for the |
651 | /// branch(es) at the end of the basic block. |
652 | /// |
653 | bool SingleUseCondition = false; |
654 | |
655 | explicit MachineBranchPredicate() = default; |
656 | }; |
657 | |
658 | /// Analyze the branching code at the end of MBB and parse it into the |
659 | /// MachineBranchPredicate structure if possible. Returns false on success |
660 | /// and true on failure. |
661 | /// |
662 | /// If AllowModify is true, then this routine is allowed to modify the basic |
663 | /// block (e.g. delete instructions after the unconditional branch). |
664 | /// |
665 | virtual bool analyzeBranchPredicate(MachineBasicBlock &MBB, |
666 | MachineBranchPredicate &MBP, |
667 | bool AllowModify = false) const { |
668 | return true; |
669 | } |
670 | |
671 | /// Remove the branching code at the end of the specific MBB. |
672 | /// This is only invoked in cases where analyzeBranch returns success. It |
673 | /// returns the number of instructions that were removed. |
674 | /// If \p BytesRemoved is non-null, report the change in code size from the |
675 | /// removed instructions. |
676 | virtual unsigned removeBranch(MachineBasicBlock &MBB, |
677 | int *BytesRemoved = nullptr) const { |
678 | llvm_unreachable("Target didn't implement TargetInstrInfo::removeBranch!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::removeBranch!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 678); |
679 | } |
680 | |
681 | /// Insert branch code into the end of the specified MachineBasicBlock. The |
682 | /// operands to this method are the same as those returned by analyzeBranch. |
683 | /// This is only invoked in cases where analyzeBranch returns success. It |
684 | /// returns the number of instructions inserted. If \p BytesAdded is non-null, |
685 | /// report the change in code size from the added instructions. |
686 | /// |
687 | /// It is also invoked by tail merging to add unconditional branches in |
688 | /// cases where analyzeBranch doesn't apply because there was no original |
689 | /// branch to analyze. At least this much must be implemented, else tail |
690 | /// merging needs to be disabled. |
691 | /// |
692 | /// The CFG information in MBB.Predecessors and MBB.Successors must be valid |
693 | /// before calling this function. |
694 | virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, |
695 | MachineBasicBlock *FBB, |
696 | ArrayRef<MachineOperand> Cond, |
697 | const DebugLoc &DL, |
698 | int *BytesAdded = nullptr) const { |
699 | llvm_unreachable("Target didn't implement TargetInstrInfo::insertBranch!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::insertBranch!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 699); |
700 | } |
701 | |
702 | unsigned insertUnconditionalBranch(MachineBasicBlock &MBB, |
703 | MachineBasicBlock *DestBB, |
704 | const DebugLoc &DL, |
705 | int *BytesAdded = nullptr) const { |
706 | return insertBranch(MBB, DestBB, nullptr, ArrayRef<MachineOperand>(), DL, |
707 | BytesAdded); |
708 | } |
709 | |
710 | /// Object returned by analyzeLoopForPipelining. Allows software pipelining |
711 | /// implementations to query attributes of the loop being pipelined and to |
712 | /// apply target-specific updates to the loop once pipelining is complete. |
713 | class PipelinerLoopInfo { |
714 | public: |
715 | virtual ~PipelinerLoopInfo(); |
716 | /// Return true if the given instruction should not be pipelined and should |
717 | /// be ignored. An example could be a loop comparison, or induction variable |
718 | /// update with no users being pipelined. |
719 | virtual bool shouldIgnoreForPipelining(const MachineInstr *MI) const = 0; |
720 | |
721 | /// Create a condition to determine if the trip count of the loop is greater |
722 | /// than TC. |
723 | /// |
724 | /// If the trip count is statically known to be greater than TC, return |
725 | /// true. If the trip count is statically known to be not greater than TC, |
726 | /// return false. Otherwise return nullopt and fill out Cond with the test |
727 | /// condition. |
728 | virtual Optional<bool> |
729 | createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB, |
730 | SmallVectorImpl<MachineOperand> &Cond) = 0; |
731 | |
732 | /// Modify the loop such that the trip count is |
733 | /// OriginalTC + TripCountAdjust. |
734 | virtual void adjustTripCount(int TripCountAdjust) = 0; |
735 | |
736 | /// Called when the loop's preheader has been modified to NewPreheader. |
737 | virtual void setPreheader(MachineBasicBlock *NewPreheader) = 0; |
738 | |
739 | /// Called when the loop is being removed. Any instructions in the preheader |
740 | /// should be removed. |
741 | /// |
742 | /// Once this function is called, no other functions on this object are |
743 | /// valid; the loop has been removed. |
744 | virtual void disposed() = 0; |
745 | }; |
746 | |
747 | /// Analyze loop L, which must be a single-basic-block loop, and if the |
748 | /// conditions can be understood enough produce a PipelinerLoopInfo object. |
749 | virtual std::unique_ptr<PipelinerLoopInfo> |
750 | analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const { |
751 | return nullptr; |
752 | } |
753 | |
754 | /// Analyze the loop code, return true if it cannot be understood. Upon |
755 | /// success, this function returns false and returns information about the |
756 | /// induction variable and compare instruction used at the end. |
757 | virtual bool analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst, |
758 | MachineInstr *&CmpInst) const { |
759 | return true; |
760 | } |
761 | |
762 | /// Generate code to reduce the loop iteration by one and check if the loop |
763 | /// is finished. Return the value/register of the new loop count. We need |
764 | /// this function when peeling off one or more iterations of a loop. This |
765 | /// function assumes the nth iteration is peeled first. |
766 | virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, |
767 | MachineBasicBlock &PreHeader, |
768 | MachineInstr *IndVar, MachineInstr &Cmp, |
769 | SmallVectorImpl<MachineOperand> &Cond, |
770 | SmallVectorImpl<MachineInstr *> &PrevInsts, |
771 | unsigned Iter, unsigned MaxIter) const { |
772 | llvm_unreachable("Target didn't implement ReduceLoopCount")::llvm::llvm_unreachable_internal("Target didn't implement ReduceLoopCount" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 772); |
773 | } |
774 | |
775 | /// Delete the instruction OldInst and everything after it, replacing it with |
776 | /// an unconditional branch to NewDest. This is used by the tail merging pass. |
777 | virtual void ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, |
778 | MachineBasicBlock *NewDest) const; |
779 | |
780 | /// Return true if it's legal to split the given basic |
781 | /// block at the specified instruction (i.e. instruction would be the start |
782 | /// of a new basic block). |
783 | virtual bool isLegalToSplitMBBAt(MachineBasicBlock &MBB, |
784 | MachineBasicBlock::iterator MBBI) const { |
785 | return true; |
786 | } |
787 | |
788 | /// Return true if it's profitable to predicate |
789 | /// instructions with accumulated instruction latency of "NumCycles" |
790 | /// of the specified basic block, where the probability of the instructions |
791 | /// being executed is given by Probability, and Confidence is a measure |
792 | /// of our confidence that it will be properly predicted. |
793 | virtual bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, |
794 | unsigned ExtraPredCycles, |
795 | BranchProbability Probability) const { |
796 | return false; |
797 | } |
798 | |
799 | /// Second variant of isProfitableToIfCvt. This one |
800 | /// checks for the case where two basic blocks from true and false path |
801 | /// of a if-then-else (diamond) are predicated on mutually exclusive |
802 | /// predicates, where the probability of the true path being taken is given |
803 | /// by Probability, and Confidence is a measure of our confidence that it |
804 | /// will be properly predicted. |
805 | virtual bool isProfitableToIfCvt(MachineBasicBlock &TMBB, unsigned NumTCycles, |
806 | unsigned ExtraTCycles, |
807 | MachineBasicBlock &FMBB, unsigned NumFCycles, |
808 | unsigned ExtraFCycles, |
809 | BranchProbability Probability) const { |
810 | return false; |
811 | } |
812 | |
813 | /// Return true if it's profitable for if-converter to duplicate instructions |
814 | /// of specified accumulated instruction latencies in the specified MBB to |
815 | /// enable if-conversion. |
816 | /// The probability of the instructions being executed is given by |
817 | /// Probability, and Confidence is a measure of our confidence that it |
818 | /// will be properly predicted. |
819 | virtual bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, |
820 | unsigned NumCycles, |
821 | BranchProbability Probability) const { |
822 | return false; |
823 | } |
824 | |
825 | /// Return the increase in code size needed to predicate a contiguous run of |
826 | /// NumInsts instructions. |
827 | virtual unsigned extraSizeToPredicateInstructions(const MachineFunction &MF, |
828 | unsigned NumInsts) const { |
829 | return 0; |
830 | } |
831 | |
832 | /// Return an estimate for the code size reduction (in bytes) which will be |
833 | /// caused by removing the given branch instruction during if-conversion. |
834 | virtual unsigned predictBranchSizeForIfCvt(MachineInstr &MI) const { |
835 | return getInstSizeInBytes(MI); |
836 | } |
837 | |
838 | /// Return true if it's profitable to unpredicate |
839 | /// one side of a 'diamond', i.e. two sides of if-else predicated on mutually |
840 | /// exclusive predicates. |
841 | /// e.g. |
842 | /// subeq r0, r1, #1 |
843 | /// addne r0, r1, #1 |
844 | /// => |
845 | /// sub r0, r1, #1 |
846 | /// addne r0, r1, #1 |
847 | /// |
848 | /// This may be profitable is conditional instructions are always executed. |
849 | virtual bool isProfitableToUnpredicate(MachineBasicBlock &TMBB, |
850 | MachineBasicBlock &FMBB) const { |
851 | return false; |
852 | } |
853 | |
854 | /// Return true if it is possible to insert a select |
855 | /// instruction that chooses between TrueReg and FalseReg based on the |
856 | /// condition code in Cond. |
857 | /// |
858 | /// When successful, also return the latency in cycles from TrueReg, |
859 | /// FalseReg, and Cond to the destination register. In most cases, a select |
860 | /// instruction will be 1 cycle, so CondCycles = TrueCycles = FalseCycles = 1 |
861 | /// |
862 | /// Some x86 implementations have 2-cycle cmov instructions. |
863 | /// |
864 | /// @param MBB Block where select instruction would be inserted. |
865 | /// @param Cond Condition returned by analyzeBranch. |
866 | /// @param DstReg Virtual dest register that the result should write to. |
867 | /// @param TrueReg Virtual register to select when Cond is true. |
868 | /// @param FalseReg Virtual register to select when Cond is false. |
869 | /// @param CondCycles Latency from Cond+Branch to select output. |
870 | /// @param TrueCycles Latency from TrueReg to select output. |
871 | /// @param FalseCycles Latency from FalseReg to select output. |
872 | virtual bool canInsertSelect(const MachineBasicBlock &MBB, |
873 | ArrayRef<MachineOperand> Cond, Register DstReg, |
874 | Register TrueReg, Register FalseReg, |
875 | int &CondCycles, int &TrueCycles, |
876 | int &FalseCycles) const { |
877 | return false; |
878 | } |
879 | |
880 | /// Insert a select instruction into MBB before I that will copy TrueReg to |
881 | /// DstReg when Cond is true, and FalseReg to DstReg when Cond is false. |
882 | /// |
883 | /// This function can only be called after canInsertSelect() returned true. |
884 | /// The condition in Cond comes from analyzeBranch, and it can be assumed |
885 | /// that the same flags or registers required by Cond are available at the |
886 | /// insertion point. |
887 | /// |
888 | /// @param MBB Block where select instruction should be inserted. |
889 | /// @param I Insertion point. |
890 | /// @param DL Source location for debugging. |
891 | /// @param DstReg Virtual register to be defined by select instruction. |
892 | /// @param Cond Condition as computed by analyzeBranch. |
893 | /// @param TrueReg Virtual register to copy when Cond is true. |
894 | /// @param FalseReg Virtual register to copy when Cons is false. |
895 | virtual void insertSelect(MachineBasicBlock &MBB, |
896 | MachineBasicBlock::iterator I, const DebugLoc &DL, |
897 | Register DstReg, ArrayRef<MachineOperand> Cond, |
898 | Register TrueReg, Register FalseReg) const { |
899 | llvm_unreachable("Target didn't implement TargetInstrInfo::insertSelect!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::insertSelect!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 899); |
900 | } |
901 | |
902 | /// Analyze the given select instruction, returning true if |
903 | /// it cannot be understood. It is assumed that MI->isSelect() is true. |
904 | /// |
905 | /// When successful, return the controlling condition and the operands that |
906 | /// determine the true and false result values. |
907 | /// |
908 | /// Result = SELECT Cond, TrueOp, FalseOp |
909 | /// |
910 | /// Some targets can optimize select instructions, for example by predicating |
911 | /// the instruction defining one of the operands. Such targets should set |
912 | /// Optimizable. |
913 | /// |
914 | /// @param MI Select instruction to analyze. |
915 | /// @param Cond Condition controlling the select. |
916 | /// @param TrueOp Operand number of the value selected when Cond is true. |
917 | /// @param FalseOp Operand number of the value selected when Cond is false. |
918 | /// @param Optimizable Returned as true if MI is optimizable. |
919 | /// @returns False on success. |
920 | virtual bool analyzeSelect(const MachineInstr &MI, |
921 | SmallVectorImpl<MachineOperand> &Cond, |
922 | unsigned &TrueOp, unsigned &FalseOp, |
923 | bool &Optimizable) const { |
924 | assert(MI.getDesc().isSelect() && "MI must be a select instruction")(static_cast <bool> (MI.getDesc().isSelect() && "MI must be a select instruction") ? void (0) : __assert_fail ("MI.getDesc().isSelect() && \"MI must be a select instruction\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 924, __extension__ __PRETTY_FUNCTION__)); |
925 | return true; |
926 | } |
927 | |
928 | /// Given a select instruction that was understood by |
929 | /// analyzeSelect and returned Optimizable = true, attempt to optimize MI by |
930 | /// merging it with one of its operands. Returns NULL on failure. |
931 | /// |
932 | /// When successful, returns the new select instruction. The client is |
933 | /// responsible for deleting MI. |
934 | /// |
935 | /// If both sides of the select can be optimized, PreferFalse is used to pick |
936 | /// a side. |
937 | /// |
938 | /// @param MI Optimizable select instruction. |
939 | /// @param NewMIs Set that record all MIs in the basic block up to \p |
940 | /// MI. Has to be updated with any newly created MI or deleted ones. |
941 | /// @param PreferFalse Try to optimize FalseOp instead of TrueOp. |
942 | /// @returns Optimized instruction or NULL. |
943 | virtual MachineInstr *optimizeSelect(MachineInstr &MI, |
944 | SmallPtrSetImpl<MachineInstr *> &NewMIs, |
945 | bool PreferFalse = false) const { |
946 | // This function must be implemented if Optimizable is ever set. |
947 | llvm_unreachable("Target must implement TargetInstrInfo::optimizeSelect!")::llvm::llvm_unreachable_internal("Target must implement TargetInstrInfo::optimizeSelect!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 947); |
948 | } |
949 | |
950 | /// Emit instructions to copy a pair of physical registers. |
951 | /// |
952 | /// This function should support copies within any legal register class as |
953 | /// well as any cross-class copies created during instruction selection. |
954 | /// |
955 | /// The source and destination registers may overlap, which may require a |
956 | /// careful implementation when multiple copy instructions are required for |
957 | /// large registers. See for example the ARM target. |
958 | virtual void copyPhysReg(MachineBasicBlock &MBB, |
959 | MachineBasicBlock::iterator MI, const DebugLoc &DL, |
960 | MCRegister DestReg, MCRegister SrcReg, |
961 | bool KillSrc) const { |
962 | llvm_unreachable("Target didn't implement TargetInstrInfo::copyPhysReg!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::copyPhysReg!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 962); |
963 | } |
964 | |
965 | /// Allow targets to tell MachineVerifier whether a specific register |
966 | /// MachineOperand can be used as part of PC-relative addressing. |
967 | /// PC-relative addressing modes in many CISC architectures contain |
968 | /// (non-PC) registers as offsets or scaling values, which inherently |
969 | /// tags the corresponding MachineOperand with OPERAND_PCREL. |
970 | /// |
971 | /// @param MO The MachineOperand in question. MO.isReg() should always |
972 | /// be true. |
973 | /// @return Whether this operand is allowed to be used PC-relatively. |
974 | virtual bool isPCRelRegisterOperandLegal(const MachineOperand &MO) const { |
975 | return false; |
976 | } |
977 | |
978 | protected: |
979 | /// Target-dependent implementation for IsCopyInstr. |
980 | /// If the specific machine instruction is a instruction that moves/copies |
981 | /// value from one register to another register return destination and source |
982 | /// registers as machine operands. |
983 | virtual Optional<DestSourcePair> |
984 | isCopyInstrImpl(const MachineInstr &MI) const { |
985 | return None; |
986 | } |
987 | |
988 | /// Return true if the given terminator MI is not expected to spill. This |
989 | /// sets the live interval as not spillable and adjusts phi node lowering to |
990 | /// not introduce copies after the terminator. Use with care, these are |
991 | /// currently used for hardware loop intrinsics in very controlled situations, |
992 | /// created prior to registry allocation in loops that only have single phi |
993 | /// users for the terminators value. They may run out of registers if not used |
994 | /// carefully. |
995 | virtual bool isUnspillableTerminatorImpl(const MachineInstr *MI) const { |
996 | return false; |
997 | } |
998 | |
999 | public: |
1000 | /// If the specific machine instruction is a instruction that moves/copies |
1001 | /// value from one register to another register return destination and source |
1002 | /// registers as machine operands. |
1003 | /// For COPY-instruction the method naturally returns destination and source |
1004 | /// registers as machine operands, for all other instructions the method calls |
1005 | /// target-dependent implementation. |
1006 | Optional<DestSourcePair> isCopyInstr(const MachineInstr &MI) const { |
1007 | if (MI.isCopy()) { |
1008 | return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; |
1009 | } |
1010 | return isCopyInstrImpl(MI); |
1011 | } |
1012 | |
1013 | /// If the specific machine instruction is an instruction that adds an |
1014 | /// immediate value and a physical register, and stores the result in |
1015 | /// the given physical register \c Reg, return a pair of the source |
1016 | /// register and the offset which has been added. |
1017 | virtual Optional<RegImmPair> isAddImmediate(const MachineInstr &MI, |
1018 | Register Reg) const { |
1019 | return None; |
1020 | } |
1021 | |
1022 | /// Returns true if MI is an instruction that defines Reg to have a constant |
1023 | /// value and the value is recorded in ImmVal. The ImmVal is a result that |
1024 | /// should be interpreted as modulo size of Reg. |
1025 | virtual bool getConstValDefinedInReg(const MachineInstr &MI, |
1026 | const Register Reg, |
1027 | int64_t &ImmVal) const { |
1028 | return false; |
1029 | } |
1030 | |
1031 | /// Store the specified register of the given register class to the specified |
1032 | /// stack frame index. The store instruction is to be added to the given |
1033 | /// machine basic block before the specified machine instruction. If isKill |
1034 | /// is true, the register operand is the last use and must be marked kill. |
1035 | virtual void storeRegToStackSlot(MachineBasicBlock &MBB, |
1036 | MachineBasicBlock::iterator MI, |
1037 | Register SrcReg, bool isKill, int FrameIndex, |
1038 | const TargetRegisterClass *RC, |
1039 | const TargetRegisterInfo *TRI) const { |
1040 | llvm_unreachable("Target didn't implement "::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::storeRegToStackSlot!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1041) |
1041 | "TargetInstrInfo::storeRegToStackSlot!")::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::storeRegToStackSlot!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1041); |
1042 | } |
1043 | |
1044 | /// Load the specified register of the given register class from the specified |
1045 | /// stack frame index. The load instruction is to be added to the given |
1046 | /// machine basic block before the specified machine instruction. |
1047 | virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, |
1048 | MachineBasicBlock::iterator MI, |
1049 | Register DestReg, int FrameIndex, |
1050 | const TargetRegisterClass *RC, |
1051 | const TargetRegisterInfo *TRI) const { |
1052 | llvm_unreachable("Target didn't implement "::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::loadRegFromStackSlot!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1053) |
1053 | "TargetInstrInfo::loadRegFromStackSlot!")::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::loadRegFromStackSlot!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1053); |
1054 | } |
1055 | |
1056 | /// This function is called for all pseudo instructions |
1057 | /// that remain after register allocation. Many pseudo instructions are |
1058 | /// created to help register allocation. This is the place to convert them |
1059 | /// into real instructions. The target can edit MI in place, or it can insert |
1060 | /// new instructions and erase MI. The function should return true if |
1061 | /// anything was changed. |
1062 | virtual bool expandPostRAPseudo(MachineInstr &MI) const { return false; } |
1063 | |
1064 | /// Check whether the target can fold a load that feeds a subreg operand |
1065 | /// (or a subreg operand that feeds a store). |
1066 | /// For example, X86 may want to return true if it can fold |
1067 | /// movl (%esp), %eax |
1068 | /// subb, %al, ... |
1069 | /// Into: |
1070 | /// subb (%esp), ... |
1071 | /// |
1072 | /// Ideally, we'd like the target implementation of foldMemoryOperand() to |
1073 | /// reject subregs - but since this behavior used to be enforced in the |
1074 | /// target-independent code, moving this responsibility to the targets |
1075 | /// has the potential of causing nasty silent breakage in out-of-tree targets. |
1076 | virtual bool isSubregFoldable() const { return false; } |
1077 | |
1078 | /// For a patchpoint, stackmap, or statepoint intrinsic, return the range of |
1079 | /// operands which can't be folded into stack references. Operands outside |
1080 | /// of the range are most likely foldable but it is not guaranteed. |
1081 | /// These instructions are unique in that stack references for some operands |
1082 | /// have the same execution cost (e.g. none) as the unfolded register forms. |
1083 | /// The ranged return is guaranteed to include all operands which can't be |
1084 | /// folded at zero cost. |
1085 | virtual std::pair<unsigned, unsigned> |
1086 | getPatchpointUnfoldableRange(const MachineInstr &MI) const; |
1087 | |
1088 | /// Attempt to fold a load or store of the specified stack |
1089 | /// slot into the specified machine instruction for the specified operand(s). |
1090 | /// If this is possible, a new instruction is returned with the specified |
1091 | /// operand folded, otherwise NULL is returned. |
1092 | /// The new instruction is inserted before MI, and the client is responsible |
1093 | /// for removing the old instruction. |
1094 | /// If VRM is passed, the assigned physregs can be inspected by target to |
1095 | /// decide on using an opcode (note that those assignments can still change). |
1096 | MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, |
1097 | int FI, |
1098 | LiveIntervals *LIS = nullptr, |
1099 | VirtRegMap *VRM = nullptr) const; |
1100 | |
1101 | /// Same as the previous version except it allows folding of any load and |
1102 | /// store from / to any address, not just from a specific stack slot. |
1103 | MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, |
1104 | MachineInstr &LoadMI, |
1105 | LiveIntervals *LIS = nullptr) const; |
1106 | |
1107 | /// Return true when there is potentially a faster code sequence |
1108 | /// for an instruction chain ending in \p Root. All potential patterns are |
1109 | /// returned in the \p Pattern vector. Pattern should be sorted in priority |
1110 | /// order since the pattern evaluator stops checking as soon as it finds a |
1111 | /// faster sequence. |
1112 | /// \param Root - Instruction that could be combined with one of its operands |
1113 | /// \param Patterns - Vector of possible combination patterns |
1114 | virtual bool |
1115 | getMachineCombinerPatterns(MachineInstr &Root, |
1116 | SmallVectorImpl<MachineCombinerPattern> &Patterns, |
1117 | bool DoRegPressureReduce) const; |
1118 | |
1119 | /// Return true if target supports reassociation of instructions in machine |
1120 | /// combiner pass to reduce register pressure for a given BB. |
1121 | virtual bool |
1122 | shouldReduceRegisterPressure(MachineBasicBlock *MBB, |
1123 | RegisterClassInfo *RegClassInfo) const { |
1124 | return false; |
1125 | } |
1126 | |
1127 | /// Fix up the placeholder we may add in genAlternativeCodeSequence(). |
1128 | virtual void |
1129 | finalizeInsInstrs(MachineInstr &Root, MachineCombinerPattern &P, |
1130 | SmallVectorImpl<MachineInstr *> &InsInstrs) const {} |
1131 | |
1132 | /// Return true when a code sequence can improve throughput. It |
1133 | /// should be called only for instructions in loops. |
1134 | /// \param Pattern - combiner pattern |
1135 | virtual bool isThroughputPattern(MachineCombinerPattern Pattern) const; |
1136 | |
1137 | /// Return true if the input \P Inst is part of a chain of dependent ops |
1138 | /// that are suitable for reassociation, otherwise return false. |
1139 | /// If the instruction's operands must be commuted to have a previous |
1140 | /// instruction of the same type define the first source operand, \P Commuted |
1141 | /// will be set to true. |
1142 | bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) const; |
1143 | |
1144 | /// Return true when \P Inst is both associative and commutative. |
1145 | virtual bool isAssociativeAndCommutative(const MachineInstr &Inst) const { |
1146 | return false; |
1147 | } |
1148 | |
1149 | /// Return true when \P Inst has reassociable operands in the same \P MBB. |
1150 | virtual bool hasReassociableOperands(const MachineInstr &Inst, |
1151 | const MachineBasicBlock *MBB) const; |
1152 | |
1153 | /// Return true when \P Inst has reassociable sibling. |
1154 | bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const; |
1155 | |
1156 | /// When getMachineCombinerPatterns() finds patterns, this function generates |
1157 | /// the instructions that could replace the original code sequence. The client |
1158 | /// has to decide whether the actual replacement is beneficial or not. |
1159 | /// \param Root - Instruction that could be combined with one of its operands |
1160 | /// \param Pattern - Combination pattern for Root |
1161 | /// \param InsInstrs - Vector of new instructions that implement P |
1162 | /// \param DelInstrs - Old instructions, including Root, that could be |
1163 | /// replaced by InsInstr |
1164 | /// \param InstIdxForVirtReg - map of virtual register to instruction in |
1165 | /// InsInstr that defines it |
1166 | virtual void genAlternativeCodeSequence( |
1167 | MachineInstr &Root, MachineCombinerPattern Pattern, |
1168 | SmallVectorImpl<MachineInstr *> &InsInstrs, |
1169 | SmallVectorImpl<MachineInstr *> &DelInstrs, |
1170 | DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const; |
1171 | |
1172 | /// Attempt to reassociate \P Root and \P Prev according to \P Pattern to |
1173 | /// reduce critical path length. |
1174 | void reassociateOps(MachineInstr &Root, MachineInstr &Prev, |
1175 | MachineCombinerPattern Pattern, |
1176 | SmallVectorImpl<MachineInstr *> &InsInstrs, |
1177 | SmallVectorImpl<MachineInstr *> &DelInstrs, |
1178 | DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const; |
1179 | |
1180 | /// The limit on resource length extension we accept in MachineCombiner Pass. |
1181 | virtual int getExtendResourceLenLimit() const { return 0; } |
1182 | |
1183 | /// This is an architecture-specific helper function of reassociateOps. |
1184 | /// Set special operand attributes for new instructions after reassociation. |
1185 | virtual void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, |
1186 | MachineInstr &NewMI1, |
1187 | MachineInstr &NewMI2) const {} |
1188 | |
1189 | virtual void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const {} |
1190 | |
1191 | /// Return true when a target supports MachineCombiner. |
1192 | virtual bool useMachineCombiner() const { return false; } |
1193 | |
1194 | /// Return true if the given SDNode can be copied during scheduling |
1195 | /// even if it has glue. |
1196 | virtual bool canCopyGluedNodeDuringSchedule(SDNode *N) const { return false; } |
1197 | |
1198 | protected: |
1199 | /// Target-dependent implementation for foldMemoryOperand. |
1200 | /// Target-independent code in foldMemoryOperand will |
1201 | /// take care of adding a MachineMemOperand to the newly created instruction. |
1202 | /// The instruction and any auxiliary instructions necessary will be inserted |
1203 | /// at InsertPt. |
1204 | virtual MachineInstr * |
1205 | foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, |
1206 | ArrayRef<unsigned> Ops, |
1207 | MachineBasicBlock::iterator InsertPt, int FrameIndex, |
1208 | LiveIntervals *LIS = nullptr, |
1209 | VirtRegMap *VRM = nullptr) const { |
1210 | return nullptr; |
1211 | } |
1212 | |
1213 | /// Target-dependent implementation for foldMemoryOperand. |
1214 | /// Target-independent code in foldMemoryOperand will |
1215 | /// take care of adding a MachineMemOperand to the newly created instruction. |
1216 | /// The instruction and any auxiliary instructions necessary will be inserted |
1217 | /// at InsertPt. |
1218 | virtual MachineInstr *foldMemoryOperandImpl( |
1219 | MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, |
1220 | MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, |
1221 | LiveIntervals *LIS = nullptr) const { |
1222 | return nullptr; |
1223 | } |
1224 | |
1225 | /// Target-dependent implementation of getRegSequenceInputs. |
1226 | /// |
1227 | /// \returns true if it is possible to build the equivalent |
1228 | /// REG_SEQUENCE inputs with the pair \p MI, \p DefIdx. False otherwise. |
1229 | /// |
1230 | /// \pre MI.isRegSequenceLike(). |
1231 | /// |
1232 | /// \see TargetInstrInfo::getRegSequenceInputs. |
1233 | virtual bool getRegSequenceLikeInputs( |
1234 | const MachineInstr &MI, unsigned DefIdx, |
1235 | SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { |
1236 | return false; |
1237 | } |
1238 | |
1239 | /// Target-dependent implementation of getExtractSubregInputs. |
1240 | /// |
1241 | /// \returns true if it is possible to build the equivalent |
1242 | /// EXTRACT_SUBREG inputs with the pair \p MI, \p DefIdx. False otherwise. |
1243 | /// |
1244 | /// \pre MI.isExtractSubregLike(). |
1245 | /// |
1246 | /// \see TargetInstrInfo::getExtractSubregInputs. |
1247 | virtual bool getExtractSubregLikeInputs(const MachineInstr &MI, |
1248 | unsigned DefIdx, |
1249 | RegSubRegPairAndIdx &InputReg) const { |
1250 | return false; |
1251 | } |
1252 | |
1253 | /// Target-dependent implementation of getInsertSubregInputs. |
1254 | /// |
1255 | /// \returns true if it is possible to build the equivalent |
1256 | /// INSERT_SUBREG inputs with the pair \p MI, \p DefIdx. False otherwise. |
1257 | /// |
1258 | /// \pre MI.isInsertSubregLike(). |
1259 | /// |
1260 | /// \see TargetInstrInfo::getInsertSubregInputs. |
1261 | virtual bool |
1262 | getInsertSubregLikeInputs(const MachineInstr &MI, unsigned DefIdx, |
1263 | RegSubRegPair &BaseReg, |
1264 | RegSubRegPairAndIdx &InsertedReg) const { |
1265 | return false; |
1266 | } |
1267 | |
1268 | public: |
1269 | /// getAddressSpaceForPseudoSourceKind - Given the kind of memory |
1270 | /// (e.g. stack) the target returns the corresponding address space. |
1271 | virtual unsigned |
1272 | getAddressSpaceForPseudoSourceKind(unsigned Kind) const { |
1273 | return 0; |
1274 | } |
1275 | |
1276 | /// unfoldMemoryOperand - Separate a single instruction which folded a load or |
1277 | /// a store or a load and a store into two or more instruction. If this is |
1278 | /// possible, returns true as well as the new instructions by reference. |
1279 | virtual bool |
1280 | unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, unsigned Reg, |
1281 | bool UnfoldLoad, bool UnfoldStore, |
1282 | SmallVectorImpl<MachineInstr *> &NewMIs) const { |
1283 | return false; |
1284 | } |
1285 | |
1286 | virtual bool unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, |
1287 | SmallVectorImpl<SDNode *> &NewNodes) const { |
1288 | return false; |
1289 | } |
1290 | |
1291 | /// Returns the opcode of the would be new |
1292 | /// instruction after load / store are unfolded from an instruction of the |
1293 | /// specified opcode. It returns zero if the specified unfolding is not |
1294 | /// possible. If LoadRegIndex is non-null, it is filled in with the operand |
1295 | /// index of the operand which will hold the register holding the loaded |
1296 | /// value. |
1297 | virtual unsigned |
1298 | getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, |
1299 | unsigned *LoadRegIndex = nullptr) const { |
1300 | return 0; |
1301 | } |
1302 | |
1303 | /// This is used by the pre-regalloc scheduler to determine if two loads are |
1304 | /// loading from the same base address. It should only return true if the base |
1305 | /// pointers are the same and the only differences between the two addresses |
1306 | /// are the offset. It also returns the offsets by reference. |
1307 | virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, |
1308 | int64_t &Offset1, |
1309 | int64_t &Offset2) const { |
1310 | return false; |
1311 | } |
1312 | |
1313 | /// This is a used by the pre-regalloc scheduler to determine (in conjunction |
1314 | /// with areLoadsFromSameBasePtr) if two loads should be scheduled together. |
1315 | /// On some targets if two loads are loading from |
1316 | /// addresses in the same cache line, it's better if they are scheduled |
1317 | /// together. This function takes two integers that represent the load offsets |
1318 | /// from the common base address. It returns true if it decides it's desirable |
1319 | /// to schedule the two loads together. "NumLoads" is the number of loads that |
1320 | /// have already been scheduled after Load1. |
1321 | virtual bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, |
1322 | int64_t Offset1, int64_t Offset2, |
1323 | unsigned NumLoads) const { |
1324 | return false; |
1325 | } |
1326 | |
1327 | /// Get the base operand and byte offset of an instruction that reads/writes |
1328 | /// memory. This is a convenience function for callers that are only prepared |
1329 | /// to handle a single base operand. |
1330 | bool getMemOperandWithOffset(const MachineInstr &MI, |
1331 | const MachineOperand *&BaseOp, int64_t &Offset, |
1332 | bool &OffsetIsScalable, |
1333 | const TargetRegisterInfo *TRI) const; |
1334 | |
1335 | /// Get zero or more base operands and the byte offset of an instruction that |
1336 | /// reads/writes memory. Note that there may be zero base operands if the |
1337 | /// instruction accesses a constant address. |
1338 | /// It returns false if MI does not read/write memory. |
1339 | /// It returns false if base operands and offset could not be determined. |
1340 | /// It is not guaranteed to always recognize base operands and offsets in all |
1341 | /// cases. |
1342 | virtual bool getMemOperandsWithOffsetWidth( |
1343 | const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps, |
1344 | int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, |
1345 | const TargetRegisterInfo *TRI) const { |
1346 | return false; |
1347 | } |
1348 | |
1349 | /// Return true if the instruction contains a base register and offset. If |
1350 | /// true, the function also sets the operand position in the instruction |
1351 | /// for the base register and offset. |
1352 | virtual bool getBaseAndOffsetPosition(const MachineInstr &MI, |
1353 | unsigned &BasePos, |
1354 | unsigned &OffsetPos) const { |
1355 | return false; |
1356 | } |
1357 | |
1358 | /// Target dependent implementation to get the values constituting the address |
1359 | /// MachineInstr that is accessing memory. These values are returned as a |
1360 | /// struct ExtAddrMode which contains all relevant information to make up the |
1361 | /// address. |
1362 | virtual Optional<ExtAddrMode> |
1363 | getAddrModeFromMemoryOp(const MachineInstr &MemI, |
1364 | const TargetRegisterInfo *TRI) const { |
1365 | return None; |
1366 | } |
1367 | |
1368 | /// Returns true if MI's Def is NullValueReg, and the MI |
1369 | /// does not change the Zero value. i.e. cases such as rax = shr rax, X where |
1370 | /// NullValueReg = rax. Note that if the NullValueReg is non-zero, this |
1371 | /// function can return true even if becomes zero. Specifically cases such as |
1372 | /// NullValueReg = shl NullValueReg, 63. |
1373 | virtual bool preservesZeroValueInReg(const MachineInstr *MI, |
1374 | const Register NullValueReg, |
1375 | const TargetRegisterInfo *TRI) const { |
1376 | return false; |
1377 | } |
1378 | |
1379 | /// If the instruction is an increment of a constant value, return the amount. |
1380 | virtual bool getIncrementValue(const MachineInstr &MI, int &Value) const { |
1381 | return false; |
1382 | } |
1383 | |
1384 | /// Returns true if the two given memory operations should be scheduled |
1385 | /// adjacent. Note that you have to add: |
1386 | /// DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); |
1387 | /// or |
1388 | /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); |
1389 | /// to TargetPassConfig::createMachineScheduler() to have an effect. |
1390 | /// |
1391 | /// \p BaseOps1 and \p BaseOps2 are memory operands of two memory operations. |
1392 | /// \p NumLoads is the number of loads that will be in the cluster if this |
1393 | /// hook returns true. |
1394 | /// \p NumBytes is the number of bytes that will be loaded from all the |
1395 | /// clustered loads if this hook returns true. |
1396 | virtual bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, |
1397 | ArrayRef<const MachineOperand *> BaseOps2, |
1398 | unsigned NumLoads, unsigned NumBytes) const { |
1399 | llvm_unreachable("target did not implement shouldClusterMemOps()")::llvm::llvm_unreachable_internal("target did not implement shouldClusterMemOps()" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1399); |
1400 | } |
1401 | |
1402 | /// Reverses the branch condition of the specified condition list, |
1403 | /// returning false on success and true if it cannot be reversed. |
1404 | virtual bool |
1405 | reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { |
1406 | return true; |
1407 | } |
1408 | |
1409 | /// Insert a noop into the instruction stream at the specified point. |
1410 | virtual void insertNoop(MachineBasicBlock &MBB, |
1411 | MachineBasicBlock::iterator MI) const; |
1412 | |
1413 | /// Insert noops into the instruction stream at the specified point. |
1414 | virtual void insertNoops(MachineBasicBlock &MBB, |
1415 | MachineBasicBlock::iterator MI, |
1416 | unsigned Quantity) const; |
1417 | |
1418 | /// Return the noop instruction to use for a noop. |
1419 | virtual MCInst getNop() const; |
1420 | |
1421 | /// Return true for post-incremented instructions. |
1422 | virtual bool isPostIncrement(const MachineInstr &MI) const { return false; } |
1423 | |
1424 | /// Returns true if the instruction is already predicated. |
1425 | virtual bool isPredicated(const MachineInstr &MI) const { return false; } |
1426 | |
1427 | // Returns a MIRPrinter comment for this machine operand. |
1428 | virtual std::string |
1429 | createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, |
1430 | unsigned OpIdx, const TargetRegisterInfo *TRI) const; |
1431 | |
1432 | /// Returns true if the instruction is a |
1433 | /// terminator instruction that has not been predicated. |
1434 | bool isUnpredicatedTerminator(const MachineInstr &MI) const; |
1435 | |
1436 | /// Returns true if MI is an unconditional tail call. |
1437 | virtual bool isUnconditionalTailCall(const MachineInstr &MI) const { |
1438 | return false; |
1439 | } |
1440 | |
1441 | /// Returns true if the tail call can be made conditional on BranchCond. |
1442 | virtual bool canMakeTailCallConditional(SmallVectorImpl<MachineOperand> &Cond, |
1443 | const MachineInstr &TailCall) const { |
1444 | return false; |
1445 | } |
1446 | |
1447 | /// Replace the conditional branch in MBB with a conditional tail call. |
1448 | virtual void replaceBranchWithTailCall(MachineBasicBlock &MBB, |
1449 | SmallVectorImpl<MachineOperand> &Cond, |
1450 | const MachineInstr &TailCall) const { |
1451 | llvm_unreachable("Target didn't implement replaceBranchWithTailCall!")::llvm::llvm_unreachable_internal("Target didn't implement replaceBranchWithTailCall!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1451); |
1452 | } |
1453 | |
1454 | /// Convert the instruction into a predicated instruction. |
1455 | /// It returns true if the operation was successful. |
1456 | virtual bool PredicateInstruction(MachineInstr &MI, |
1457 | ArrayRef<MachineOperand> Pred) const; |
1458 | |
1459 | /// Returns true if the first specified predicate |
1460 | /// subsumes the second, e.g. GE subsumes GT. |
1461 | virtual bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1, |
1462 | ArrayRef<MachineOperand> Pred2) const { |
1463 | return false; |
1464 | } |
1465 | |
1466 | /// If the specified instruction defines any predicate |
1467 | /// or condition code register(s) used for predication, returns true as well |
1468 | /// as the definition predicate(s) by reference. |
1469 | /// SkipDead should be set to false at any point that dead |
1470 | /// predicate instructions should be considered as being defined. |
1471 | /// A dead predicate instruction is one that is guaranteed to be removed |
1472 | /// after a call to PredicateInstruction. |
1473 | virtual bool ClobbersPredicate(MachineInstr &MI, |
1474 | std::vector<MachineOperand> &Pred, |
1475 | bool SkipDead) const { |
1476 | return false; |
1477 | } |
1478 | |
1479 | /// Return true if the specified instruction can be predicated. |
1480 | /// By default, this returns true for every instruction with a |
1481 | /// PredicateOperand. |
1482 | virtual bool isPredicable(const MachineInstr &MI) const { |
1483 | return MI.getDesc().isPredicable(); |
1484 | } |
1485 | |
1486 | /// Return true if it's safe to move a machine |
1487 | /// instruction that defines the specified register class. |
1488 | virtual bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { |
1489 | return true; |
1490 | } |
1491 | |
1492 | /// Test if the given instruction should be considered a scheduling boundary. |
1493 | /// This primarily includes labels and terminators. |
1494 | virtual bool isSchedulingBoundary(const MachineInstr &MI, |
1495 | const MachineBasicBlock *MBB, |
1496 | const MachineFunction &MF) const; |
1497 | |
1498 | /// Measure the specified inline asm to determine an approximation of its |
1499 | /// length. |
1500 | virtual unsigned getInlineAsmLength( |
1501 | const char *Str, const MCAsmInfo &MAI, |
1502 | const TargetSubtargetInfo *STI = nullptr) const; |
1503 | |
1504 | /// Allocate and return a hazard recognizer to use for this target when |
1505 | /// scheduling the machine instructions before register allocation. |
1506 | virtual ScheduleHazardRecognizer * |
1507 | CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, |
1508 | const ScheduleDAG *DAG) const; |
1509 | |
1510 | /// Allocate and return a hazard recognizer to use for this target when |
1511 | /// scheduling the machine instructions before register allocation. |
1512 | virtual ScheduleHazardRecognizer * |
1513 | CreateTargetMIHazardRecognizer(const InstrItineraryData *, |
1514 | const ScheduleDAGMI *DAG) const; |
1515 | |
1516 | /// Allocate and return a hazard recognizer to use for this target when |
1517 | /// scheduling the machine instructions after register allocation. |
1518 | virtual ScheduleHazardRecognizer * |
1519 | CreateTargetPostRAHazardRecognizer(const InstrItineraryData *, |
1520 | const ScheduleDAG *DAG) const; |
1521 | |
1522 | /// Allocate and return a hazard recognizer to use for by non-scheduling |
1523 | /// passes. |
1524 | virtual ScheduleHazardRecognizer * |
1525 | CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { |
1526 | return nullptr; |
1527 | } |
1528 | |
1529 | /// Provide a global flag for disabling the PreRA hazard recognizer that |
1530 | /// targets may choose to honor. |
1531 | bool usePreRAHazardRecognizer() const; |
1532 | |
1533 | /// For a comparison instruction, return the source registers |
1534 | /// in SrcReg and SrcReg2 if having two register operands, and the value it |
1535 | /// compares against in CmpValue. Return true if the comparison instruction |
1536 | /// can be analyzed. |
1537 | virtual bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, |
1538 | Register &SrcReg2, int64_t &Mask, |
1539 | int64_t &Value) const { |
1540 | return false; |
1541 | } |
1542 | |
1543 | /// See if the comparison instruction can be converted |
1544 | /// into something more efficient. E.g., on ARM most instructions can set the |
1545 | /// flags register, obviating the need for a separate CMP. |
1546 | virtual bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, |
1547 | Register SrcReg2, int64_t Mask, |
1548 | int64_t Value, |
1549 | const MachineRegisterInfo *MRI) const { |
1550 | return false; |
1551 | } |
1552 | virtual bool optimizeCondBranch(MachineInstr &MI) const { return false; } |
1553 | |
1554 | /// Try to remove the load by folding it to a register operand at the use. |
1555 | /// We fold the load instructions if and only if the |
1556 | /// def and use are in the same BB. We only look at one load and see |
1557 | /// whether it can be folded into MI. FoldAsLoadDefReg is the virtual register |
1558 | /// defined by the load we are trying to fold. DefMI returns the machine |
1559 | /// instruction that defines FoldAsLoadDefReg, and the function returns |
1560 | /// the machine instruction generated due to folding. |
1561 | virtual MachineInstr *optimizeLoadInstr(MachineInstr &MI, |
1562 | const MachineRegisterInfo *MRI, |
1563 | Register &FoldAsLoadDefReg, |
1564 | MachineInstr *&DefMI) const { |
1565 | return nullptr; |
1566 | } |
1567 | |
1568 | /// 'Reg' is known to be defined by a move immediate instruction, |
1569 | /// try to fold the immediate into the use instruction. |
1570 | /// If MRI->hasOneNonDBGUse(Reg) is true, and this function returns true, |
1571 | /// then the caller may assume that DefMI has been erased from its parent |
1572 | /// block. The caller may assume that it will not be erased by this |
1573 | /// function otherwise. |
1574 | virtual bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, |
1575 | Register Reg, MachineRegisterInfo *MRI) const { |
1576 | return false; |
1577 | } |
1578 | |
1579 | /// Return the number of u-operations the given machine |
1580 | /// instruction will be decoded to on the target cpu. The itinerary's |
1581 | /// IssueWidth is the number of microops that can be dispatched each |
1582 | /// cycle. An instruction with zero microops takes no dispatch resources. |
1583 | virtual unsigned getNumMicroOps(const InstrItineraryData *ItinData, |
1584 | const MachineInstr &MI) const; |
1585 | |
1586 | /// Return true for pseudo instructions that don't consume any |
1587 | /// machine resources in their current form. These are common cases that the |
1588 | /// scheduler should consider free, rather than conservatively handling them |
1589 | /// as instructions with no itinerary. |
1590 | bool isZeroCost(unsigned Opcode) const { |
1591 | return Opcode <= TargetOpcode::COPY; |
1592 | } |
1593 | |
1594 | virtual int getOperandLatency(const InstrItineraryData *ItinData, |
1595 | SDNode *DefNode, unsigned DefIdx, |
1596 | SDNode *UseNode, unsigned UseIdx) const; |
1597 | |
1598 | /// Compute and return the use operand latency of a given pair of def and use. |
1599 | /// In most cases, the static scheduling itinerary was enough to determine the |
1600 | /// operand latency. But it may not be possible for instructions with variable |
1601 | /// number of defs / uses. |
1602 | /// |
1603 | /// This is a raw interface to the itinerary that may be directly overridden |
1604 | /// by a target. Use computeOperandLatency to get the best estimate of |
1605 | /// latency. |
1606 | virtual int getOperandLatency(const InstrItineraryData *ItinData, |
1607 | const MachineInstr &DefMI, unsigned DefIdx, |
1608 | const MachineInstr &UseMI, |
1609 | unsigned UseIdx) const; |
1610 | |
1611 | /// Compute the instruction latency of a given instruction. |
1612 | /// If the instruction has higher cost when predicated, it's returned via |
1613 | /// PredCost. |
1614 | virtual unsigned getInstrLatency(const InstrItineraryData *ItinData, |
1615 | const MachineInstr &MI, |
1616 | unsigned *PredCost = nullptr) const; |
1617 | |
1618 | virtual unsigned getPredicationCost(const MachineInstr &MI) const; |
1619 | |
1620 | virtual int getInstrLatency(const InstrItineraryData *ItinData, |
1621 | SDNode *Node) const; |
1622 | |
1623 | /// Return the default expected latency for a def based on its opcode. |
1624 | unsigned defaultDefLatency(const MCSchedModel &SchedModel, |
1625 | const MachineInstr &DefMI) const; |
1626 | |
1627 | /// Return true if this opcode has high latency to its result. |
1628 | virtual bool isHighLatencyDef(int opc) const { return false; } |
1629 | |
1630 | /// Compute operand latency between a def of 'Reg' |
1631 | /// and a use in the current loop. Return true if the target considered |
1632 | /// it 'high'. This is used by optimization passes such as machine LICM to |
1633 | /// determine whether it makes sense to hoist an instruction out even in a |
1634 | /// high register pressure situation. |
1635 | virtual bool hasHighOperandLatency(const TargetSchedModel &SchedModel, |
1636 | const MachineRegisterInfo *MRI, |
1637 | const MachineInstr &DefMI, unsigned DefIdx, |
1638 | const MachineInstr &UseMI, |
1639 | unsigned UseIdx) const { |
1640 | return false; |
1641 | } |
1642 | |
1643 | /// Compute operand latency of a def of 'Reg'. Return true |
1644 | /// if the target considered it 'low'. |
1645 | virtual bool hasLowDefLatency(const TargetSchedModel &SchedModel, |
1646 | const MachineInstr &DefMI, |
1647 | unsigned DefIdx) const; |
1648 | |
1649 | /// Perform target-specific instruction verification. |
1650 | virtual bool verifyInstruction(const MachineInstr &MI, |
1651 | StringRef &ErrInfo) const { |
1652 | return true; |
1653 | } |
1654 | |
1655 | /// Return the current execution domain and bit mask of |
1656 | /// possible domains for instruction. |
1657 | /// |
1658 | /// Some micro-architectures have multiple execution domains, and multiple |
1659 | /// opcodes that perform the same operation in different domains. For |
1660 | /// example, the x86 architecture provides the por, orps, and orpd |
1661 | /// instructions that all do the same thing. There is a latency penalty if a |
1662 | /// register is written in one domain and read in another. |
1663 | /// |
1664 | /// This function returns a pair (domain, mask) containing the execution |
1665 | /// domain of MI, and a bit mask of possible domains. The setExecutionDomain |
1666 | /// function can be used to change the opcode to one of the domains in the |
1667 | /// bit mask. Instructions whose execution domain can't be changed should |
1668 | /// return a 0 mask. |
1669 | /// |
1670 | /// The execution domain numbers don't have any special meaning except domain |
1671 | /// 0 is used for instructions that are not associated with any interesting |
1672 | /// execution domain. |
1673 | /// |
1674 | virtual std::pair<uint16_t, uint16_t> |
1675 | getExecutionDomain(const MachineInstr &MI) const { |
1676 | return std::make_pair(0, 0); |
1677 | } |
1678 | |
1679 | /// Change the opcode of MI to execute in Domain. |
1680 | /// |
1681 | /// The bit (1 << Domain) must be set in the mask returned from |
1682 | /// getExecutionDomain(MI). |
1683 | virtual void setExecutionDomain(MachineInstr &MI, unsigned Domain) const {} |
1684 | |
1685 | /// Returns the preferred minimum clearance |
1686 | /// before an instruction with an unwanted partial register update. |
1687 | /// |
1688 | /// Some instructions only write part of a register, and implicitly need to |
1689 | /// read the other parts of the register. This may cause unwanted stalls |
1690 | /// preventing otherwise unrelated instructions from executing in parallel in |
1691 | /// an out-of-order CPU. |
1692 | /// |
1693 | /// For example, the x86 instruction cvtsi2ss writes its result to bits |
1694 | /// [31:0] of the destination xmm register. Bits [127:32] are unaffected, so |
1695 | /// the instruction needs to wait for the old value of the register to become |
1696 | /// available: |
1697 | /// |
1698 | /// addps %xmm1, %xmm0 |
1699 | /// movaps %xmm0, (%rax) |
1700 | /// cvtsi2ss %rbx, %xmm0 |
1701 | /// |
1702 | /// In the code above, the cvtsi2ss instruction needs to wait for the addps |
1703 | /// instruction before it can issue, even though the high bits of %xmm0 |
1704 | /// probably aren't needed. |
1705 | /// |
1706 | /// This hook returns the preferred clearance before MI, measured in |
1707 | /// instructions. Other defs of MI's operand OpNum are avoided in the last N |
1708 | /// instructions before MI. It should only return a positive value for |
1709 | /// unwanted dependencies. If the old bits of the defined register have |
1710 | /// useful values, or if MI is determined to otherwise read the dependency, |
1711 | /// the hook should return 0. |
1712 | /// |
1713 | /// The unwanted dependency may be handled by: |
1714 | /// |
1715 | /// 1. Allocating the same register for an MI def and use. That makes the |
1716 | /// unwanted dependency identical to a required dependency. |
1717 | /// |
1718 | /// 2. Allocating a register for the def that has no defs in the previous N |
1719 | /// instructions. |
1720 | /// |
1721 | /// 3. Calling breakPartialRegDependency() with the same arguments. This |
1722 | /// allows the target to insert a dependency breaking instruction. |
1723 | /// |
1724 | virtual unsigned |
1725 | getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum, |
1726 | const TargetRegisterInfo *TRI) const { |
1727 | // The default implementation returns 0 for no partial register dependency. |
1728 | return 0; |
1729 | } |
1730 | |
1731 | /// Return the minimum clearance before an instruction that reads an |
1732 | /// unused register. |
1733 | /// |
1734 | /// For example, AVX instructions may copy part of a register operand into |
1735 | /// the unused high bits of the destination register. |
1736 | /// |
1737 | /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 |
1738 | /// |
1739 | /// In the code above, vcvtsi2sdq copies %xmm0[127:64] into %xmm14 creating a |
1740 | /// false dependence on any previous write to %xmm0. |
1741 | /// |
1742 | /// This hook works similarly to getPartialRegUpdateClearance, except that it |
1743 | /// does not take an operand index. Instead sets \p OpNum to the index of the |
1744 | /// unused register. |
1745 | virtual unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum, |
1746 | const TargetRegisterInfo *TRI) const { |
1747 | // The default implementation returns 0 for no undef register dependency. |
1748 | return 0; |
1749 | } |
1750 | |
1751 | /// Insert a dependency-breaking instruction |
1752 | /// before MI to eliminate an unwanted dependency on OpNum. |
1753 | /// |
1754 | /// If it wasn't possible to avoid a def in the last N instructions before MI |
1755 | /// (see getPartialRegUpdateClearance), this hook will be called to break the |
1756 | /// unwanted dependency. |
1757 | /// |
1758 | /// On x86, an xorps instruction can be used as a dependency breaker: |
1759 | /// |
1760 | /// addps %xmm1, %xmm0 |
1761 | /// movaps %xmm0, (%rax) |
1762 | /// xorps %xmm0, %xmm0 |
1763 | /// cvtsi2ss %rbx, %xmm0 |
1764 | /// |
1765 | /// An <imp-kill> operand should be added to MI if an instruction was |
1766 | /// inserted. This ties the instructions together in the post-ra scheduler. |
1767 | /// |
1768 | virtual void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum, |
1769 | const TargetRegisterInfo *TRI) const {} |
1770 | |
1771 | /// Create machine specific model for scheduling. |
1772 | virtual DFAPacketizer * |
1773 | CreateTargetScheduleState(const TargetSubtargetInfo &) const { |
1774 | return nullptr; |
1775 | } |
1776 | |
1777 | /// Sometimes, it is possible for the target |
1778 | /// to tell, even without aliasing information, that two MIs access different |
1779 | /// memory addresses. This function returns true if two MIs access different |
1780 | /// memory addresses and false otherwise. |
1781 | /// |
1782 | /// Assumes any physical registers used to compute addresses have the same |
1783 | /// value for both instructions. (This is the most useful assumption for |
1784 | /// post-RA scheduling.) |
1785 | /// |
1786 | /// See also MachineInstr::mayAlias, which is implemented on top of this |
1787 | /// function. |
1788 | virtual bool |
1789 | areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, |
1790 | const MachineInstr &MIb) const { |
1791 | assert(MIa.mayLoadOrStore() &&(static_cast <bool> (MIa.mayLoadOrStore() && "MIa must load from or modify a memory location" ) ? void (0) : __assert_fail ("MIa.mayLoadOrStore() && \"MIa must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1792, __extension__ __PRETTY_FUNCTION__)) |
1792 | "MIa must load from or modify a memory location")(static_cast <bool> (MIa.mayLoadOrStore() && "MIa must load from or modify a memory location" ) ? void (0) : __assert_fail ("MIa.mayLoadOrStore() && \"MIa must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1792, __extension__ __PRETTY_FUNCTION__)); |
1793 | assert(MIb.mayLoadOrStore() &&(static_cast <bool> (MIb.mayLoadOrStore() && "MIb must load from or modify a memory location" ) ? void (0) : __assert_fail ("MIb.mayLoadOrStore() && \"MIb must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1794, __extension__ __PRETTY_FUNCTION__)) |
1794 | "MIb must load from or modify a memory location")(static_cast <bool> (MIb.mayLoadOrStore() && "MIb must load from or modify a memory location" ) ? void (0) : __assert_fail ("MIb.mayLoadOrStore() && \"MIb must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1794, __extension__ __PRETTY_FUNCTION__)); |
1795 | return false; |
1796 | } |
1797 | |
1798 | /// Return the value to use for the MachineCSE's LookAheadLimit, |
1799 | /// which is a heuristic used for CSE'ing phys reg defs. |
1800 | virtual unsigned getMachineCSELookAheadLimit() const { |
1801 | // The default lookahead is small to prevent unprofitable quadratic |
1802 | // behavior. |
1803 | return 5; |
1804 | } |
1805 | |
1806 | /// Return the maximal number of alias checks on memory operands. For |
1807 | /// instructions with more than one memory operands, the alias check on a |
1808 | /// single MachineInstr pair has quadratic overhead and results in |
1809 | /// unacceptable performance in the worst case. The limit here is to clamp |
1810 | /// that maximal checks performed. Usually, that's the product of memory |
1811 | /// operand numbers from that pair of MachineInstr to be checked. For |
1812 | /// instance, with two MachineInstrs with 4 and 5 memory operands |
1813 | /// correspondingly, a total of 20 checks are required. With this limit set to |
1814 | /// 16, their alias check is skipped. We choose to limit the product instead |
1815 | /// of the individual instruction as targets may have special MachineInstrs |
1816 | /// with a considerably high number of memory operands, such as `ldm` in ARM. |
1817 | /// Setting this limit per MachineInstr would result in either too high |
1818 | /// overhead or too rigid restriction. |
1819 | virtual unsigned getMemOperandAACheckLimit() const { return 16; } |
1820 | |
1821 | /// Return an array that contains the ids of the target indices (used for the |
1822 | /// TargetIndex machine operand) and their names. |
1823 | /// |
1824 | /// MIR Serialization is able to serialize only the target indices that are |
1825 | /// defined by this method. |
1826 | virtual ArrayRef<std::pair<int, const char *>> |
1827 | getSerializableTargetIndices() const { |
1828 | return None; |
1829 | } |
1830 | |
1831 | /// Decompose the machine operand's target flags into two values - the direct |
1832 | /// target flag value and any of bit flags that are applied. |
1833 | virtual std::pair<unsigned, unsigned> |
1834 | decomposeMachineOperandsTargetFlags(unsigned /*TF*/) const { |
1835 | return std::make_pair(0u, 0u); |
1836 | } |
1837 | |
1838 | /// Return an array that contains the direct target flag values and their |
1839 | /// names. |
1840 | /// |
1841 | /// MIR Serialization is able to serialize only the target flags that are |
1842 | /// defined by this method. |
1843 | virtual ArrayRef<std::pair<unsigned, const char *>> |
1844 | getSerializableDirectMachineOperandTargetFlags() const { |
1845 | return None; |
1846 | } |
1847 | |
1848 | /// Return an array that contains the bitmask target flag values and their |
1849 | /// names. |
1850 | /// |
1851 | /// MIR Serialization is able to serialize only the target flags that are |
1852 | /// defined by this method. |
1853 | virtual ArrayRef<std::pair<unsigned, const char *>> |
1854 | getSerializableBitmaskMachineOperandTargetFlags() const { |
1855 | return None; |
1856 | } |
1857 | |
1858 | /// Return an array that contains the MMO target flag values and their |
1859 | /// names. |
1860 | /// |
1861 | /// MIR Serialization is able to serialize only the MMO target flags that are |
1862 | /// defined by this method. |
1863 | virtual ArrayRef<std::pair<MachineMemOperand::Flags, const char *>> |
1864 | getSerializableMachineMemOperandTargetFlags() const { |
1865 | return None; |
1866 | } |
1867 | |
1868 | /// Determines whether \p Inst is a tail call instruction. Override this |
1869 | /// method on targets that do not properly set MCID::Return and MCID::Call on |
1870 | /// tail call instructions." |
1871 | virtual bool isTailCall(const MachineInstr &Inst) const { |
1872 | return Inst.isReturn() && Inst.isCall(); |
1873 | } |
1874 | |
1875 | /// True if the instruction is bound to the top of its basic block and no |
1876 | /// other instructions shall be inserted before it. This can be implemented |
1877 | /// to prevent register allocator to insert spills before such instructions. |
1878 | virtual bool isBasicBlockPrologue(const MachineInstr &MI) const { |
1879 | return false; |
1880 | } |
1881 | |
1882 | /// During PHI eleimination lets target to make necessary checks and |
1883 | /// insert the copy to the PHI destination register in a target specific |
1884 | /// manner. |
1885 | virtual MachineInstr *createPHIDestinationCopy( |
1886 | MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, |
1887 | const DebugLoc &DL, Register Src, Register Dst) const { |
1888 | return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst) |
1889 | .addReg(Src); |
1890 | } |
1891 | |
1892 | /// During PHI eleimination lets target to make necessary checks and |
1893 | /// insert the copy to the PHI destination register in a target specific |
1894 | /// manner. |
1895 | virtual MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB, |
1896 | MachineBasicBlock::iterator InsPt, |
1897 | const DebugLoc &DL, Register Src, |
1898 | unsigned SrcSubReg, |
1899 | Register Dst) const { |
1900 | return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst) |
1901 | .addReg(Src, 0, SrcSubReg); |
1902 | } |
1903 | |
1904 | /// Returns a \p outliner::OutlinedFunction struct containing target-specific |
1905 | /// information for a set of outlining candidates. |
1906 | virtual outliner::OutlinedFunction getOutliningCandidateInfo( |
1907 | std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { |
1908 | llvm_unreachable(::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::getOutliningCandidateInfo!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1909) |
1909 | "Target didn't implement TargetInstrInfo::getOutliningCandidateInfo!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::getOutliningCandidateInfo!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1909); |
1910 | } |
1911 | |
1912 | /// Returns how or if \p MI should be outlined. |
1913 | virtual outliner::InstrType |
1914 | getOutliningType(MachineBasicBlock::iterator &MIT, unsigned Flags) const { |
1915 | llvm_unreachable(::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::getOutliningType!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1916) |
1916 | "Target didn't implement TargetInstrInfo::getOutliningType!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::getOutliningType!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1916); |
1917 | } |
1918 | |
1919 | /// Optional target hook that returns true if \p MBB is safe to outline from, |
1920 | /// and returns any target-specific information in \p Flags. |
1921 | virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, |
1922 | unsigned &Flags) const { |
1923 | return true; |
1924 | } |
1925 | |
1926 | /// Insert a custom frame for outlined functions. |
1927 | virtual void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, |
1928 | const outliner::OutlinedFunction &OF) const { |
1929 | llvm_unreachable(::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::buildOutlinedFrame!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1930) |
1930 | "Target didn't implement TargetInstrInfo::buildOutlinedFrame!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::buildOutlinedFrame!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1930); |
1931 | } |
1932 | |
1933 | /// Insert a call to an outlined function into the program. |
1934 | /// Returns an iterator to the spot where we inserted the call. This must be |
1935 | /// implemented by the target. |
1936 | virtual MachineBasicBlock::iterator |
1937 | insertOutlinedCall(Module &M, MachineBasicBlock &MBB, |
1938 | MachineBasicBlock::iterator &It, MachineFunction &MF, |
1939 | const outliner::Candidate &C) const { |
1940 | llvm_unreachable(::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::insertOutlinedCall!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1941) |
1941 | "Target didn't implement TargetInstrInfo::insertOutlinedCall!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::insertOutlinedCall!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1941); |
1942 | } |
1943 | |
1944 | /// Return true if the function can safely be outlined from. |
1945 | /// A function \p MF is considered safe for outlining if an outlined function |
1946 | /// produced from instructions in F will produce a program which produces the |
1947 | /// same output for any set of given inputs. |
1948 | virtual bool isFunctionSafeToOutlineFrom(MachineFunction &MF, |
1949 | bool OutlineFromLinkOnceODRs) const { |
1950 | llvm_unreachable("Target didn't implement "::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::isFunctionSafeToOutlineFrom!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1951) |
1951 | "TargetInstrInfo::isFunctionSafeToOutlineFrom!")::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::isFunctionSafeToOutlineFrom!" , "/build/llvm-toolchain-snapshot-14~++20211108111423+c2b91eef275d/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1951); |
1952 | } |
1953 | |
1954 | /// Return true if the function should be outlined from by default. |
1955 | virtual bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const { |
1956 | return false; |
1957 | } |
1958 | |
1959 | /// Produce the expression describing the \p MI loading a value into |
1960 | /// the physical register \p Reg. This hook should only be used with |
1961 | /// \p MIs belonging to VReg-less functions. |
1962 | virtual Optional<ParamLoadedValue> describeLoadedValue(const MachineInstr &MI, |
1963 | Register Reg) const; |
1964 | |
1965 | /// Given the generic extension instruction \p ExtMI, returns true if this |
1966 | /// extension is a likely candidate for being folded into an another |
1967 | /// instruction. |
1968 | virtual bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, |
1969 | MachineRegisterInfo &MRI) const { |
1970 | return false; |
1971 | } |
1972 | |
1973 | /// Return MIR formatter to format/parse MIR operands. Target can override |
1974 | /// this virtual function and return target specific MIR formatter. |
1975 | virtual const MIRFormatter *getMIRFormatter() const { |
1976 | if (!Formatter.get()) |
1977 | Formatter = std::make_unique<MIRFormatter>(); |
1978 | return Formatter.get(); |
1979 | } |
1980 | |
1981 | /// Returns the target-specific default value for tail duplication. |
1982 | /// This value will be used if the tail-dup-placement-threshold argument is |
1983 | /// not provided. |
1984 | virtual unsigned getTailDuplicateSize(CodeGenOpt::Level OptLevel) const { |
1985 | return OptLevel >= CodeGenOpt::Aggressive ? 4 : 2; |
1986 | } |
1987 | |
1988 | /// Returns the callee operand from the given \p MI. |
1989 | virtual const MachineOperand &getCalleeOperand(const MachineInstr &MI) const { |
1990 | return MI.getOperand(0); |
1991 | } |
1992 | |
1993 | private: |
1994 | mutable std::unique_ptr<MIRFormatter> Formatter; |
1995 | unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode; |
1996 | unsigned CatchRetOpcode; |
1997 | unsigned ReturnOpcode; |
1998 | }; |
1999 | |
2000 | /// Provide DenseMapInfo for TargetInstrInfo::RegSubRegPair. |
2001 | template <> struct DenseMapInfo<TargetInstrInfo::RegSubRegPair> { |
2002 | using RegInfo = DenseMapInfo<unsigned>; |
2003 | |
2004 | static inline TargetInstrInfo::RegSubRegPair getEmptyKey() { |
2005 | return TargetInstrInfo::RegSubRegPair(RegInfo::getEmptyKey(), |
2006 | RegInfo::getEmptyKey()); |
2007 | } |
2008 | |
2009 | static inline TargetInstrInfo::RegSubRegPair getTombstoneKey() { |
2010 | return TargetInstrInfo::RegSubRegPair(RegInfo::getTombstoneKey(), |
2011 | RegInfo::getTombstoneKey()); |
2012 | } |
2013 | |
2014 | /// Reuse getHashValue implementation from |
2015 | /// std::pair<unsigned, unsigned>. |
2016 | static unsigned getHashValue(const TargetInstrInfo::RegSubRegPair &Val) { |
2017 | std::pair<unsigned, unsigned> PairVal = std::make_pair(Val.Reg, Val.SubReg); |
2018 | return DenseMapInfo<std::pair<unsigned, unsigned>>::getHashValue(PairVal); |
2019 | } |
2020 | |
2021 | static bool isEqual(const TargetInstrInfo::RegSubRegPair &LHS, |
2022 | const TargetInstrInfo::RegSubRegPair &RHS) { |
2023 | return RegInfo::isEqual(LHS.Reg, RHS.Reg) && |
2024 | RegInfo::isEqual(LHS.SubReg, RHS.SubReg); |
2025 | } |
2026 | }; |
2027 | |
2028 | } // end namespace llvm |
2029 | |
2030 | #endif // LLVM_CODEGEN_TARGETINSTRINFO_H |