File: | llvm/lib/CodeGen/RegisterCoalescer.cpp |
Warning: | line 2551, column 11 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- RegisterCoalescer.cpp - Generic Register Coalescing Interface ------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the generic RegisterCoalescer interface which | |||
10 | // is used as the common interface used by all clients and | |||
11 | // implementations of register coalescing. | |||
12 | // | |||
13 | //===----------------------------------------------------------------------===// | |||
14 | ||||
15 | #include "RegisterCoalescer.h" | |||
16 | #include "llvm/ADT/ArrayRef.h" | |||
17 | #include "llvm/ADT/BitVector.h" | |||
18 | #include "llvm/ADT/DenseSet.h" | |||
19 | #include "llvm/ADT/STLExtras.h" | |||
20 | #include "llvm/ADT/SmallPtrSet.h" | |||
21 | #include "llvm/ADT/SmallVector.h" | |||
22 | #include "llvm/ADT/Statistic.h" | |||
23 | #include "llvm/Analysis/AliasAnalysis.h" | |||
24 | #include "llvm/CodeGen/LiveInterval.h" | |||
25 | #include "llvm/CodeGen/LiveIntervals.h" | |||
26 | #include "llvm/CodeGen/LiveRangeEdit.h" | |||
27 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
28 | #include "llvm/CodeGen/MachineFunction.h" | |||
29 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
30 | #include "llvm/CodeGen/MachineInstr.h" | |||
31 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
32 | #include "llvm/CodeGen/MachineLoopInfo.h" | |||
33 | #include "llvm/CodeGen/MachineOperand.h" | |||
34 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
35 | #include "llvm/CodeGen/Passes.h" | |||
36 | #include "llvm/CodeGen/RegisterClassInfo.h" | |||
37 | #include "llvm/CodeGen/SlotIndexes.h" | |||
38 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
39 | #include "llvm/CodeGen/TargetOpcodes.h" | |||
40 | #include "llvm/CodeGen/TargetRegisterInfo.h" | |||
41 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
42 | #include "llvm/IR/DebugLoc.h" | |||
43 | #include "llvm/InitializePasses.h" | |||
44 | #include "llvm/MC/LaneBitmask.h" | |||
45 | #include "llvm/MC/MCInstrDesc.h" | |||
46 | #include "llvm/MC/MCRegisterInfo.h" | |||
47 | #include "llvm/Pass.h" | |||
48 | #include "llvm/Support/CommandLine.h" | |||
49 | #include "llvm/Support/Compiler.h" | |||
50 | #include "llvm/Support/Debug.h" | |||
51 | #include "llvm/Support/ErrorHandling.h" | |||
52 | #include "llvm/Support/raw_ostream.h" | |||
53 | #include <algorithm> | |||
54 | #include <cassert> | |||
55 | #include <iterator> | |||
56 | #include <limits> | |||
57 | #include <tuple> | |||
58 | #include <utility> | |||
59 | #include <vector> | |||
60 | ||||
61 | using namespace llvm; | |||
62 | ||||
63 | #define DEBUG_TYPE"regalloc" "regalloc" | |||
64 | ||||
65 | STATISTIC(numJoins , "Number of interval joins performed")static llvm::Statistic numJoins = {"regalloc", "numJoins", "Number of interval joins performed" }; | |||
66 | STATISTIC(numCrossRCs , "Number of cross class joins performed")static llvm::Statistic numCrossRCs = {"regalloc", "numCrossRCs" , "Number of cross class joins performed"}; | |||
67 | STATISTIC(numCommutes , "Number of instruction commuting performed")static llvm::Statistic numCommutes = {"regalloc", "numCommutes" , "Number of instruction commuting performed"}; | |||
68 | STATISTIC(numExtends , "Number of copies extended")static llvm::Statistic numExtends = {"regalloc", "numExtends" , "Number of copies extended"}; | |||
69 | STATISTIC(NumReMats , "Number of instructions re-materialized")static llvm::Statistic NumReMats = {"regalloc", "NumReMats", "Number of instructions re-materialized" }; | |||
70 | STATISTIC(NumInflated , "Number of register classes inflated")static llvm::Statistic NumInflated = {"regalloc", "NumInflated" , "Number of register classes inflated"}; | |||
71 | STATISTIC(NumLaneConflicts, "Number of dead lane conflicts tested")static llvm::Statistic NumLaneConflicts = {"regalloc", "NumLaneConflicts" , "Number of dead lane conflicts tested"}; | |||
72 | STATISTIC(NumLaneResolves, "Number of dead lane conflicts resolved")static llvm::Statistic NumLaneResolves = {"regalloc", "NumLaneResolves" , "Number of dead lane conflicts resolved"}; | |||
73 | STATISTIC(NumShrinkToUses, "Number of shrinkToUses called")static llvm::Statistic NumShrinkToUses = {"regalloc", "NumShrinkToUses" , "Number of shrinkToUses called"}; | |||
74 | ||||
75 | static cl::opt<bool> EnableJoining("join-liveintervals", | |||
76 | cl::desc("Coalesce copies (default=true)"), | |||
77 | cl::init(true), cl::Hidden); | |||
78 | ||||
79 | static cl::opt<bool> UseTerminalRule("terminal-rule", | |||
80 | cl::desc("Apply the terminal rule"), | |||
81 | cl::init(false), cl::Hidden); | |||
82 | ||||
83 | /// Temporary flag to test critical edge unsplitting. | |||
84 | static cl::opt<bool> | |||
85 | EnableJoinSplits("join-splitedges", | |||
86 | cl::desc("Coalesce copies on split edges (default=subtarget)"), cl::Hidden); | |||
87 | ||||
88 | /// Temporary flag to test global copy optimization. | |||
89 | static cl::opt<cl::boolOrDefault> | |||
90 | EnableGlobalCopies("join-globalcopies", | |||
91 | cl::desc("Coalesce copies that span blocks (default=subtarget)"), | |||
92 | cl::init(cl::BOU_UNSET), cl::Hidden); | |||
93 | ||||
94 | static cl::opt<bool> | |||
95 | VerifyCoalescing("verify-coalescing", | |||
96 | cl::desc("Verify machine instrs before and after register coalescing"), | |||
97 | cl::Hidden); | |||
98 | ||||
99 | static cl::opt<unsigned> LateRematUpdateThreshold( | |||
100 | "late-remat-update-threshold", cl::Hidden, | |||
101 | cl::desc("During rematerialization for a copy, if the def instruction has " | |||
102 | "many other copy uses to be rematerialized, delay the multiple " | |||
103 | "separate live interval update work and do them all at once after " | |||
104 | "all those rematerialization are done. It will save a lot of " | |||
105 | "repeated work. "), | |||
106 | cl::init(100)); | |||
107 | ||||
108 | static cl::opt<unsigned> LargeIntervalSizeThreshold( | |||
109 | "large-interval-size-threshold", cl::Hidden, | |||
110 | cl::desc("If the valnos size of an interval is larger than the threshold, " | |||
111 | "it is regarded as a large interval. "), | |||
112 | cl::init(100)); | |||
113 | ||||
114 | static cl::opt<unsigned> LargeIntervalFreqThreshold( | |||
115 | "large-interval-freq-threshold", cl::Hidden, | |||
116 | cl::desc("For a large interval, if it is coalesed with other live " | |||
117 | "intervals many times more than the threshold, stop its " | |||
118 | "coalescing to control the compile time. "), | |||
119 | cl::init(100)); | |||
120 | ||||
121 | namespace { | |||
122 | ||||
123 | class JoinVals; | |||
124 | ||||
125 | class RegisterCoalescer : public MachineFunctionPass, | |||
126 | private LiveRangeEdit::Delegate { | |||
127 | MachineFunction* MF = nullptr; | |||
128 | MachineRegisterInfo* MRI = nullptr; | |||
129 | const TargetRegisterInfo* TRI = nullptr; | |||
130 | const TargetInstrInfo* TII = nullptr; | |||
131 | LiveIntervals *LIS = nullptr; | |||
132 | const MachineLoopInfo* Loops = nullptr; | |||
133 | AliasAnalysis *AA = nullptr; | |||
134 | RegisterClassInfo RegClassInfo; | |||
135 | ||||
136 | /// Debug variable location tracking -- for each VReg, maintain an | |||
137 | /// ordered-by-slot-index set of DBG_VALUEs, to help quick | |||
138 | /// identification of whether coalescing may change location validity. | |||
139 | using DbgValueLoc = std::pair<SlotIndex, MachineInstr*>; | |||
140 | DenseMap<Register, std::vector<DbgValueLoc>> DbgVRegToValues; | |||
141 | ||||
142 | /// VRegs may be repeatedly coalesced, and have many DBG_VALUEs attached. | |||
143 | /// To avoid repeatedly merging sets of DbgValueLocs, instead record | |||
144 | /// which vregs have been coalesced, and where to. This map is from | |||
145 | /// vreg => {set of vregs merged in}. | |||
146 | DenseMap<Register, SmallVector<Register, 4>> DbgMergedVRegNums; | |||
147 | ||||
148 | /// A LaneMask to remember on which subregister live ranges we need to call | |||
149 | /// shrinkToUses() later. | |||
150 | LaneBitmask ShrinkMask; | |||
151 | ||||
152 | /// True if the main range of the currently coalesced intervals should be | |||
153 | /// checked for smaller live intervals. | |||
154 | bool ShrinkMainRange = false; | |||
155 | ||||
156 | /// True if the coalescer should aggressively coalesce global copies | |||
157 | /// in favor of keeping local copies. | |||
158 | bool JoinGlobalCopies = false; | |||
159 | ||||
160 | /// True if the coalescer should aggressively coalesce fall-thru | |||
161 | /// blocks exclusively containing copies. | |||
162 | bool JoinSplitEdges = false; | |||
163 | ||||
164 | /// Copy instructions yet to be coalesced. | |||
165 | SmallVector<MachineInstr*, 8> WorkList; | |||
166 | SmallVector<MachineInstr*, 8> LocalWorkList; | |||
167 | ||||
168 | /// Set of instruction pointers that have been erased, and | |||
169 | /// that may be present in WorkList. | |||
170 | SmallPtrSet<MachineInstr*, 8> ErasedInstrs; | |||
171 | ||||
172 | /// Dead instructions that are about to be deleted. | |||
173 | SmallVector<MachineInstr*, 8> DeadDefs; | |||
174 | ||||
175 | /// Virtual registers to be considered for register class inflation. | |||
176 | SmallVector<Register, 8> InflateRegs; | |||
177 | ||||
178 | /// The collection of live intervals which should have been updated | |||
179 | /// immediately after rematerialiation but delayed until | |||
180 | /// lateLiveIntervalUpdate is called. | |||
181 | DenseSet<Register> ToBeUpdated; | |||
182 | ||||
183 | /// Record how many times the large live interval with many valnos | |||
184 | /// has been tried to join with other live interval. | |||
185 | DenseMap<Register, unsigned long> LargeLIVisitCounter; | |||
186 | ||||
187 | /// Recursively eliminate dead defs in DeadDefs. | |||
188 | void eliminateDeadDefs(); | |||
189 | ||||
190 | /// LiveRangeEdit callback for eliminateDeadDefs(). | |||
191 | void LRE_WillEraseInstruction(MachineInstr *MI) override; | |||
192 | ||||
193 | /// Coalesce the LocalWorkList. | |||
194 | void coalesceLocals(); | |||
195 | ||||
196 | /// Join compatible live intervals | |||
197 | void joinAllIntervals(); | |||
198 | ||||
199 | /// Coalesce copies in the specified MBB, putting | |||
200 | /// copies that cannot yet be coalesced into WorkList. | |||
201 | void copyCoalesceInMBB(MachineBasicBlock *MBB); | |||
202 | ||||
203 | /// Tries to coalesce all copies in CurrList. Returns true if any progress | |||
204 | /// was made. | |||
205 | bool copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList); | |||
206 | ||||
207 | /// If one def has many copy like uses, and those copy uses are all | |||
208 | /// rematerialized, the live interval update needed for those | |||
209 | /// rematerializations will be delayed and done all at once instead | |||
210 | /// of being done multiple times. This is to save compile cost because | |||
211 | /// live interval update is costly. | |||
212 | void lateLiveIntervalUpdate(); | |||
213 | ||||
214 | /// Check if the incoming value defined by a COPY at \p SLRQ in the subrange | |||
215 | /// has no value defined in the predecessors. If the incoming value is the | |||
216 | /// same as defined by the copy itself, the value is considered undefined. | |||
217 | bool copyValueUndefInPredecessors(LiveRange &S, | |||
218 | const MachineBasicBlock *MBB, | |||
219 | LiveQueryResult SLRQ); | |||
220 | ||||
221 | /// Set necessary undef flags on subregister uses after pruning out undef | |||
222 | /// lane segments from the subrange. | |||
223 | void setUndefOnPrunedSubRegUses(LiveInterval &LI, Register Reg, | |||
224 | LaneBitmask PrunedLanes); | |||
225 | ||||
226 | /// Attempt to join intervals corresponding to SrcReg/DstReg, which are the | |||
227 | /// src/dst of the copy instruction CopyMI. This returns true if the copy | |||
228 | /// was successfully coalesced away. If it is not currently possible to | |||
229 | /// coalesce this interval, but it may be possible if other things get | |||
230 | /// coalesced, then it returns true by reference in 'Again'. | |||
231 | bool joinCopy(MachineInstr *CopyMI, bool &Again); | |||
232 | ||||
233 | /// Attempt to join these two intervals. On failure, this | |||
234 | /// returns false. The output "SrcInt" will not have been modified, so we | |||
235 | /// can use this information below to update aliases. | |||
236 | bool joinIntervals(CoalescerPair &CP); | |||
237 | ||||
238 | /// Attempt joining two virtual registers. Return true on success. | |||
239 | bool joinVirtRegs(CoalescerPair &CP); | |||
240 | ||||
241 | /// If a live interval has many valnos and is coalesced with other | |||
242 | /// live intervals many times, we regard such live interval as having | |||
243 | /// high compile time cost. | |||
244 | bool isHighCostLiveInterval(LiveInterval &LI); | |||
245 | ||||
246 | /// Attempt joining with a reserved physreg. | |||
247 | bool joinReservedPhysReg(CoalescerPair &CP); | |||
248 | ||||
249 | /// Add the LiveRange @p ToMerge as a subregister liverange of @p LI. | |||
250 | /// Subranges in @p LI which only partially interfere with the desired | |||
251 | /// LaneMask are split as necessary. @p LaneMask are the lanes that | |||
252 | /// @p ToMerge will occupy in the coalescer register. @p LI has its subrange | |||
253 | /// lanemasks already adjusted to the coalesced register. | |||
254 | void mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge, | |||
255 | LaneBitmask LaneMask, CoalescerPair &CP, | |||
256 | unsigned DstIdx); | |||
257 | ||||
258 | /// Join the liveranges of two subregisters. Joins @p RRange into | |||
259 | /// @p LRange, @p RRange may be invalid afterwards. | |||
260 | void joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, | |||
261 | LaneBitmask LaneMask, const CoalescerPair &CP); | |||
262 | ||||
263 | /// We found a non-trivially-coalescable copy. If the source value number is | |||
264 | /// defined by a copy from the destination reg see if we can merge these two | |||
265 | /// destination reg valno# into a single value number, eliminating a copy. | |||
266 | /// This returns true if an interval was modified. | |||
267 | bool adjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI); | |||
268 | ||||
269 | /// Return true if there are definitions of IntB | |||
270 | /// other than BValNo val# that can reach uses of AValno val# of IntA. | |||
271 | bool hasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB, | |||
272 | VNInfo *AValNo, VNInfo *BValNo); | |||
273 | ||||
274 | /// We found a non-trivially-coalescable copy. | |||
275 | /// If the source value number is defined by a commutable instruction and | |||
276 | /// its other operand is coalesced to the copy dest register, see if we | |||
277 | /// can transform the copy into a noop by commuting the definition. | |||
278 | /// This returns a pair of two flags: | |||
279 | /// - the first element is true if an interval was modified, | |||
280 | /// - the second element is true if the destination interval needs | |||
281 | /// to be shrunk after deleting the copy. | |||
282 | std::pair<bool,bool> removeCopyByCommutingDef(const CoalescerPair &CP, | |||
283 | MachineInstr *CopyMI); | |||
284 | ||||
285 | /// We found a copy which can be moved to its less frequent predecessor. | |||
286 | bool removePartialRedundancy(const CoalescerPair &CP, MachineInstr &CopyMI); | |||
287 | ||||
288 | /// If the source of a copy is defined by a | |||
289 | /// trivial computation, replace the copy by rematerialize the definition. | |||
290 | bool reMaterializeTrivialDef(const CoalescerPair &CP, MachineInstr *CopyMI, | |||
291 | bool &IsDefCopy); | |||
292 | ||||
293 | /// Return true if a copy involving a physreg should be joined. | |||
294 | bool canJoinPhys(const CoalescerPair &CP); | |||
295 | ||||
296 | /// Replace all defs and uses of SrcReg to DstReg and update the subregister | |||
297 | /// number if it is not zero. If DstReg is a physical register and the | |||
298 | /// existing subregister number of the def / use being updated is not zero, | |||
299 | /// make sure to set it to the correct physical subregister. | |||
300 | void updateRegDefsUses(Register SrcReg, Register DstReg, unsigned SubIdx); | |||
301 | ||||
302 | /// If the given machine operand reads only undefined lanes add an undef | |||
303 | /// flag. | |||
304 | /// This can happen when undef uses were previously concealed by a copy | |||
305 | /// which we coalesced. Example: | |||
306 | /// %0:sub0<def,read-undef> = ... | |||
307 | /// %1 = COPY %0 <-- Coalescing COPY reveals undef | |||
308 | /// = use %1:sub1 <-- hidden undef use | |||
309 | void addUndefFlag(const LiveInterval &Int, SlotIndex UseIdx, | |||
310 | MachineOperand &MO, unsigned SubRegIdx); | |||
311 | ||||
312 | /// Handle copies of undef values. If the undef value is an incoming | |||
313 | /// PHI value, it will convert @p CopyMI to an IMPLICIT_DEF. | |||
314 | /// Returns nullptr if @p CopyMI was not in any way eliminable. Otherwise, | |||
315 | /// it returns @p CopyMI (which could be an IMPLICIT_DEF at this point). | |||
316 | MachineInstr *eliminateUndefCopy(MachineInstr *CopyMI); | |||
317 | ||||
318 | /// Check whether or not we should apply the terminal rule on the | |||
319 | /// destination (Dst) of \p Copy. | |||
320 | /// When the terminal rule applies, Copy is not profitable to | |||
321 | /// coalesce. | |||
322 | /// Dst is terminal if it has exactly one affinity (Dst, Src) and | |||
323 | /// at least one interference (Dst, Dst2). If Dst is terminal, the | |||
324 | /// terminal rule consists in checking that at least one of | |||
325 | /// interfering node, say Dst2, has an affinity of equal or greater | |||
326 | /// weight with Src. | |||
327 | /// In that case, Dst2 and Dst will not be able to be both coalesced | |||
328 | /// with Src. Since Dst2 exposes more coalescing opportunities than | |||
329 | /// Dst, we can drop \p Copy. | |||
330 | bool applyTerminalRule(const MachineInstr &Copy) const; | |||
331 | ||||
332 | /// Wrapper method for \see LiveIntervals::shrinkToUses. | |||
333 | /// This method does the proper fixing of the live-ranges when the afore | |||
334 | /// mentioned method returns true. | |||
335 | void shrinkToUses(LiveInterval *LI, | |||
336 | SmallVectorImpl<MachineInstr * > *Dead = nullptr) { | |||
337 | NumShrinkToUses++; | |||
338 | if (LIS->shrinkToUses(LI, Dead)) { | |||
339 | /// Check whether or not \p LI is composed by multiple connected | |||
340 | /// components and if that is the case, fix that. | |||
341 | SmallVector<LiveInterval*, 8> SplitLIs; | |||
342 | LIS->splitSeparateComponents(*LI, SplitLIs); | |||
343 | } | |||
344 | } | |||
345 | ||||
346 | /// Wrapper Method to do all the necessary work when an Instruction is | |||
347 | /// deleted. | |||
348 | /// Optimizations should use this to make sure that deleted instructions | |||
349 | /// are always accounted for. | |||
350 | void deleteInstr(MachineInstr* MI) { | |||
351 | ErasedInstrs.insert(MI); | |||
352 | LIS->RemoveMachineInstrFromMaps(*MI); | |||
353 | MI->eraseFromParent(); | |||
354 | } | |||
355 | ||||
356 | /// Walk over function and initialize the DbgVRegToValues map. | |||
357 | void buildVRegToDbgValueMap(MachineFunction &MF); | |||
358 | ||||
359 | /// Test whether, after merging, any DBG_VALUEs would refer to a | |||
360 | /// different value number than before merging, and whether this can | |||
361 | /// be resolved. If not, mark the DBG_VALUE as being undef. | |||
362 | void checkMergingChangesDbgValues(CoalescerPair &CP, LiveRange &LHS, | |||
363 | JoinVals &LHSVals, LiveRange &RHS, | |||
364 | JoinVals &RHSVals); | |||
365 | ||||
366 | void checkMergingChangesDbgValuesImpl(Register Reg, LiveRange &OtherRange, | |||
367 | LiveRange &RegRange, JoinVals &Vals2); | |||
368 | ||||
369 | public: | |||
370 | static char ID; ///< Class identification, replacement for typeinfo | |||
371 | ||||
372 | RegisterCoalescer() : MachineFunctionPass(ID) { | |||
373 | initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); | |||
374 | } | |||
375 | ||||
376 | void getAnalysisUsage(AnalysisUsage &AU) const override; | |||
377 | ||||
378 | void releaseMemory() override; | |||
379 | ||||
380 | /// This is the pass entry point. | |||
381 | bool runOnMachineFunction(MachineFunction&) override; | |||
382 | ||||
383 | /// Implement the dump method. | |||
384 | void print(raw_ostream &O, const Module* = nullptr) const override; | |||
385 | }; | |||
386 | ||||
387 | } // end anonymous namespace | |||
388 | ||||
389 | char RegisterCoalescer::ID = 0; | |||
390 | ||||
391 | char &llvm::RegisterCoalescerID = RegisterCoalescer::ID; | |||
392 | ||||
393 | INITIALIZE_PASS_BEGIN(RegisterCoalescer, "simple-register-coalescing",static void *initializeRegisterCoalescerPassOnce(PassRegistry &Registry) { | |||
394 | "Simple Register Coalescing", false, false)static void *initializeRegisterCoalescerPassOnce(PassRegistry &Registry) { | |||
395 | INITIALIZE_PASS_DEPENDENCY(LiveIntervals)initializeLiveIntervalsPass(Registry); | |||
396 | INITIALIZE_PASS_DEPENDENCY(SlotIndexes)initializeSlotIndexesPass(Registry); | |||
397 | INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)initializeMachineLoopInfoPass(Registry); | |||
398 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry); | |||
399 | INITIALIZE_PASS_END(RegisterCoalescer, "simple-register-coalescing",PassInfo *PI = new PassInfo( "Simple Register Coalescing", "simple-register-coalescing" , &RegisterCoalescer::ID, PassInfo::NormalCtor_t(callDefaultCtor <RegisterCoalescer>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeRegisterCoalescerPassFlag ; void llvm::initializeRegisterCoalescerPass(PassRegistry & Registry) { llvm::call_once(InitializeRegisterCoalescerPassFlag , initializeRegisterCoalescerPassOnce, std::ref(Registry)); } | |||
400 | "Simple Register Coalescing", false, false)PassInfo *PI = new PassInfo( "Simple Register Coalescing", "simple-register-coalescing" , &RegisterCoalescer::ID, PassInfo::NormalCtor_t(callDefaultCtor <RegisterCoalescer>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeRegisterCoalescerPassFlag ; void llvm::initializeRegisterCoalescerPass(PassRegistry & Registry) { llvm::call_once(InitializeRegisterCoalescerPassFlag , initializeRegisterCoalescerPassOnce, std::ref(Registry)); } | |||
401 | ||||
402 | LLVM_NODISCARD[[clang::warn_unused_result]] static bool isMoveInstr(const TargetRegisterInfo &tri, | |||
403 | const MachineInstr *MI, Register &Src, | |||
404 | Register &Dst, unsigned &SrcSub, | |||
405 | unsigned &DstSub) { | |||
406 | if (MI->isCopy()) { | |||
407 | Dst = MI->getOperand(0).getReg(); | |||
408 | DstSub = MI->getOperand(0).getSubReg(); | |||
409 | Src = MI->getOperand(1).getReg(); | |||
410 | SrcSub = MI->getOperand(1).getSubReg(); | |||
411 | } else if (MI->isSubregToReg()) { | |||
412 | Dst = MI->getOperand(0).getReg(); | |||
413 | DstSub = tri.composeSubRegIndices(MI->getOperand(0).getSubReg(), | |||
414 | MI->getOperand(3).getImm()); | |||
415 | Src = MI->getOperand(2).getReg(); | |||
416 | SrcSub = MI->getOperand(2).getSubReg(); | |||
417 | } else | |||
418 | return false; | |||
419 | return true; | |||
420 | } | |||
421 | ||||
422 | /// Return true if this block should be vacated by the coalescer to eliminate | |||
423 | /// branches. The important cases to handle in the coalescer are critical edges | |||
424 | /// split during phi elimination which contain only copies. Simple blocks that | |||
425 | /// contain non-branches should also be vacated, but this can be handled by an | |||
426 | /// earlier pass similar to early if-conversion. | |||
427 | static bool isSplitEdge(const MachineBasicBlock *MBB) { | |||
428 | if (MBB->pred_size() != 1 || MBB->succ_size() != 1) | |||
429 | return false; | |||
430 | ||||
431 | for (const auto &MI : *MBB) { | |||
432 | if (!MI.isCopyLike() && !MI.isUnconditionalBranch()) | |||
433 | return false; | |||
434 | } | |||
435 | return true; | |||
436 | } | |||
437 | ||||
438 | bool CoalescerPair::setRegisters(const MachineInstr *MI) { | |||
439 | SrcReg = DstReg = Register(); | |||
440 | SrcIdx = DstIdx = 0; | |||
441 | NewRC = nullptr; | |||
442 | Flipped = CrossClass = false; | |||
443 | ||||
444 | Register Src, Dst; | |||
445 | unsigned SrcSub, DstSub; | |||
446 | if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub)) | |||
447 | return false; | |||
448 | Partial = SrcSub || DstSub; | |||
449 | ||||
450 | // If one register is a physreg, it must be Dst. | |||
451 | if (Register::isPhysicalRegister(Src)) { | |||
452 | if (Register::isPhysicalRegister(Dst)) | |||
453 | return false; | |||
454 | std::swap(Src, Dst); | |||
455 | std::swap(SrcSub, DstSub); | |||
456 | Flipped = true; | |||
457 | } | |||
458 | ||||
459 | const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo(); | |||
460 | ||||
461 | if (Register::isPhysicalRegister(Dst)) { | |||
462 | // Eliminate DstSub on a physreg. | |||
463 | if (DstSub) { | |||
464 | Dst = TRI.getSubReg(Dst, DstSub); | |||
465 | if (!Dst) return false; | |||
466 | DstSub = 0; | |||
467 | } | |||
468 | ||||
469 | // Eliminate SrcSub by picking a corresponding Dst superregister. | |||
470 | if (SrcSub) { | |||
471 | Dst = TRI.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src)); | |||
472 | if (!Dst) return false; | |||
473 | } else if (!MRI.getRegClass(Src)->contains(Dst)) { | |||
474 | return false; | |||
475 | } | |||
476 | } else { | |||
477 | // Both registers are virtual. | |||
478 | const TargetRegisterClass *SrcRC = MRI.getRegClass(Src); | |||
479 | const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); | |||
480 | ||||
481 | // Both registers have subreg indices. | |||
482 | if (SrcSub && DstSub) { | |||
483 | // Copies between different sub-registers are never coalescable. | |||
484 | if (Src == Dst && SrcSub != DstSub) | |||
485 | return false; | |||
486 | ||||
487 | NewRC = TRI.getCommonSuperRegClass(SrcRC, SrcSub, DstRC, DstSub, | |||
488 | SrcIdx, DstIdx); | |||
489 | if (!NewRC) | |||
490 | return false; | |||
491 | } else if (DstSub) { | |||
492 | // SrcReg will be merged with a sub-register of DstReg. | |||
493 | SrcIdx = DstSub; | |||
494 | NewRC = TRI.getMatchingSuperRegClass(DstRC, SrcRC, DstSub); | |||
495 | } else if (SrcSub) { | |||
496 | // DstReg will be merged with a sub-register of SrcReg. | |||
497 | DstIdx = SrcSub; | |||
498 | NewRC = TRI.getMatchingSuperRegClass(SrcRC, DstRC, SrcSub); | |||
499 | } else { | |||
500 | // This is a straight copy without sub-registers. | |||
501 | NewRC = TRI.getCommonSubClass(DstRC, SrcRC); | |||
502 | } | |||
503 | ||||
504 | // The combined constraint may be impossible to satisfy. | |||
505 | if (!NewRC) | |||
506 | return false; | |||
507 | ||||
508 | // Prefer SrcReg to be a sub-register of DstReg. | |||
509 | // FIXME: Coalescer should support subregs symmetrically. | |||
510 | if (DstIdx && !SrcIdx) { | |||
511 | std::swap(Src, Dst); | |||
512 | std::swap(SrcIdx, DstIdx); | |||
513 | Flipped = !Flipped; | |||
514 | } | |||
515 | ||||
516 | CrossClass = NewRC != DstRC || NewRC != SrcRC; | |||
517 | } | |||
518 | // Check our invariants | |||
519 | assert(Register::isVirtualRegister(Src) && "Src must be virtual")((Register::isVirtualRegister(Src) && "Src must be virtual" ) ? static_cast<void> (0) : __assert_fail ("Register::isVirtualRegister(Src) && \"Src must be virtual\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 519, __PRETTY_FUNCTION__)); | |||
520 | assert(!(Register::isPhysicalRegister(Dst) && DstSub) &&((!(Register::isPhysicalRegister(Dst) && DstSub) && "Cannot have a physical SubIdx") ? static_cast<void> ( 0) : __assert_fail ("!(Register::isPhysicalRegister(Dst) && DstSub) && \"Cannot have a physical SubIdx\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 521, __PRETTY_FUNCTION__)) | |||
521 | "Cannot have a physical SubIdx")((!(Register::isPhysicalRegister(Dst) && DstSub) && "Cannot have a physical SubIdx") ? static_cast<void> ( 0) : __assert_fail ("!(Register::isPhysicalRegister(Dst) && DstSub) && \"Cannot have a physical SubIdx\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 521, __PRETTY_FUNCTION__)); | |||
522 | SrcReg = Src; | |||
523 | DstReg = Dst; | |||
524 | return true; | |||
525 | } | |||
526 | ||||
527 | bool CoalescerPair::flip() { | |||
528 | if (Register::isPhysicalRegister(DstReg)) | |||
529 | return false; | |||
530 | std::swap(SrcReg, DstReg); | |||
531 | std::swap(SrcIdx, DstIdx); | |||
532 | Flipped = !Flipped; | |||
533 | return true; | |||
534 | } | |||
535 | ||||
536 | bool CoalescerPair::isCoalescable(const MachineInstr *MI) const { | |||
537 | if (!MI) | |||
538 | return false; | |||
539 | Register Src, Dst; | |||
540 | unsigned SrcSub, DstSub; | |||
541 | if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub)) | |||
542 | return false; | |||
543 | ||||
544 | // Find the virtual register that is SrcReg. | |||
545 | if (Dst == SrcReg) { | |||
546 | std::swap(Src, Dst); | |||
547 | std::swap(SrcSub, DstSub); | |||
548 | } else if (Src != SrcReg) { | |||
549 | return false; | |||
550 | } | |||
551 | ||||
552 | // Now check that Dst matches DstReg. | |||
553 | if (DstReg.isPhysical()) { | |||
554 | if (!Dst.isPhysical()) | |||
555 | return false; | |||
556 | assert(!DstIdx && !SrcIdx && "Inconsistent CoalescerPair state.")((!DstIdx && !SrcIdx && "Inconsistent CoalescerPair state." ) ? static_cast<void> (0) : __assert_fail ("!DstIdx && !SrcIdx && \"Inconsistent CoalescerPair state.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 556, __PRETTY_FUNCTION__)); | |||
557 | // DstSub could be set for a physreg from INSERT_SUBREG. | |||
558 | if (DstSub) | |||
559 | Dst = TRI.getSubReg(Dst, DstSub); | |||
560 | // Full copy of Src. | |||
561 | if (!SrcSub) | |||
562 | return DstReg == Dst; | |||
563 | // This is a partial register copy. Check that the parts match. | |||
564 | return Register(TRI.getSubReg(DstReg, SrcSub)) == Dst; | |||
565 | } else { | |||
566 | // DstReg is virtual. | |||
567 | if (DstReg != Dst) | |||
568 | return false; | |||
569 | // Registers match, do the subregisters line up? | |||
570 | return TRI.composeSubRegIndices(SrcIdx, SrcSub) == | |||
571 | TRI.composeSubRegIndices(DstIdx, DstSub); | |||
572 | } | |||
573 | } | |||
574 | ||||
575 | void RegisterCoalescer::getAnalysisUsage(AnalysisUsage &AU) const { | |||
576 | AU.setPreservesCFG(); | |||
577 | AU.addRequired<AAResultsWrapperPass>(); | |||
578 | AU.addRequired<LiveIntervals>(); | |||
579 | AU.addPreserved<LiveIntervals>(); | |||
580 | AU.addPreserved<SlotIndexes>(); | |||
581 | AU.addRequired<MachineLoopInfo>(); | |||
582 | AU.addPreserved<MachineLoopInfo>(); | |||
583 | AU.addPreservedID(MachineDominatorsID); | |||
584 | MachineFunctionPass::getAnalysisUsage(AU); | |||
585 | } | |||
586 | ||||
587 | void RegisterCoalescer::eliminateDeadDefs() { | |||
588 | SmallVector<Register, 8> NewRegs; | |||
589 | LiveRangeEdit(nullptr, NewRegs, *MF, *LIS, | |||
590 | nullptr, this).eliminateDeadDefs(DeadDefs); | |||
591 | } | |||
592 | ||||
593 | void RegisterCoalescer::LRE_WillEraseInstruction(MachineInstr *MI) { | |||
594 | // MI may be in WorkList. Make sure we don't visit it. | |||
595 | ErasedInstrs.insert(MI); | |||
596 | } | |||
597 | ||||
598 | bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP, | |||
599 | MachineInstr *CopyMI) { | |||
600 | assert(!CP.isPartial() && "This doesn't work for partial copies.")((!CP.isPartial() && "This doesn't work for partial copies." ) ? static_cast<void> (0) : __assert_fail ("!CP.isPartial() && \"This doesn't work for partial copies.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 600, __PRETTY_FUNCTION__)); | |||
601 | assert(!CP.isPhys() && "This doesn't work for physreg copies.")((!CP.isPhys() && "This doesn't work for physreg copies." ) ? static_cast<void> (0) : __assert_fail ("!CP.isPhys() && \"This doesn't work for physreg copies.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 601, __PRETTY_FUNCTION__)); | |||
602 | ||||
603 | LiveInterval &IntA = | |||
604 | LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); | |||
605 | LiveInterval &IntB = | |||
606 | LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); | |||
607 | SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot(); | |||
608 | ||||
609 | // We have a non-trivially-coalescable copy with IntA being the source and | |||
610 | // IntB being the dest, thus this defines a value number in IntB. If the | |||
611 | // source value number (in IntA) is defined by a copy from B, see if we can | |||
612 | // merge these two pieces of B into a single value number, eliminating a copy. | |||
613 | // For example: | |||
614 | // | |||
615 | // A3 = B0 | |||
616 | // ... | |||
617 | // B1 = A3 <- this copy | |||
618 | // | |||
619 | // In this case, B0 can be extended to where the B1 copy lives, allowing the | |||
620 | // B1 value number to be replaced with B0 (which simplifies the B | |||
621 | // liveinterval). | |||
622 | ||||
623 | // BValNo is a value number in B that is defined by a copy from A. 'B1' in | |||
624 | // the example above. | |||
625 | LiveInterval::iterator BS = IntB.FindSegmentContaining(CopyIdx); | |||
626 | if (BS == IntB.end()) return false; | |||
627 | VNInfo *BValNo = BS->valno; | |||
628 | ||||
629 | // Get the location that B is defined at. Two options: either this value has | |||
630 | // an unknown definition point or it is defined at CopyIdx. If unknown, we | |||
631 | // can't process it. | |||
632 | if (BValNo->def != CopyIdx) return false; | |||
633 | ||||
634 | // AValNo is the value number in A that defines the copy, A3 in the example. | |||
635 | SlotIndex CopyUseIdx = CopyIdx.getRegSlot(true); | |||
636 | LiveInterval::iterator AS = IntA.FindSegmentContaining(CopyUseIdx); | |||
637 | // The live segment might not exist after fun with physreg coalescing. | |||
638 | if (AS == IntA.end()) return false; | |||
639 | VNInfo *AValNo = AS->valno; | |||
640 | ||||
641 | // If AValNo is defined as a copy from IntB, we can potentially process this. | |||
642 | // Get the instruction that defines this value number. | |||
643 | MachineInstr *ACopyMI = LIS->getInstructionFromIndex(AValNo->def); | |||
644 | // Don't allow any partial copies, even if isCoalescable() allows them. | |||
645 | if (!CP.isCoalescable(ACopyMI) || !ACopyMI->isFullCopy()) | |||
646 | return false; | |||
647 | ||||
648 | // Get the Segment in IntB that this value number starts with. | |||
649 | LiveInterval::iterator ValS = | |||
650 | IntB.FindSegmentContaining(AValNo->def.getPrevSlot()); | |||
651 | if (ValS == IntB.end()) | |||
652 | return false; | |||
653 | ||||
654 | // Make sure that the end of the live segment is inside the same block as | |||
655 | // CopyMI. | |||
656 | MachineInstr *ValSEndInst = | |||
657 | LIS->getInstructionFromIndex(ValS->end.getPrevSlot()); | |||
658 | if (!ValSEndInst || ValSEndInst->getParent() != CopyMI->getParent()) | |||
659 | return false; | |||
660 | ||||
661 | // Okay, we now know that ValS ends in the same block that the CopyMI | |||
662 | // live-range starts. If there are no intervening live segments between them | |||
663 | // in IntB, we can merge them. | |||
664 | if (ValS+1 != BS) return false; | |||
665 | ||||
666 | LLVM_DEBUG(dbgs() << "Extending: " << printReg(IntB.reg(), TRI))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Extending: " << printReg (IntB.reg(), TRI); } } while (false); | |||
667 | ||||
668 | SlotIndex FillerStart = ValS->end, FillerEnd = BS->start; | |||
669 | // We are about to delete CopyMI, so need to remove it as the 'instruction | |||
670 | // that defines this value #'. Update the valnum with the new defining | |||
671 | // instruction #. | |||
672 | BValNo->def = FillerStart; | |||
673 | ||||
674 | // Okay, we can merge them. We need to insert a new liverange: | |||
675 | // [ValS.end, BS.begin) of either value number, then we merge the | |||
676 | // two value numbers. | |||
677 | IntB.addSegment(LiveInterval::Segment(FillerStart, FillerEnd, BValNo)); | |||
678 | ||||
679 | // Okay, merge "B1" into the same value number as "B0". | |||
680 | if (BValNo != ValS->valno) | |||
681 | IntB.MergeValueNumberInto(BValNo, ValS->valno); | |||
682 | ||||
683 | // Do the same for the subregister segments. | |||
684 | for (LiveInterval::SubRange &S : IntB.subranges()) { | |||
685 | // Check for SubRange Segments of the form [1234r,1234d:0) which can be | |||
686 | // removed to prevent creating bogus SubRange Segments. | |||
687 | LiveInterval::iterator SS = S.FindSegmentContaining(CopyIdx); | |||
688 | if (SS != S.end() && SlotIndex::isSameInstr(SS->start, SS->end)) { | |||
689 | S.removeSegment(*SS, true); | |||
690 | continue; | |||
691 | } | |||
692 | // The subrange may have ended before FillerStart. If so, extend it. | |||
693 | if (!S.getVNInfoAt(FillerStart)) { | |||
694 | SlotIndex BBStart = | |||
695 | LIS->getMBBStartIdx(LIS->getMBBFromIndex(FillerStart)); | |||
696 | S.extendInBlock(BBStart, FillerStart); | |||
697 | } | |||
698 | VNInfo *SubBValNo = S.getVNInfoAt(CopyIdx); | |||
699 | S.addSegment(LiveInterval::Segment(FillerStart, FillerEnd, SubBValNo)); | |||
700 | VNInfo *SubValSNo = S.getVNInfoAt(AValNo->def.getPrevSlot()); | |||
701 | if (SubBValNo != SubValSNo) | |||
702 | S.MergeValueNumberInto(SubBValNo, SubValSNo); | |||
703 | } | |||
704 | ||||
705 | LLVM_DEBUG(dbgs() << " result = " << IntB << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << " result = " << IntB << '\n'; } } while (false); | |||
706 | ||||
707 | // If the source instruction was killing the source register before the | |||
708 | // merge, unset the isKill marker given the live range has been extended. | |||
709 | int UIdx = ValSEndInst->findRegisterUseOperandIdx(IntB.reg(), true); | |||
710 | if (UIdx != -1) { | |||
711 | ValSEndInst->getOperand(UIdx).setIsKill(false); | |||
712 | } | |||
713 | ||||
714 | // Rewrite the copy. | |||
715 | CopyMI->substituteRegister(IntA.reg(), IntB.reg(), 0, *TRI); | |||
716 | // If the copy instruction was killing the destination register or any | |||
717 | // subrange before the merge trim the live range. | |||
718 | bool RecomputeLiveRange = AS->end == CopyIdx; | |||
719 | if (!RecomputeLiveRange) { | |||
720 | for (LiveInterval::SubRange &S : IntA.subranges()) { | |||
721 | LiveInterval::iterator SS = S.FindSegmentContaining(CopyUseIdx); | |||
722 | if (SS != S.end() && SS->end == CopyIdx) { | |||
723 | RecomputeLiveRange = true; | |||
724 | break; | |||
725 | } | |||
726 | } | |||
727 | } | |||
728 | if (RecomputeLiveRange) | |||
729 | shrinkToUses(&IntA); | |||
730 | ||||
731 | ++numExtends; | |||
732 | return true; | |||
733 | } | |||
734 | ||||
735 | bool RegisterCoalescer::hasOtherReachingDefs(LiveInterval &IntA, | |||
736 | LiveInterval &IntB, | |||
737 | VNInfo *AValNo, | |||
738 | VNInfo *BValNo) { | |||
739 | // If AValNo has PHI kills, conservatively assume that IntB defs can reach | |||
740 | // the PHI values. | |||
741 | if (LIS->hasPHIKill(IntA, AValNo)) | |||
742 | return true; | |||
743 | ||||
744 | for (LiveRange::Segment &ASeg : IntA.segments) { | |||
745 | if (ASeg.valno != AValNo) continue; | |||
746 | LiveInterval::iterator BI = llvm::upper_bound(IntB, ASeg.start); | |||
747 | if (BI != IntB.begin()) | |||
748 | --BI; | |||
749 | for (; BI != IntB.end() && ASeg.end >= BI->start; ++BI) { | |||
750 | if (BI->valno == BValNo) | |||
751 | continue; | |||
752 | if (BI->start <= ASeg.start && BI->end > ASeg.start) | |||
753 | return true; | |||
754 | if (BI->start > ASeg.start && BI->start < ASeg.end) | |||
755 | return true; | |||
756 | } | |||
757 | } | |||
758 | return false; | |||
759 | } | |||
760 | ||||
761 | /// Copy segments with value number @p SrcValNo from liverange @p Src to live | |||
762 | /// range @Dst and use value number @p DstValNo there. | |||
763 | static std::pair<bool,bool> | |||
764 | addSegmentsWithValNo(LiveRange &Dst, VNInfo *DstValNo, const LiveRange &Src, | |||
765 | const VNInfo *SrcValNo) { | |||
766 | bool Changed = false; | |||
767 | bool MergedWithDead = false; | |||
768 | for (const LiveRange::Segment &S : Src.segments) { | |||
769 | if (S.valno != SrcValNo) | |||
770 | continue; | |||
771 | // This is adding a segment from Src that ends in a copy that is about | |||
772 | // to be removed. This segment is going to be merged with a pre-existing | |||
773 | // segment in Dst. This works, except in cases when the corresponding | |||
774 | // segment in Dst is dead. For example: adding [192r,208r:1) from Src | |||
775 | // to [208r,208d:1) in Dst would create [192r,208d:1) in Dst. | |||
776 | // Recognized such cases, so that the segments can be shrunk. | |||
777 | LiveRange::Segment Added = LiveRange::Segment(S.start, S.end, DstValNo); | |||
778 | LiveRange::Segment &Merged = *Dst.addSegment(Added); | |||
779 | if (Merged.end.isDead()) | |||
780 | MergedWithDead = true; | |||
781 | Changed = true; | |||
782 | } | |||
783 | return std::make_pair(Changed, MergedWithDead); | |||
784 | } | |||
785 | ||||
786 | std::pair<bool,bool> | |||
787 | RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP, | |||
788 | MachineInstr *CopyMI) { | |||
789 | assert(!CP.isPhys())((!CP.isPhys()) ? static_cast<void> (0) : __assert_fail ("!CP.isPhys()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 789, __PRETTY_FUNCTION__)); | |||
790 | ||||
791 | LiveInterval &IntA = | |||
792 | LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); | |||
793 | LiveInterval &IntB = | |||
794 | LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); | |||
795 | ||||
796 | // We found a non-trivially-coalescable copy with IntA being the source and | |||
797 | // IntB being the dest, thus this defines a value number in IntB. If the | |||
798 | // source value number (in IntA) is defined by a commutable instruction and | |||
799 | // its other operand is coalesced to the copy dest register, see if we can | |||
800 | // transform the copy into a noop by commuting the definition. For example, | |||
801 | // | |||
802 | // A3 = op A2 killed B0 | |||
803 | // ... | |||
804 | // B1 = A3 <- this copy | |||
805 | // ... | |||
806 | // = op A3 <- more uses | |||
807 | // | |||
808 | // ==> | |||
809 | // | |||
810 | // B2 = op B0 killed A2 | |||
811 | // ... | |||
812 | // B1 = B2 <- now an identity copy | |||
813 | // ... | |||
814 | // = op B2 <- more uses | |||
815 | ||||
816 | // BValNo is a value number in B that is defined by a copy from A. 'B1' in | |||
817 | // the example above. | |||
818 | SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot(); | |||
819 | VNInfo *BValNo = IntB.getVNInfoAt(CopyIdx); | |||
820 | assert(BValNo != nullptr && BValNo->def == CopyIdx)((BValNo != nullptr && BValNo->def == CopyIdx) ? static_cast <void> (0) : __assert_fail ("BValNo != nullptr && BValNo->def == CopyIdx" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 820, __PRETTY_FUNCTION__)); | |||
821 | ||||
822 | // AValNo is the value number in A that defines the copy, A3 in the example. | |||
823 | VNInfo *AValNo = IntA.getVNInfoAt(CopyIdx.getRegSlot(true)); | |||
824 | assert(AValNo && !AValNo->isUnused() && "COPY source not live")((AValNo && !AValNo->isUnused() && "COPY source not live" ) ? static_cast<void> (0) : __assert_fail ("AValNo && !AValNo->isUnused() && \"COPY source not live\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 824, __PRETTY_FUNCTION__)); | |||
825 | if (AValNo->isPHIDef()) | |||
826 | return { false, false }; | |||
827 | MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def); | |||
828 | if (!DefMI) | |||
829 | return { false, false }; | |||
830 | if (!DefMI->isCommutable()) | |||
831 | return { false, false }; | |||
832 | // If DefMI is a two-address instruction then commuting it will change the | |||
833 | // destination register. | |||
834 | int DefIdx = DefMI->findRegisterDefOperandIdx(IntA.reg()); | |||
835 | assert(DefIdx != -1)((DefIdx != -1) ? static_cast<void> (0) : __assert_fail ("DefIdx != -1", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 835, __PRETTY_FUNCTION__)); | |||
836 | unsigned UseOpIdx; | |||
837 | if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx)) | |||
838 | return { false, false }; | |||
839 | ||||
840 | // FIXME: The code below tries to commute 'UseOpIdx' operand with some other | |||
841 | // commutable operand which is expressed by 'CommuteAnyOperandIndex'value | |||
842 | // passed to the method. That _other_ operand is chosen by | |||
843 | // the findCommutedOpIndices() method. | |||
844 | // | |||
845 | // That is obviously an area for improvement in case of instructions having | |||
846 | // more than 2 operands. For example, if some instruction has 3 commutable | |||
847 | // operands then all possible variants (i.e. op#1<->op#2, op#1<->op#3, | |||
848 | // op#2<->op#3) of commute transformation should be considered/tried here. | |||
849 | unsigned NewDstIdx = TargetInstrInfo::CommuteAnyOperandIndex; | |||
850 | if (!TII->findCommutedOpIndices(*DefMI, UseOpIdx, NewDstIdx)) | |||
851 | return { false, false }; | |||
852 | ||||
853 | MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx); | |||
854 | Register NewReg = NewDstMO.getReg(); | |||
855 | if (NewReg != IntB.reg() || !IntB.Query(AValNo->def).isKill()) | |||
856 | return { false, false }; | |||
857 | ||||
858 | // Make sure there are no other definitions of IntB that would reach the | |||
859 | // uses which the new definition can reach. | |||
860 | if (hasOtherReachingDefs(IntA, IntB, AValNo, BValNo)) | |||
861 | return { false, false }; | |||
862 | ||||
863 | // If some of the uses of IntA.reg is already coalesced away, return false. | |||
864 | // It's not possible to determine whether it's safe to perform the coalescing. | |||
865 | for (MachineOperand &MO : MRI->use_nodbg_operands(IntA.reg())) { | |||
866 | MachineInstr *UseMI = MO.getParent(); | |||
867 | unsigned OpNo = &MO - &UseMI->getOperand(0); | |||
868 | SlotIndex UseIdx = LIS->getInstructionIndex(*UseMI); | |||
869 | LiveInterval::iterator US = IntA.FindSegmentContaining(UseIdx); | |||
870 | if (US == IntA.end() || US->valno != AValNo) | |||
871 | continue; | |||
872 | // If this use is tied to a def, we can't rewrite the register. | |||
873 | if (UseMI->isRegTiedToDefOperand(OpNo)) | |||
874 | return { false, false }; | |||
875 | } | |||
876 | ||||
877 | LLVM_DEBUG(dbgs() << "\tremoveCopyByCommutingDef: " << AValNo->def << '\t'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremoveCopyByCommutingDef: " << AValNo->def << '\t' << *DefMI; } } while (false) | |||
878 | << *DefMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremoveCopyByCommutingDef: " << AValNo->def << '\t' << *DefMI; } } while (false); | |||
879 | ||||
880 | // At this point we have decided that it is legal to do this | |||
881 | // transformation. Start by commuting the instruction. | |||
882 | MachineBasicBlock *MBB = DefMI->getParent(); | |||
883 | MachineInstr *NewMI = | |||
884 | TII->commuteInstruction(*DefMI, false, UseOpIdx, NewDstIdx); | |||
885 | if (!NewMI) | |||
886 | return { false, false }; | |||
887 | if (Register::isVirtualRegister(IntA.reg()) && | |||
888 | Register::isVirtualRegister(IntB.reg()) && | |||
889 | !MRI->constrainRegClass(IntB.reg(), MRI->getRegClass(IntA.reg()))) | |||
890 | return { false, false }; | |||
891 | if (NewMI != DefMI) { | |||
892 | LIS->ReplaceMachineInstrInMaps(*DefMI, *NewMI); | |||
893 | MachineBasicBlock::iterator Pos = DefMI; | |||
894 | MBB->insert(Pos, NewMI); | |||
895 | MBB->erase(DefMI); | |||
896 | } | |||
897 | ||||
898 | // If ALR and BLR overlaps and end of BLR extends beyond end of ALR, e.g. | |||
899 | // A = or A, B | |||
900 | // ... | |||
901 | // B = A | |||
902 | // ... | |||
903 | // C = killed A | |||
904 | // ... | |||
905 | // = B | |||
906 | ||||
907 | // Update uses of IntA of the specific Val# with IntB. | |||
908 | for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(IntA.reg()), | |||
909 | UE = MRI->use_end(); | |||
910 | UI != UE; | |||
911 | /* ++UI is below because of possible MI removal */) { | |||
912 | MachineOperand &UseMO = *UI; | |||
913 | ++UI; | |||
914 | if (UseMO.isUndef()) | |||
915 | continue; | |||
916 | MachineInstr *UseMI = UseMO.getParent(); | |||
917 | if (UseMI->isDebugValue()) { | |||
918 | // FIXME These don't have an instruction index. Not clear we have enough | |||
919 | // info to decide whether to do this replacement or not. For now do it. | |||
920 | UseMO.setReg(NewReg); | |||
921 | continue; | |||
922 | } | |||
923 | SlotIndex UseIdx = LIS->getInstructionIndex(*UseMI).getRegSlot(true); | |||
924 | LiveInterval::iterator US = IntA.FindSegmentContaining(UseIdx); | |||
925 | assert(US != IntA.end() && "Use must be live")((US != IntA.end() && "Use must be live") ? static_cast <void> (0) : __assert_fail ("US != IntA.end() && \"Use must be live\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 925, __PRETTY_FUNCTION__)); | |||
926 | if (US->valno != AValNo) | |||
927 | continue; | |||
928 | // Kill flags are no longer accurate. They are recomputed after RA. | |||
929 | UseMO.setIsKill(false); | |||
930 | if (Register::isPhysicalRegister(NewReg)) | |||
931 | UseMO.substPhysReg(NewReg, *TRI); | |||
932 | else | |||
933 | UseMO.setReg(NewReg); | |||
934 | if (UseMI == CopyMI) | |||
935 | continue; | |||
936 | if (!UseMI->isCopy()) | |||
937 | continue; | |||
938 | if (UseMI->getOperand(0).getReg() != IntB.reg() || | |||
939 | UseMI->getOperand(0).getSubReg()) | |||
940 | continue; | |||
941 | ||||
942 | // This copy will become a noop. If it's defining a new val#, merge it into | |||
943 | // BValNo. | |||
944 | SlotIndex DefIdx = UseIdx.getRegSlot(); | |||
945 | VNInfo *DVNI = IntB.getVNInfoAt(DefIdx); | |||
946 | if (!DVNI) | |||
947 | continue; | |||
948 | LLVM_DEBUG(dbgs() << "\t\tnoop: " << DefIdx << '\t' << *UseMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tnoop: " << DefIdx << '\t' << *UseMI; } } while (false); | |||
949 | assert(DVNI->def == DefIdx)((DVNI->def == DefIdx) ? static_cast<void> (0) : __assert_fail ("DVNI->def == DefIdx", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 949, __PRETTY_FUNCTION__)); | |||
950 | BValNo = IntB.MergeValueNumberInto(DVNI, BValNo); | |||
951 | for (LiveInterval::SubRange &S : IntB.subranges()) { | |||
952 | VNInfo *SubDVNI = S.getVNInfoAt(DefIdx); | |||
953 | if (!SubDVNI) | |||
954 | continue; | |||
955 | VNInfo *SubBValNo = S.getVNInfoAt(CopyIdx); | |||
956 | assert(SubBValNo->def == CopyIdx)((SubBValNo->def == CopyIdx) ? static_cast<void> (0) : __assert_fail ("SubBValNo->def == CopyIdx", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 956, __PRETTY_FUNCTION__)); | |||
957 | S.MergeValueNumberInto(SubDVNI, SubBValNo); | |||
958 | } | |||
959 | ||||
960 | deleteInstr(UseMI); | |||
961 | } | |||
962 | ||||
963 | // Extend BValNo by merging in IntA live segments of AValNo. Val# definition | |||
964 | // is updated. | |||
965 | bool ShrinkB = false; | |||
966 | BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); | |||
967 | if (IntA.hasSubRanges() || IntB.hasSubRanges()) { | |||
968 | if (!IntA.hasSubRanges()) { | |||
969 | LaneBitmask Mask = MRI->getMaxLaneMaskForVReg(IntA.reg()); | |||
970 | IntA.createSubRangeFrom(Allocator, Mask, IntA); | |||
971 | } else if (!IntB.hasSubRanges()) { | |||
972 | LaneBitmask Mask = MRI->getMaxLaneMaskForVReg(IntB.reg()); | |||
973 | IntB.createSubRangeFrom(Allocator, Mask, IntB); | |||
974 | } | |||
975 | SlotIndex AIdx = CopyIdx.getRegSlot(true); | |||
976 | LaneBitmask MaskA; | |||
977 | const SlotIndexes &Indexes = *LIS->getSlotIndexes(); | |||
978 | for (LiveInterval::SubRange &SA : IntA.subranges()) { | |||
979 | VNInfo *ASubValNo = SA.getVNInfoAt(AIdx); | |||
980 | // Even if we are dealing with a full copy, some lanes can | |||
981 | // still be undefined. | |||
982 | // E.g., | |||
983 | // undef A.subLow = ... | |||
984 | // B = COPY A <== A.subHigh is undefined here and does | |||
985 | // not have a value number. | |||
986 | if (!ASubValNo) | |||
987 | continue; | |||
988 | MaskA |= SA.LaneMask; | |||
989 | ||||
990 | IntB.refineSubRanges( | |||
991 | Allocator, SA.LaneMask, | |||
992 | [&Allocator, &SA, CopyIdx, ASubValNo, | |||
993 | &ShrinkB](LiveInterval::SubRange &SR) { | |||
994 | VNInfo *BSubValNo = SR.empty() ? SR.getNextValue(CopyIdx, Allocator) | |||
995 | : SR.getVNInfoAt(CopyIdx); | |||
996 | assert(BSubValNo != nullptr)((BSubValNo != nullptr) ? static_cast<void> (0) : __assert_fail ("BSubValNo != nullptr", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 996, __PRETTY_FUNCTION__)); | |||
997 | auto P = addSegmentsWithValNo(SR, BSubValNo, SA, ASubValNo); | |||
998 | ShrinkB |= P.second; | |||
999 | if (P.first) | |||
1000 | BSubValNo->def = ASubValNo->def; | |||
1001 | }, | |||
1002 | Indexes, *TRI); | |||
1003 | } | |||
1004 | // Go over all subranges of IntB that have not been covered by IntA, | |||
1005 | // and delete the segments starting at CopyIdx. This can happen if | |||
1006 | // IntA has undef lanes that are defined in IntB. | |||
1007 | for (LiveInterval::SubRange &SB : IntB.subranges()) { | |||
1008 | if ((SB.LaneMask & MaskA).any()) | |||
1009 | continue; | |||
1010 | if (LiveRange::Segment *S = SB.getSegmentContaining(CopyIdx)) | |||
1011 | if (S->start.getBaseIndex() == CopyIdx.getBaseIndex()) | |||
1012 | SB.removeSegment(*S, true); | |||
1013 | } | |||
1014 | } | |||
1015 | ||||
1016 | BValNo->def = AValNo->def; | |||
1017 | auto P = addSegmentsWithValNo(IntB, BValNo, IntA, AValNo); | |||
1018 | ShrinkB |= P.second; | |||
1019 | LLVM_DEBUG(dbgs() << "\t\textended: " << IntB << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\textended: " << IntB << '\n'; } } while (false); | |||
1020 | ||||
1021 | LIS->removeVRegDefAt(IntA, AValNo->def); | |||
1022 | ||||
1023 | LLVM_DEBUG(dbgs() << "\t\ttrimmed: " << IntA << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttrimmed: " << IntA << '\n'; } } while (false); | |||
1024 | ++numCommutes; | |||
1025 | return { true, ShrinkB }; | |||
1026 | } | |||
1027 | ||||
1028 | /// For copy B = A in BB2, if A is defined by A = B in BB0 which is a | |||
1029 | /// predecessor of BB2, and if B is not redefined on the way from A = B | |||
1030 | /// in BB0 to B = A in BB2, B = A in BB2 is partially redundant if the | |||
1031 | /// execution goes through the path from BB0 to BB2. We may move B = A | |||
1032 | /// to the predecessor without such reversed copy. | |||
1033 | /// So we will transform the program from: | |||
1034 | /// BB0: | |||
1035 | /// A = B; BB1: | |||
1036 | /// ... ... | |||
1037 | /// / \ / | |||
1038 | /// BB2: | |||
1039 | /// ... | |||
1040 | /// B = A; | |||
1041 | /// | |||
1042 | /// to: | |||
1043 | /// | |||
1044 | /// BB0: BB1: | |||
1045 | /// A = B; ... | |||
1046 | /// ... B = A; | |||
1047 | /// / \ / | |||
1048 | /// BB2: | |||
1049 | /// ... | |||
1050 | /// | |||
1051 | /// A special case is when BB0 and BB2 are the same BB which is the only | |||
1052 | /// BB in a loop: | |||
1053 | /// BB1: | |||
1054 | /// ... | |||
1055 | /// BB0/BB2: ---- | |||
1056 | /// B = A; | | |||
1057 | /// ... | | |||
1058 | /// A = B; | | |||
1059 | /// |------- | |||
1060 | /// | | |||
1061 | /// We may hoist B = A from BB0/BB2 to BB1. | |||
1062 | /// | |||
1063 | /// The major preconditions for correctness to remove such partial | |||
1064 | /// redundancy include: | |||
1065 | /// 1. A in B = A in BB2 is defined by a PHI in BB2, and one operand of | |||
1066 | /// the PHI is defined by the reversed copy A = B in BB0. | |||
1067 | /// 2. No B is referenced from the start of BB2 to B = A. | |||
1068 | /// 3. No B is defined from A = B to the end of BB0. | |||
1069 | /// 4. BB1 has only one successor. | |||
1070 | /// | |||
1071 | /// 2 and 4 implicitly ensure B is not live at the end of BB1. | |||
1072 | /// 4 guarantees BB2 is hotter than BB1, so we can only move a copy to a | |||
1073 | /// colder place, which not only prevent endless loop, but also make sure | |||
1074 | /// the movement of copy is beneficial. | |||
1075 | bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP, | |||
1076 | MachineInstr &CopyMI) { | |||
1077 | assert(!CP.isPhys())((!CP.isPhys()) ? static_cast<void> (0) : __assert_fail ("!CP.isPhys()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1077, __PRETTY_FUNCTION__)); | |||
1078 | if (!CopyMI.isFullCopy()) | |||
1079 | return false; | |||
1080 | ||||
1081 | MachineBasicBlock &MBB = *CopyMI.getParent(); | |||
1082 | // If this block is the target of an invoke/inlineasm_br, moving the copy into | |||
1083 | // the predecessor is tricker, and we don't handle it. | |||
1084 | if (MBB.isEHPad() || MBB.isInlineAsmBrIndirectTarget()) | |||
1085 | return false; | |||
1086 | ||||
1087 | if (MBB.pred_size() != 2) | |||
1088 | return false; | |||
1089 | ||||
1090 | LiveInterval &IntA = | |||
1091 | LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); | |||
1092 | LiveInterval &IntB = | |||
1093 | LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); | |||
1094 | ||||
1095 | // A is defined by PHI at the entry of MBB. | |||
1096 | SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getRegSlot(true); | |||
1097 | VNInfo *AValNo = IntA.getVNInfoAt(CopyIdx); | |||
1098 | assert(AValNo && !AValNo->isUnused() && "COPY source not live")((AValNo && !AValNo->isUnused() && "COPY source not live" ) ? static_cast<void> (0) : __assert_fail ("AValNo && !AValNo->isUnused() && \"COPY source not live\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1098, __PRETTY_FUNCTION__)); | |||
1099 | if (!AValNo->isPHIDef()) | |||
1100 | return false; | |||
1101 | ||||
1102 | // No B is referenced before CopyMI in MBB. | |||
1103 | if (IntB.overlaps(LIS->getMBBStartIdx(&MBB), CopyIdx)) | |||
1104 | return false; | |||
1105 | ||||
1106 | // MBB has two predecessors: one contains A = B so no copy will be inserted | |||
1107 | // for it. The other one will have a copy moved from MBB. | |||
1108 | bool FoundReverseCopy = false; | |||
1109 | MachineBasicBlock *CopyLeftBB = nullptr; | |||
1110 | for (MachineBasicBlock *Pred : MBB.predecessors()) { | |||
1111 | VNInfo *PVal = IntA.getVNInfoBefore(LIS->getMBBEndIdx(Pred)); | |||
1112 | MachineInstr *DefMI = LIS->getInstructionFromIndex(PVal->def); | |||
1113 | if (!DefMI || !DefMI->isFullCopy()) { | |||
1114 | CopyLeftBB = Pred; | |||
1115 | continue; | |||
1116 | } | |||
1117 | // Check DefMI is a reverse copy and it is in BB Pred. | |||
1118 | if (DefMI->getOperand(0).getReg() != IntA.reg() || | |||
1119 | DefMI->getOperand(1).getReg() != IntB.reg() || | |||
1120 | DefMI->getParent() != Pred) { | |||
1121 | CopyLeftBB = Pred; | |||
1122 | continue; | |||
1123 | } | |||
1124 | // If there is any other def of B after DefMI and before the end of Pred, | |||
1125 | // we need to keep the copy of B = A at the end of Pred if we remove | |||
1126 | // B = A from MBB. | |||
1127 | bool ValB_Changed = false; | |||
1128 | for (auto VNI : IntB.valnos) { | |||
1129 | if (VNI->isUnused()) | |||
1130 | continue; | |||
1131 | if (PVal->def < VNI->def && VNI->def < LIS->getMBBEndIdx(Pred)) { | |||
1132 | ValB_Changed = true; | |||
1133 | break; | |||
1134 | } | |||
1135 | } | |||
1136 | if (ValB_Changed) { | |||
1137 | CopyLeftBB = Pred; | |||
1138 | continue; | |||
1139 | } | |||
1140 | FoundReverseCopy = true; | |||
1141 | } | |||
1142 | ||||
1143 | // If no reverse copy is found in predecessors, nothing to do. | |||
1144 | if (!FoundReverseCopy) | |||
1145 | return false; | |||
1146 | ||||
1147 | // If CopyLeftBB is nullptr, it means every predecessor of MBB contains | |||
1148 | // reverse copy, CopyMI can be removed trivially if only IntA/IntB is updated. | |||
1149 | // If CopyLeftBB is not nullptr, move CopyMI from MBB to CopyLeftBB and | |||
1150 | // update IntA/IntB. | |||
1151 | // | |||
1152 | // If CopyLeftBB is not nullptr, ensure CopyLeftBB has a single succ so | |||
1153 | // MBB is hotter than CopyLeftBB. | |||
1154 | if (CopyLeftBB && CopyLeftBB->succ_size() > 1) | |||
1155 | return false; | |||
1156 | ||||
1157 | // Now (almost sure it's) ok to move copy. | |||
1158 | if (CopyLeftBB) { | |||
1159 | // Position in CopyLeftBB where we should insert new copy. | |||
1160 | auto InsPos = CopyLeftBB->getFirstTerminator(); | |||
1161 | ||||
1162 | // Make sure that B isn't referenced in the terminators (if any) at the end | |||
1163 | // of the predecessor since we're about to insert a new definition of B | |||
1164 | // before them. | |||
1165 | if (InsPos != CopyLeftBB->end()) { | |||
1166 | SlotIndex InsPosIdx = LIS->getInstructionIndex(*InsPos).getRegSlot(true); | |||
1167 | if (IntB.overlaps(InsPosIdx, LIS->getMBBEndIdx(CopyLeftBB))) | |||
1168 | return false; | |||
1169 | } | |||
1170 | ||||
1171 | LLVM_DEBUG(dbgs() << "\tremovePartialRedundancy: Move the copy to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremovePartialRedundancy: Move the copy to " << printMBBReference(*CopyLeftBB) << '\t' << CopyMI; } } while (false) | |||
1172 | << printMBBReference(*CopyLeftBB) << '\t' << CopyMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremovePartialRedundancy: Move the copy to " << printMBBReference(*CopyLeftBB) << '\t' << CopyMI; } } while (false); | |||
1173 | ||||
1174 | // Insert new copy to CopyLeftBB. | |||
1175 | MachineInstr *NewCopyMI = BuildMI(*CopyLeftBB, InsPos, CopyMI.getDebugLoc(), | |||
1176 | TII->get(TargetOpcode::COPY), IntB.reg()) | |||
1177 | .addReg(IntA.reg()); | |||
1178 | SlotIndex NewCopyIdx = | |||
1179 | LIS->InsertMachineInstrInMaps(*NewCopyMI).getRegSlot(); | |||
1180 | IntB.createDeadDef(NewCopyIdx, LIS->getVNInfoAllocator()); | |||
1181 | for (LiveInterval::SubRange &SR : IntB.subranges()) | |||
1182 | SR.createDeadDef(NewCopyIdx, LIS->getVNInfoAllocator()); | |||
1183 | ||||
1184 | // If the newly created Instruction has an address of an instruction that was | |||
1185 | // deleted before (object recycled by the allocator) it needs to be removed from | |||
1186 | // the deleted list. | |||
1187 | ErasedInstrs.erase(NewCopyMI); | |||
1188 | } else { | |||
1189 | LLVM_DEBUG(dbgs() << "\tremovePartialRedundancy: Remove the copy from "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremovePartialRedundancy: Remove the copy from " << printMBBReference(MBB) << '\t' << CopyMI ; } } while (false) | |||
1190 | << printMBBReference(MBB) << '\t' << CopyMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremovePartialRedundancy: Remove the copy from " << printMBBReference(MBB) << '\t' << CopyMI ; } } while (false); | |||
1191 | } | |||
1192 | ||||
1193 | // Remove CopyMI. | |||
1194 | // Note: This is fine to remove the copy before updating the live-ranges. | |||
1195 | // While updating the live-ranges, we only look at slot indices and | |||
1196 | // never go back to the instruction. | |||
1197 | // Mark instructions as deleted. | |||
1198 | deleteInstr(&CopyMI); | |||
1199 | ||||
1200 | // Update the liveness. | |||
1201 | SmallVector<SlotIndex, 8> EndPoints; | |||
1202 | VNInfo *BValNo = IntB.Query(CopyIdx).valueOutOrDead(); | |||
1203 | LIS->pruneValue(*static_cast<LiveRange *>(&IntB), CopyIdx.getRegSlot(), | |||
1204 | &EndPoints); | |||
1205 | BValNo->markUnused(); | |||
1206 | // Extend IntB to the EndPoints of its original live interval. | |||
1207 | LIS->extendToIndices(IntB, EndPoints); | |||
1208 | ||||
1209 | // Now, do the same for its subranges. | |||
1210 | for (LiveInterval::SubRange &SR : IntB.subranges()) { | |||
1211 | EndPoints.clear(); | |||
1212 | VNInfo *BValNo = SR.Query(CopyIdx).valueOutOrDead(); | |||
1213 | assert(BValNo && "All sublanes should be live")((BValNo && "All sublanes should be live") ? static_cast <void> (0) : __assert_fail ("BValNo && \"All sublanes should be live\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1213, __PRETTY_FUNCTION__)); | |||
1214 | LIS->pruneValue(SR, CopyIdx.getRegSlot(), &EndPoints); | |||
1215 | BValNo->markUnused(); | |||
1216 | // We can have a situation where the result of the original copy is live, | |||
1217 | // but is immediately dead in this subrange, e.g. [336r,336d:0). That makes | |||
1218 | // the copy appear as an endpoint from pruneValue(), but we don't want it | |||
1219 | // to because the copy has been removed. We can go ahead and remove that | |||
1220 | // endpoint; there is no other situation here that there could be a use at | |||
1221 | // the same place as we know that the copy is a full copy. | |||
1222 | for (unsigned I = 0; I != EndPoints.size(); ) { | |||
1223 | if (SlotIndex::isSameInstr(EndPoints[I], CopyIdx)) { | |||
1224 | EndPoints[I] = EndPoints.back(); | |||
1225 | EndPoints.pop_back(); | |||
1226 | continue; | |||
1227 | } | |||
1228 | ++I; | |||
1229 | } | |||
1230 | SmallVector<SlotIndex, 8> Undefs; | |||
1231 | IntB.computeSubRangeUndefs(Undefs, SR.LaneMask, *MRI, | |||
1232 | *LIS->getSlotIndexes()); | |||
1233 | LIS->extendToIndices(SR, EndPoints, Undefs); | |||
1234 | } | |||
1235 | // If any dead defs were extended, truncate them. | |||
1236 | shrinkToUses(&IntB); | |||
1237 | ||||
1238 | // Finally, update the live-range of IntA. | |||
1239 | shrinkToUses(&IntA); | |||
1240 | return true; | |||
1241 | } | |||
1242 | ||||
1243 | /// Returns true if @p MI defines the full vreg @p Reg, as opposed to just | |||
1244 | /// defining a subregister. | |||
1245 | static bool definesFullReg(const MachineInstr &MI, Register Reg) { | |||
1246 | assert(!Reg.isPhysical() && "This code cannot handle physreg aliasing")((!Reg.isPhysical() && "This code cannot handle physreg aliasing" ) ? static_cast<void> (0) : __assert_fail ("!Reg.isPhysical() && \"This code cannot handle physreg aliasing\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1246, __PRETTY_FUNCTION__)); | |||
1247 | ||||
1248 | for (const MachineOperand &Op : MI.operands()) { | |||
1249 | if (!Op.isReg() || !Op.isDef() || Op.getReg() != Reg) | |||
1250 | continue; | |||
1251 | // Return true if we define the full register or don't care about the value | |||
1252 | // inside other subregisters. | |||
1253 | if (Op.getSubReg() == 0 || Op.isUndef()) | |||
1254 | return true; | |||
1255 | } | |||
1256 | return false; | |||
1257 | } | |||
1258 | ||||
1259 | bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP, | |||
1260 | MachineInstr *CopyMI, | |||
1261 | bool &IsDefCopy) { | |||
1262 | IsDefCopy = false; | |||
1263 | Register SrcReg = CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg(); | |||
1264 | unsigned SrcIdx = CP.isFlipped() ? CP.getDstIdx() : CP.getSrcIdx(); | |||
1265 | Register DstReg = CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg(); | |||
1266 | unsigned DstIdx = CP.isFlipped() ? CP.getSrcIdx() : CP.getDstIdx(); | |||
1267 | if (Register::isPhysicalRegister(SrcReg)) | |||
1268 | return false; | |||
1269 | ||||
1270 | LiveInterval &SrcInt = LIS->getInterval(SrcReg); | |||
1271 | SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI); | |||
1272 | VNInfo *ValNo = SrcInt.Query(CopyIdx).valueIn(); | |||
1273 | if (!ValNo) | |||
1274 | return false; | |||
1275 | if (ValNo->isPHIDef() || ValNo->isUnused()) | |||
1276 | return false; | |||
1277 | MachineInstr *DefMI = LIS->getInstructionFromIndex(ValNo->def); | |||
1278 | if (!DefMI) | |||
1279 | return false; | |||
1280 | if (DefMI->isCopyLike()) { | |||
1281 | IsDefCopy = true; | |||
1282 | return false; | |||
1283 | } | |||
1284 | if (!TII->isAsCheapAsAMove(*DefMI)) | |||
1285 | return false; | |||
1286 | if (!TII->isTriviallyReMaterializable(*DefMI, AA)) | |||
1287 | return false; | |||
1288 | if (!definesFullReg(*DefMI, SrcReg)) | |||
1289 | return false; | |||
1290 | bool SawStore = false; | |||
1291 | if (!DefMI->isSafeToMove(AA, SawStore)) | |||
1292 | return false; | |||
1293 | const MCInstrDesc &MCID = DefMI->getDesc(); | |||
1294 | if (MCID.getNumDefs() != 1) | |||
1295 | return false; | |||
1296 | // Only support subregister destinations when the def is read-undef. | |||
1297 | MachineOperand &DstOperand = CopyMI->getOperand(0); | |||
1298 | Register CopyDstReg = DstOperand.getReg(); | |||
1299 | if (DstOperand.getSubReg() && !DstOperand.isUndef()) | |||
1300 | return false; | |||
1301 | ||||
1302 | // If both SrcIdx and DstIdx are set, correct rematerialization would widen | |||
1303 | // the register substantially (beyond both source and dest size). This is bad | |||
1304 | // for performance since it can cascade through a function, introducing many | |||
1305 | // extra spills and fills (e.g. ARM can easily end up copying QQQQPR registers | |||
1306 | // around after a few subreg copies). | |||
1307 | if (SrcIdx && DstIdx) | |||
1308 | return false; | |||
1309 | ||||
1310 | const TargetRegisterClass *DefRC = TII->getRegClass(MCID, 0, TRI, *MF); | |||
1311 | if (!DefMI->isImplicitDef()) { | |||
1312 | if (DstReg.isPhysical()) { | |||
1313 | Register NewDstReg = DstReg; | |||
1314 | ||||
1315 | unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(), | |||
1316 | DefMI->getOperand(0).getSubReg()); | |||
1317 | if (NewDstIdx) | |||
1318 | NewDstReg = TRI->getSubReg(DstReg, NewDstIdx); | |||
1319 | ||||
1320 | // Finally, make sure that the physical subregister that will be | |||
1321 | // constructed later is permitted for the instruction. | |||
1322 | if (!DefRC->contains(NewDstReg)) | |||
1323 | return false; | |||
1324 | } else { | |||
1325 | // Theoretically, some stack frame reference could exist. Just make sure | |||
1326 | // it hasn't actually happened. | |||
1327 | assert(Register::isVirtualRegister(DstReg) &&((Register::isVirtualRegister(DstReg) && "Only expect to deal with virtual or physical registers" ) ? static_cast<void> (0) : __assert_fail ("Register::isVirtualRegister(DstReg) && \"Only expect to deal with virtual or physical registers\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1328, __PRETTY_FUNCTION__)) | |||
1328 | "Only expect to deal with virtual or physical registers")((Register::isVirtualRegister(DstReg) && "Only expect to deal with virtual or physical registers" ) ? static_cast<void> (0) : __assert_fail ("Register::isVirtualRegister(DstReg) && \"Only expect to deal with virtual or physical registers\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1328, __PRETTY_FUNCTION__)); | |||
1329 | } | |||
1330 | } | |||
1331 | ||||
1332 | DebugLoc DL = CopyMI->getDebugLoc(); | |||
1333 | MachineBasicBlock *MBB = CopyMI->getParent(); | |||
1334 | MachineBasicBlock::iterator MII = | |||
1335 | std::next(MachineBasicBlock::iterator(CopyMI)); | |||
1336 | TII->reMaterialize(*MBB, MII, DstReg, SrcIdx, *DefMI, *TRI); | |||
1337 | MachineInstr &NewMI = *std::prev(MII); | |||
1338 | NewMI.setDebugLoc(DL); | |||
1339 | ||||
1340 | // In a situation like the following: | |||
1341 | // %0:subreg = instr ; DefMI, subreg = DstIdx | |||
1342 | // %1 = copy %0:subreg ; CopyMI, SrcIdx = 0 | |||
1343 | // instead of widening %1 to the register class of %0 simply do: | |||
1344 | // %1 = instr | |||
1345 | const TargetRegisterClass *NewRC = CP.getNewRC(); | |||
1346 | if (DstIdx != 0) { | |||
1347 | MachineOperand &DefMO = NewMI.getOperand(0); | |||
1348 | if (DefMO.getSubReg() == DstIdx) { | |||
1349 | assert(SrcIdx == 0 && CP.isFlipped()((SrcIdx == 0 && CP.isFlipped() && "Shouldn't have SrcIdx+DstIdx at this point" ) ? static_cast<void> (0) : __assert_fail ("SrcIdx == 0 && CP.isFlipped() && \"Shouldn't have SrcIdx+DstIdx at this point\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1350, __PRETTY_FUNCTION__)) | |||
1350 | && "Shouldn't have SrcIdx+DstIdx at this point")((SrcIdx == 0 && CP.isFlipped() && "Shouldn't have SrcIdx+DstIdx at this point" ) ? static_cast<void> (0) : __assert_fail ("SrcIdx == 0 && CP.isFlipped() && \"Shouldn't have SrcIdx+DstIdx at this point\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1350, __PRETTY_FUNCTION__)); | |||
1351 | const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); | |||
1352 | const TargetRegisterClass *CommonRC = | |||
1353 | TRI->getCommonSubClass(DefRC, DstRC); | |||
1354 | if (CommonRC != nullptr) { | |||
1355 | NewRC = CommonRC; | |||
1356 | DstIdx = 0; | |||
1357 | DefMO.setSubReg(0); | |||
1358 | DefMO.setIsUndef(false); // Only subregs can have def+undef. | |||
1359 | } | |||
1360 | } | |||
1361 | } | |||
1362 | ||||
1363 | // CopyMI may have implicit operands, save them so that we can transfer them | |||
1364 | // over to the newly materialized instruction after CopyMI is removed. | |||
1365 | SmallVector<MachineOperand, 4> ImplicitOps; | |||
1366 | ImplicitOps.reserve(CopyMI->getNumOperands() - | |||
1367 | CopyMI->getDesc().getNumOperands()); | |||
1368 | for (unsigned I = CopyMI->getDesc().getNumOperands(), | |||
1369 | E = CopyMI->getNumOperands(); | |||
1370 | I != E; ++I) { | |||
1371 | MachineOperand &MO = CopyMI->getOperand(I); | |||
1372 | if (MO.isReg()) { | |||
1373 | assert(MO.isImplicit() && "No explicit operands after implicit operands.")((MO.isImplicit() && "No explicit operands after implicit operands." ) ? static_cast<void> (0) : __assert_fail ("MO.isImplicit() && \"No explicit operands after implicit operands.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1373, __PRETTY_FUNCTION__)); | |||
1374 | // Discard VReg implicit defs. | |||
1375 | if (Register::isPhysicalRegister(MO.getReg())) | |||
1376 | ImplicitOps.push_back(MO); | |||
1377 | } | |||
1378 | } | |||
1379 | ||||
1380 | LIS->ReplaceMachineInstrInMaps(*CopyMI, NewMI); | |||
1381 | CopyMI->eraseFromParent(); | |||
1382 | ErasedInstrs.insert(CopyMI); | |||
1383 | ||||
1384 | // NewMI may have dead implicit defs (E.g. EFLAGS for MOV<bits>r0 on X86). | |||
1385 | // We need to remember these so we can add intervals once we insert | |||
1386 | // NewMI into SlotIndexes. | |||
1387 | SmallVector<MCRegister, 4> NewMIImplDefs; | |||
1388 | for (unsigned i = NewMI.getDesc().getNumOperands(), | |||
1389 | e = NewMI.getNumOperands(); | |||
1390 | i != e; ++i) { | |||
1391 | MachineOperand &MO = NewMI.getOperand(i); | |||
1392 | if (MO.isReg() && MO.isDef()) { | |||
1393 | assert(MO.isImplicit() && MO.isDead() &&((MO.isImplicit() && MO.isDead() && Register:: isPhysicalRegister(MO.getReg())) ? static_cast<void> (0 ) : __assert_fail ("MO.isImplicit() && MO.isDead() && Register::isPhysicalRegister(MO.getReg())" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1394, __PRETTY_FUNCTION__)) | |||
1394 | Register::isPhysicalRegister(MO.getReg()))((MO.isImplicit() && MO.isDead() && Register:: isPhysicalRegister(MO.getReg())) ? static_cast<void> (0 ) : __assert_fail ("MO.isImplicit() && MO.isDead() && Register::isPhysicalRegister(MO.getReg())" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1394, __PRETTY_FUNCTION__)); | |||
1395 | NewMIImplDefs.push_back(MO.getReg().asMCReg()); | |||
1396 | } | |||
1397 | } | |||
1398 | ||||
1399 | if (DstReg.isVirtual()) { | |||
1400 | unsigned NewIdx = NewMI.getOperand(0).getSubReg(); | |||
1401 | ||||
1402 | if (DefRC != nullptr) { | |||
1403 | if (NewIdx) | |||
1404 | NewRC = TRI->getMatchingSuperRegClass(NewRC, DefRC, NewIdx); | |||
1405 | else | |||
1406 | NewRC = TRI->getCommonSubClass(NewRC, DefRC); | |||
1407 | assert(NewRC && "subreg chosen for remat incompatible with instruction")((NewRC && "subreg chosen for remat incompatible with instruction" ) ? static_cast<void> (0) : __assert_fail ("NewRC && \"subreg chosen for remat incompatible with instruction\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1407, __PRETTY_FUNCTION__)); | |||
1408 | } | |||
1409 | // Remap subranges to new lanemask and change register class. | |||
1410 | LiveInterval &DstInt = LIS->getInterval(DstReg); | |||
1411 | for (LiveInterval::SubRange &SR : DstInt.subranges()) { | |||
1412 | SR.LaneMask = TRI->composeSubRegIndexLaneMask(DstIdx, SR.LaneMask); | |||
1413 | } | |||
1414 | MRI->setRegClass(DstReg, NewRC); | |||
1415 | ||||
1416 | // Update machine operands and add flags. | |||
1417 | updateRegDefsUses(DstReg, DstReg, DstIdx); | |||
1418 | NewMI.getOperand(0).setSubReg(NewIdx); | |||
1419 | // updateRegDefUses can add an "undef" flag to the definition, since | |||
1420 | // it will replace DstReg with DstReg.DstIdx. If NewIdx is 0, make | |||
1421 | // sure that "undef" is not set. | |||
1422 | if (NewIdx == 0) | |||
1423 | NewMI.getOperand(0).setIsUndef(false); | |||
1424 | // Add dead subregister definitions if we are defining the whole register | |||
1425 | // but only part of it is live. | |||
1426 | // This could happen if the rematerialization instruction is rematerializing | |||
1427 | // more than actually is used in the register. | |||
1428 | // An example would be: | |||
1429 | // %1 = LOAD CONSTANTS 5, 8 ; Loading both 5 and 8 in different subregs | |||
1430 | // ; Copying only part of the register here, but the rest is undef. | |||
1431 | // %2:sub_16bit<def, read-undef> = COPY %1:sub_16bit | |||
1432 | // ==> | |||
1433 | // ; Materialize all the constants but only using one | |||
1434 | // %2 = LOAD_CONSTANTS 5, 8 | |||
1435 | // | |||
1436 | // at this point for the part that wasn't defined before we could have | |||
1437 | // subranges missing the definition. | |||
1438 | if (NewIdx == 0 && DstInt.hasSubRanges()) { | |||
1439 | SlotIndex CurrIdx = LIS->getInstructionIndex(NewMI); | |||
1440 | SlotIndex DefIndex = | |||
1441 | CurrIdx.getRegSlot(NewMI.getOperand(0).isEarlyClobber()); | |||
1442 | LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(DstReg); | |||
1443 | VNInfo::Allocator& Alloc = LIS->getVNInfoAllocator(); | |||
1444 | for (LiveInterval::SubRange &SR : DstInt.subranges()) { | |||
1445 | if (!SR.liveAt(DefIndex)) | |||
1446 | SR.createDeadDef(DefIndex, Alloc); | |||
1447 | MaxMask &= ~SR.LaneMask; | |||
1448 | } | |||
1449 | if (MaxMask.any()) { | |||
1450 | LiveInterval::SubRange *SR = DstInt.createSubRange(Alloc, MaxMask); | |||
1451 | SR->createDeadDef(DefIndex, Alloc); | |||
1452 | } | |||
1453 | } | |||
1454 | ||||
1455 | // Make sure that the subrange for resultant undef is removed | |||
1456 | // For example: | |||
1457 | // %1:sub1<def,read-undef> = LOAD CONSTANT 1 | |||
1458 | // %2 = COPY %1 | |||
1459 | // ==> | |||
1460 | // %2:sub1<def, read-undef> = LOAD CONSTANT 1 | |||
1461 | // ; Correct but need to remove the subrange for %2:sub0 | |||
1462 | // ; as it is now undef | |||
1463 | if (NewIdx != 0 && DstInt.hasSubRanges()) { | |||
1464 | // The affected subregister segments can be removed. | |||
1465 | SlotIndex CurrIdx = LIS->getInstructionIndex(NewMI); | |||
1466 | LaneBitmask DstMask = TRI->getSubRegIndexLaneMask(NewIdx); | |||
1467 | bool UpdatedSubRanges = false; | |||
1468 | SlotIndex DefIndex = | |||
1469 | CurrIdx.getRegSlot(NewMI.getOperand(0).isEarlyClobber()); | |||
1470 | VNInfo::Allocator &Alloc = LIS->getVNInfoAllocator(); | |||
1471 | for (LiveInterval::SubRange &SR : DstInt.subranges()) { | |||
1472 | if ((SR.LaneMask & DstMask).none()) { | |||
1473 | LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Removing undefined SubRange " << PrintLaneMask(SR.LaneMask) << " : " << SR << "\n"; } } while (false) | |||
1474 | << "Removing undefined SubRange "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Removing undefined SubRange " << PrintLaneMask(SR.LaneMask) << " : " << SR << "\n"; } } while (false) | |||
1475 | << PrintLaneMask(SR.LaneMask) << " : " << SR << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Removing undefined SubRange " << PrintLaneMask(SR.LaneMask) << " : " << SR << "\n"; } } while (false); | |||
1476 | // VNI is in ValNo - remove any segments in this SubRange that have this ValNo | |||
1477 | if (VNInfo *RmValNo = SR.getVNInfoAt(CurrIdx.getRegSlot())) { | |||
1478 | SR.removeValNo(RmValNo); | |||
1479 | UpdatedSubRanges = true; | |||
1480 | } | |||
1481 | } else { | |||
1482 | // We know that this lane is defined by this instruction, | |||
1483 | // but at this point it may be empty because it is not used by | |||
1484 | // anything. This happens when updateRegDefUses adds the missing | |||
1485 | // lanes. Assign that lane a dead def so that the interferences | |||
1486 | // are properly modeled. | |||
1487 | if (SR.empty()) | |||
1488 | SR.createDeadDef(DefIndex, Alloc); | |||
1489 | } | |||
1490 | } | |||
1491 | if (UpdatedSubRanges) | |||
1492 | DstInt.removeEmptySubRanges(); | |||
1493 | } | |||
1494 | } else if (NewMI.getOperand(0).getReg() != CopyDstReg) { | |||
1495 | // The New instruction may be defining a sub-register of what's actually | |||
1496 | // been asked for. If so it must implicitly define the whole thing. | |||
1497 | assert(Register::isPhysicalRegister(DstReg) &&((Register::isPhysicalRegister(DstReg) && "Only expect virtual or physical registers in remat" ) ? static_cast<void> (0) : __assert_fail ("Register::isPhysicalRegister(DstReg) && \"Only expect virtual or physical registers in remat\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1498, __PRETTY_FUNCTION__)) | |||
1498 | "Only expect virtual or physical registers in remat")((Register::isPhysicalRegister(DstReg) && "Only expect virtual or physical registers in remat" ) ? static_cast<void> (0) : __assert_fail ("Register::isPhysicalRegister(DstReg) && \"Only expect virtual or physical registers in remat\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1498, __PRETTY_FUNCTION__)); | |||
1499 | NewMI.getOperand(0).setIsDead(true); | |||
1500 | NewMI.addOperand(MachineOperand::CreateReg( | |||
1501 | CopyDstReg, true /*IsDef*/, true /*IsImp*/, false /*IsKill*/)); | |||
1502 | // Record small dead def live-ranges for all the subregisters | |||
1503 | // of the destination register. | |||
1504 | // Otherwise, variables that live through may miss some | |||
1505 | // interferences, thus creating invalid allocation. | |||
1506 | // E.g., i386 code: | |||
1507 | // %1 = somedef ; %1 GR8 | |||
1508 | // %2 = remat ; %2 GR32 | |||
1509 | // CL = COPY %2.sub_8bit | |||
1510 | // = somedef %1 ; %1 GR8 | |||
1511 | // => | |||
1512 | // %1 = somedef ; %1 GR8 | |||
1513 | // dead ECX = remat ; implicit-def CL | |||
1514 | // = somedef %1 ; %1 GR8 | |||
1515 | // %1 will see the interferences with CL but not with CH since | |||
1516 | // no live-ranges would have been created for ECX. | |||
1517 | // Fix that! | |||
1518 | SlotIndex NewMIIdx = LIS->getInstructionIndex(NewMI); | |||
1519 | for (MCRegUnitIterator Units(NewMI.getOperand(0).getReg(), TRI); | |||
1520 | Units.isValid(); ++Units) | |||
1521 | if (LiveRange *LR = LIS->getCachedRegUnit(*Units)) | |||
1522 | LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator()); | |||
1523 | } | |||
1524 | ||||
1525 | if (NewMI.getOperand(0).getSubReg()) | |||
1526 | NewMI.getOperand(0).setIsUndef(); | |||
1527 | ||||
1528 | // Transfer over implicit operands to the rematerialized instruction. | |||
1529 | for (MachineOperand &MO : ImplicitOps) | |||
1530 | NewMI.addOperand(MO); | |||
1531 | ||||
1532 | SlotIndex NewMIIdx = LIS->getInstructionIndex(NewMI); | |||
1533 | for (unsigned i = 0, e = NewMIImplDefs.size(); i != e; ++i) { | |||
1534 | MCRegister Reg = NewMIImplDefs[i]; | |||
1535 | for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) | |||
1536 | if (LiveRange *LR = LIS->getCachedRegUnit(*Units)) | |||
1537 | LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator()); | |||
1538 | } | |||
1539 | ||||
1540 | LLVM_DEBUG(dbgs() << "Remat: " << NewMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Remat: " << NewMI; } } while (false); | |||
1541 | ++NumReMats; | |||
1542 | ||||
1543 | // If the virtual SrcReg is completely eliminated, update all DBG_VALUEs | |||
1544 | // to describe DstReg instead. | |||
1545 | if (MRI->use_nodbg_empty(SrcReg)) { | |||
1546 | for (MachineOperand &UseMO : MRI->use_operands(SrcReg)) { | |||
1547 | MachineInstr *UseMI = UseMO.getParent(); | |||
1548 | if (UseMI->isDebugValue()) { | |||
1549 | if (Register::isPhysicalRegister(DstReg)) | |||
1550 | UseMO.substPhysReg(DstReg, *TRI); | |||
1551 | else | |||
1552 | UseMO.setReg(DstReg); | |||
1553 | // Move the debug value directly after the def of the rematerialized | |||
1554 | // value in DstReg. | |||
1555 | MBB->splice(std::next(NewMI.getIterator()), UseMI->getParent(), UseMI); | |||
1556 | LLVM_DEBUG(dbgs() << "\t\tupdated: " << *UseMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tupdated: " << *UseMI ; } } while (false); | |||
1557 | } | |||
1558 | } | |||
1559 | } | |||
1560 | ||||
1561 | if (ToBeUpdated.count(SrcReg)) | |||
1562 | return true; | |||
1563 | ||||
1564 | unsigned NumCopyUses = 0; | |||
1565 | for (MachineOperand &UseMO : MRI->use_nodbg_operands(SrcReg)) { | |||
1566 | if (UseMO.getParent()->isCopyLike()) | |||
1567 | NumCopyUses++; | |||
1568 | } | |||
1569 | if (NumCopyUses < LateRematUpdateThreshold) { | |||
1570 | // The source interval can become smaller because we removed a use. | |||
1571 | shrinkToUses(&SrcInt, &DeadDefs); | |||
1572 | if (!DeadDefs.empty()) | |||
1573 | eliminateDeadDefs(); | |||
1574 | } else { | |||
1575 | ToBeUpdated.insert(SrcReg); | |||
1576 | } | |||
1577 | return true; | |||
1578 | } | |||
1579 | ||||
1580 | MachineInstr *RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) { | |||
1581 | // ProcessImplicitDefs may leave some copies of <undef> values, it only | |||
1582 | // removes local variables. When we have a copy like: | |||
1583 | // | |||
1584 | // %1 = COPY undef %2 | |||
1585 | // | |||
1586 | // We delete the copy and remove the corresponding value number from %1. | |||
1587 | // Any uses of that value number are marked as <undef>. | |||
1588 | ||||
1589 | // Note that we do not query CoalescerPair here but redo isMoveInstr as the | |||
1590 | // CoalescerPair may have a new register class with adjusted subreg indices | |||
1591 | // at this point. | |||
1592 | Register SrcReg, DstReg; | |||
1593 | unsigned SrcSubIdx, DstSubIdx; | |||
1594 | if(!isMoveInstr(*TRI, CopyMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx)) | |||
1595 | return nullptr; | |||
1596 | ||||
1597 | SlotIndex Idx = LIS->getInstructionIndex(*CopyMI); | |||
1598 | const LiveInterval &SrcLI = LIS->getInterval(SrcReg); | |||
1599 | // CopyMI is undef iff SrcReg is not live before the instruction. | |||
1600 | if (SrcSubIdx != 0 && SrcLI.hasSubRanges()) { | |||
1601 | LaneBitmask SrcMask = TRI->getSubRegIndexLaneMask(SrcSubIdx); | |||
1602 | for (const LiveInterval::SubRange &SR : SrcLI.subranges()) { | |||
1603 | if ((SR.LaneMask & SrcMask).none()) | |||
1604 | continue; | |||
1605 | if (SR.liveAt(Idx)) | |||
1606 | return nullptr; | |||
1607 | } | |||
1608 | } else if (SrcLI.liveAt(Idx)) | |||
1609 | return nullptr; | |||
1610 | ||||
1611 | // If the undef copy defines a live-out value (i.e. an input to a PHI def), | |||
1612 | // then replace it with an IMPLICIT_DEF. | |||
1613 | LiveInterval &DstLI = LIS->getInterval(DstReg); | |||
1614 | SlotIndex RegIndex = Idx.getRegSlot(); | |||
1615 | LiveRange::Segment *Seg = DstLI.getSegmentContaining(RegIndex); | |||
1616 | assert(Seg != nullptr && "No segment for defining instruction")((Seg != nullptr && "No segment for defining instruction" ) ? static_cast<void> (0) : __assert_fail ("Seg != nullptr && \"No segment for defining instruction\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1616, __PRETTY_FUNCTION__)); | |||
1617 | if (VNInfo *V = DstLI.getVNInfoAt(Seg->end)) { | |||
1618 | if (V->isPHIDef()) { | |||
1619 | CopyMI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); | |||
1620 | for (unsigned i = CopyMI->getNumOperands(); i != 0; --i) { | |||
1621 | MachineOperand &MO = CopyMI->getOperand(i-1); | |||
1622 | if (MO.isReg() && MO.isUse()) | |||
1623 | CopyMI->RemoveOperand(i-1); | |||
1624 | } | |||
1625 | LLVM_DEBUG(dbgs() << "\tReplaced copy of <undef> value with an "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tReplaced copy of <undef> value with an " "implicit def\n"; } } while (false) | |||
1626 | "implicit def\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tReplaced copy of <undef> value with an " "implicit def\n"; } } while (false); | |||
1627 | return CopyMI; | |||
1628 | } | |||
1629 | } | |||
1630 | ||||
1631 | // Remove any DstReg segments starting at the instruction. | |||
1632 | LLVM_DEBUG(dbgs() << "\tEliminating copy of <undef> value\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tEliminating copy of <undef> value\n" ; } } while (false); | |||
1633 | ||||
1634 | // Remove value or merge with previous one in case of a subregister def. | |||
1635 | if (VNInfo *PrevVNI = DstLI.getVNInfoAt(Idx)) { | |||
1636 | VNInfo *VNI = DstLI.getVNInfoAt(RegIndex); | |||
1637 | DstLI.MergeValueNumberInto(VNI, PrevVNI); | |||
1638 | ||||
1639 | // The affected subregister segments can be removed. | |||
1640 | LaneBitmask DstMask = TRI->getSubRegIndexLaneMask(DstSubIdx); | |||
1641 | for (LiveInterval::SubRange &SR : DstLI.subranges()) { | |||
1642 | if ((SR.LaneMask & DstMask).none()) | |||
1643 | continue; | |||
1644 | ||||
1645 | VNInfo *SVNI = SR.getVNInfoAt(RegIndex); | |||
1646 | assert(SVNI != nullptr && SlotIndex::isSameInstr(SVNI->def, RegIndex))((SVNI != nullptr && SlotIndex::isSameInstr(SVNI-> def, RegIndex)) ? static_cast<void> (0) : __assert_fail ("SVNI != nullptr && SlotIndex::isSameInstr(SVNI->def, RegIndex)" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1646, __PRETTY_FUNCTION__)); | |||
1647 | SR.removeValNo(SVNI); | |||
1648 | } | |||
1649 | DstLI.removeEmptySubRanges(); | |||
1650 | } else | |||
1651 | LIS->removeVRegDefAt(DstLI, RegIndex); | |||
1652 | ||||
1653 | // Mark uses as undef. | |||
1654 | for (MachineOperand &MO : MRI->reg_nodbg_operands(DstReg)) { | |||
1655 | if (MO.isDef() /*|| MO.isUndef()*/) | |||
1656 | continue; | |||
1657 | const MachineInstr &MI = *MO.getParent(); | |||
1658 | SlotIndex UseIdx = LIS->getInstructionIndex(MI); | |||
1659 | LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(MO.getSubReg()); | |||
1660 | bool isLive; | |||
1661 | if (!UseMask.all() && DstLI.hasSubRanges()) { | |||
1662 | isLive = false; | |||
1663 | for (const LiveInterval::SubRange &SR : DstLI.subranges()) { | |||
1664 | if ((SR.LaneMask & UseMask).none()) | |||
1665 | continue; | |||
1666 | if (SR.liveAt(UseIdx)) { | |||
1667 | isLive = true; | |||
1668 | break; | |||
1669 | } | |||
1670 | } | |||
1671 | } else | |||
1672 | isLive = DstLI.liveAt(UseIdx); | |||
1673 | if (isLive) | |||
1674 | continue; | |||
1675 | MO.setIsUndef(true); | |||
1676 | LLVM_DEBUG(dbgs() << "\tnew undef: " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tnew undef: " << UseIdx << '\t' << MI; } } while (false); | |||
1677 | } | |||
1678 | ||||
1679 | // A def of a subregister may be a use of the other subregisters, so | |||
1680 | // deleting a def of a subregister may also remove uses. Since CopyMI | |||
1681 | // is still part of the function (but about to be erased), mark all | |||
1682 | // defs of DstReg in it as <undef>, so that shrinkToUses would | |||
1683 | // ignore them. | |||
1684 | for (MachineOperand &MO : CopyMI->operands()) | |||
1685 | if (MO.isReg() && MO.isDef() && MO.getReg() == DstReg) | |||
1686 | MO.setIsUndef(true); | |||
1687 | LIS->shrinkToUses(&DstLI); | |||
1688 | ||||
1689 | return CopyMI; | |||
1690 | } | |||
1691 | ||||
1692 | void RegisterCoalescer::addUndefFlag(const LiveInterval &Int, SlotIndex UseIdx, | |||
1693 | MachineOperand &MO, unsigned SubRegIdx) { | |||
1694 | LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubRegIdx); | |||
1695 | if (MO.isDef()) | |||
1696 | Mask = ~Mask; | |||
1697 | bool IsUndef = true; | |||
1698 | for (const LiveInterval::SubRange &S : Int.subranges()) { | |||
1699 | if ((S.LaneMask & Mask).none()) | |||
1700 | continue; | |||
1701 | if (S.liveAt(UseIdx)) { | |||
1702 | IsUndef = false; | |||
1703 | break; | |||
1704 | } | |||
1705 | } | |||
1706 | if (IsUndef) { | |||
1707 | MO.setIsUndef(true); | |||
1708 | // We found out some subregister use is actually reading an undefined | |||
1709 | // value. In some cases the whole vreg has become undefined at this | |||
1710 | // point so we have to potentially shrink the main range if the | |||
1711 | // use was ending a live segment there. | |||
1712 | LiveQueryResult Q = Int.Query(UseIdx); | |||
1713 | if (Q.valueOut() == nullptr) | |||
1714 | ShrinkMainRange = true; | |||
1715 | } | |||
1716 | } | |||
1717 | ||||
1718 | void RegisterCoalescer::updateRegDefsUses(Register SrcReg, Register DstReg, | |||
1719 | unsigned SubIdx) { | |||
1720 | bool DstIsPhys = Register::isPhysicalRegister(DstReg); | |||
1721 | LiveInterval *DstInt = DstIsPhys ? nullptr : &LIS->getInterval(DstReg); | |||
1722 | ||||
1723 | if (DstInt && DstInt->hasSubRanges() && DstReg != SrcReg) { | |||
1724 | for (MachineOperand &MO : MRI->reg_operands(DstReg)) { | |||
1725 | unsigned SubReg = MO.getSubReg(); | |||
1726 | if (SubReg == 0 || MO.isUndef()) | |||
1727 | continue; | |||
1728 | MachineInstr &MI = *MO.getParent(); | |||
1729 | if (MI.isDebugValue()) | |||
1730 | continue; | |||
1731 | SlotIndex UseIdx = LIS->getInstructionIndex(MI).getRegSlot(true); | |||
1732 | addUndefFlag(*DstInt, UseIdx, MO, SubReg); | |||
1733 | } | |||
1734 | } | |||
1735 | ||||
1736 | SmallPtrSet<MachineInstr*, 8> Visited; | |||
1737 | for (MachineRegisterInfo::reg_instr_iterator | |||
1738 | I = MRI->reg_instr_begin(SrcReg), E = MRI->reg_instr_end(); | |||
1739 | I != E; ) { | |||
1740 | MachineInstr *UseMI = &*(I++); | |||
1741 | ||||
1742 | // Each instruction can only be rewritten once because sub-register | |||
1743 | // composition is not always idempotent. When SrcReg != DstReg, rewriting | |||
1744 | // the UseMI operands removes them from the SrcReg use-def chain, but when | |||
1745 | // SrcReg is DstReg we could encounter UseMI twice if it has multiple | |||
1746 | // operands mentioning the virtual register. | |||
1747 | if (SrcReg == DstReg && !Visited.insert(UseMI).second) | |||
1748 | continue; | |||
1749 | ||||
1750 | SmallVector<unsigned,8> Ops; | |||
1751 | bool Reads, Writes; | |||
1752 | std::tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops); | |||
1753 | ||||
1754 | // If SrcReg wasn't read, it may still be the case that DstReg is live-in | |||
1755 | // because SrcReg is a sub-register. | |||
1756 | if (DstInt && !Reads && SubIdx && !UseMI->isDebugValue()) | |||
1757 | Reads = DstInt->liveAt(LIS->getInstructionIndex(*UseMI)); | |||
1758 | ||||
1759 | // Replace SrcReg with DstReg in all UseMI operands. | |||
1760 | for (unsigned i = 0, e = Ops.size(); i != e; ++i) { | |||
1761 | MachineOperand &MO = UseMI->getOperand(Ops[i]); | |||
1762 | ||||
1763 | // Adjust <undef> flags in case of sub-register joins. We don't want to | |||
1764 | // turn a full def into a read-modify-write sub-register def and vice | |||
1765 | // versa. | |||
1766 | if (SubIdx && MO.isDef()) | |||
1767 | MO.setIsUndef(!Reads); | |||
1768 | ||||
1769 | // A subreg use of a partially undef (super) register may be a complete | |||
1770 | // undef use now and then has to be marked that way. | |||
1771 | if (SubIdx != 0 && MO.isUse() && MRI->shouldTrackSubRegLiveness(DstReg)) { | |||
1772 | if (!DstInt->hasSubRanges()) { | |||
1773 | BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); | |||
1774 | LaneBitmask FullMask = MRI->getMaxLaneMaskForVReg(DstInt->reg()); | |||
1775 | LaneBitmask UsedLanes = TRI->getSubRegIndexLaneMask(SubIdx); | |||
1776 | LaneBitmask UnusedLanes = FullMask & ~UsedLanes; | |||
1777 | DstInt->createSubRangeFrom(Allocator, UsedLanes, *DstInt); | |||
1778 | // The unused lanes are just empty live-ranges at this point. | |||
1779 | // It is the caller responsibility to set the proper | |||
1780 | // dead segments if there is an actual dead def of the | |||
1781 | // unused lanes. This may happen with rematerialization. | |||
1782 | DstInt->createSubRange(Allocator, UnusedLanes); | |||
1783 | } | |||
1784 | SlotIndex MIIdx = UseMI->isDebugValue() | |||
1785 | ? LIS->getSlotIndexes()->getIndexBefore(*UseMI) | |||
1786 | : LIS->getInstructionIndex(*UseMI); | |||
1787 | SlotIndex UseIdx = MIIdx.getRegSlot(true); | |||
1788 | addUndefFlag(*DstInt, UseIdx, MO, SubIdx); | |||
1789 | } | |||
1790 | ||||
1791 | if (DstIsPhys) | |||
1792 | MO.substPhysReg(DstReg, *TRI); | |||
1793 | else | |||
1794 | MO.substVirtReg(DstReg, SubIdx, *TRI); | |||
1795 | } | |||
1796 | ||||
1797 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tupdated: "; if (!UseMI ->isDebugValue()) dbgs() << LIS->getInstructionIndex (*UseMI) << "\t"; dbgs() << *UseMI; }; } } while ( false) | |||
1798 | dbgs() << "\t\tupdated: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tupdated: "; if (!UseMI ->isDebugValue()) dbgs() << LIS->getInstructionIndex (*UseMI) << "\t"; dbgs() << *UseMI; }; } } while ( false) | |||
1799 | if (!UseMI->isDebugValue())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tupdated: "; if (!UseMI ->isDebugValue()) dbgs() << LIS->getInstructionIndex (*UseMI) << "\t"; dbgs() << *UseMI; }; } } while ( false) | |||
1800 | dbgs() << LIS->getInstructionIndex(*UseMI) << "\t";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tupdated: "; if (!UseMI ->isDebugValue()) dbgs() << LIS->getInstructionIndex (*UseMI) << "\t"; dbgs() << *UseMI; }; } } while ( false) | |||
1801 | dbgs() << *UseMI;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tupdated: "; if (!UseMI ->isDebugValue()) dbgs() << LIS->getInstructionIndex (*UseMI) << "\t"; dbgs() << *UseMI; }; } } while ( false) | |||
1802 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tupdated: "; if (!UseMI ->isDebugValue()) dbgs() << LIS->getInstructionIndex (*UseMI) << "\t"; dbgs() << *UseMI; }; } } while ( false); | |||
1803 | } | |||
1804 | } | |||
1805 | ||||
1806 | bool RegisterCoalescer::canJoinPhys(const CoalescerPair &CP) { | |||
1807 | // Always join simple intervals that are defined by a single copy from a | |||
1808 | // reserved register. This doesn't increase register pressure, so it is | |||
1809 | // always beneficial. | |||
1810 | if (!MRI->isReserved(CP.getDstReg())) { | |||
1811 | LLVM_DEBUG(dbgs() << "\tCan only merge into reserved registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tCan only merge into reserved registers.\n" ; } } while (false); | |||
1812 | return false; | |||
1813 | } | |||
1814 | ||||
1815 | LiveInterval &JoinVInt = LIS->getInterval(CP.getSrcReg()); | |||
1816 | if (JoinVInt.containsOneValue()) | |||
1817 | return true; | |||
1818 | ||||
1819 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tCannot join complex intervals into reserved register.\n" ; } } while (false) | |||
1820 | dbgs() << "\tCannot join complex intervals into reserved register.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tCannot join complex intervals into reserved register.\n" ; } } while (false); | |||
1821 | return false; | |||
1822 | } | |||
1823 | ||||
1824 | bool RegisterCoalescer::copyValueUndefInPredecessors( | |||
1825 | LiveRange &S, const MachineBasicBlock *MBB, LiveQueryResult SLRQ) { | |||
1826 | for (const MachineBasicBlock *Pred : MBB->predecessors()) { | |||
1827 | SlotIndex PredEnd = LIS->getMBBEndIdx(Pred); | |||
1828 | if (VNInfo *V = S.getVNInfoAt(PredEnd.getPrevSlot())) { | |||
1829 | // If this is a self loop, we may be reading the same value. | |||
1830 | if (V->id != SLRQ.valueOutOrDead()->id) | |||
1831 | return false; | |||
1832 | } | |||
1833 | } | |||
1834 | ||||
1835 | return true; | |||
1836 | } | |||
1837 | ||||
1838 | void RegisterCoalescer::setUndefOnPrunedSubRegUses(LiveInterval &LI, | |||
1839 | Register Reg, | |||
1840 | LaneBitmask PrunedLanes) { | |||
1841 | // If we had other instructions in the segment reading the undef sublane | |||
1842 | // value, we need to mark them with undef. | |||
1843 | for (MachineOperand &MO : MRI->use_nodbg_operands(Reg)) { | |||
1844 | unsigned SubRegIdx = MO.getSubReg(); | |||
1845 | if (SubRegIdx == 0 || MO.isUndef()) | |||
1846 | continue; | |||
1847 | ||||
1848 | LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(SubRegIdx); | |||
1849 | SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent()); | |||
1850 | for (LiveInterval::SubRange &S : LI.subranges()) { | |||
1851 | if (!S.liveAt(Pos) && (PrunedLanes & SubRegMask).any()) { | |||
1852 | MO.setIsUndef(); | |||
1853 | break; | |||
1854 | } | |||
1855 | } | |||
1856 | } | |||
1857 | ||||
1858 | LI.removeEmptySubRanges(); | |||
1859 | ||||
1860 | // A def of a subregister may be a use of other register lanes. Replacing | |||
1861 | // such a def with a def of a different register will eliminate the use, | |||
1862 | // and may cause the recorded live range to be larger than the actual | |||
1863 | // liveness in the program IR. | |||
1864 | LIS->shrinkToUses(&LI); | |||
1865 | } | |||
1866 | ||||
1867 | bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) { | |||
1868 | Again = false; | |||
1869 | LLVM_DEBUG(dbgs() << LIS->getInstructionIndex(*CopyMI) << '\t' << *CopyMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << LIS->getInstructionIndex(* CopyMI) << '\t' << *CopyMI; } } while (false); | |||
1870 | ||||
1871 | CoalescerPair CP(*TRI); | |||
1872 | if (!CP.setRegisters(CopyMI)) { | |||
1873 | LLVM_DEBUG(dbgs() << "\tNot coalescable.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tNot coalescable.\n"; } } while (false); | |||
1874 | return false; | |||
1875 | } | |||
1876 | ||||
1877 | if (CP.getNewRC()) { | |||
1878 | auto SrcRC = MRI->getRegClass(CP.getSrcReg()); | |||
1879 | auto DstRC = MRI->getRegClass(CP.getDstReg()); | |||
1880 | unsigned SrcIdx = CP.getSrcIdx(); | |||
1881 | unsigned DstIdx = CP.getDstIdx(); | |||
1882 | if (CP.isFlipped()) { | |||
1883 | std::swap(SrcIdx, DstIdx); | |||
1884 | std::swap(SrcRC, DstRC); | |||
1885 | } | |||
1886 | if (!TRI->shouldCoalesce(CopyMI, SrcRC, SrcIdx, DstRC, DstIdx, | |||
1887 | CP.getNewRC(), *LIS)) { | |||
1888 | LLVM_DEBUG(dbgs() << "\tSubtarget bailed on coalescing.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tSubtarget bailed on coalescing.\n" ; } } while (false); | |||
1889 | return false; | |||
1890 | } | |||
1891 | } | |||
1892 | ||||
1893 | // Dead code elimination. This really should be handled by MachineDCE, but | |||
1894 | // sometimes dead copies slip through, and we can't generate invalid live | |||
1895 | // ranges. | |||
1896 | if (!CP.isPhys() && CopyMI->allDefsAreDead()) { | |||
1897 | LLVM_DEBUG(dbgs() << "\tCopy is dead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tCopy is dead.\n"; } } while (false); | |||
1898 | DeadDefs.push_back(CopyMI); | |||
1899 | eliminateDeadDefs(); | |||
1900 | return true; | |||
1901 | } | |||
1902 | ||||
1903 | // Eliminate undefs. | |||
1904 | if (!CP.isPhys()) { | |||
1905 | // If this is an IMPLICIT_DEF, leave it alone, but don't try to coalesce. | |||
1906 | if (MachineInstr *UndefMI = eliminateUndefCopy(CopyMI)) { | |||
1907 | if (UndefMI->isImplicitDef()) | |||
1908 | return false; | |||
1909 | deleteInstr(CopyMI); | |||
1910 | return false; // Not coalescable. | |||
1911 | } | |||
1912 | } | |||
1913 | ||||
1914 | // Coalesced copies are normally removed immediately, but transformations | |||
1915 | // like removeCopyByCommutingDef() can inadvertently create identity copies. | |||
1916 | // When that happens, just join the values and remove the copy. | |||
1917 | if (CP.getSrcReg() == CP.getDstReg()) { | |||
1918 | LiveInterval &LI = LIS->getInterval(CP.getSrcReg()); | |||
1919 | LLVM_DEBUG(dbgs() << "\tCopy already coalesced: " << LI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tCopy already coalesced: " << LI << '\n'; } } while (false); | |||
1920 | const SlotIndex CopyIdx = LIS->getInstructionIndex(*CopyMI); | |||
1921 | LiveQueryResult LRQ = LI.Query(CopyIdx); | |||
1922 | if (VNInfo *DefVNI = LRQ.valueDefined()) { | |||
1923 | VNInfo *ReadVNI = LRQ.valueIn(); | |||
1924 | assert(ReadVNI && "No value before copy and no <undef> flag.")((ReadVNI && "No value before copy and no <undef> flag." ) ? static_cast<void> (0) : __assert_fail ("ReadVNI && \"No value before copy and no <undef> flag.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1924, __PRETTY_FUNCTION__)); | |||
1925 | assert(ReadVNI != DefVNI && "Cannot read and define the same value.")((ReadVNI != DefVNI && "Cannot read and define the same value." ) ? static_cast<void> (0) : __assert_fail ("ReadVNI != DefVNI && \"Cannot read and define the same value.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 1925, __PRETTY_FUNCTION__)); | |||
1926 | ||||
1927 | // Track incoming undef lanes we need to eliminate from the subrange. | |||
1928 | LaneBitmask PrunedLanes; | |||
1929 | MachineBasicBlock *MBB = CopyMI->getParent(); | |||
1930 | ||||
1931 | // Process subregister liveranges. | |||
1932 | for (LiveInterval::SubRange &S : LI.subranges()) { | |||
1933 | LiveQueryResult SLRQ = S.Query(CopyIdx); | |||
1934 | if (VNInfo *SDefVNI = SLRQ.valueDefined()) { | |||
1935 | if (VNInfo *SReadVNI = SLRQ.valueIn()) | |||
1936 | SDefVNI = S.MergeValueNumberInto(SDefVNI, SReadVNI); | |||
1937 | ||||
1938 | // If this copy introduced an undef subrange from an incoming value, | |||
1939 | // we need to eliminate the undef live in values from the subrange. | |||
1940 | if (copyValueUndefInPredecessors(S, MBB, SLRQ)) { | |||
1941 | LLVM_DEBUG(dbgs() << "Incoming sublane value is undef at copy\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Incoming sublane value is undef at copy\n" ; } } while (false); | |||
1942 | PrunedLanes |= S.LaneMask; | |||
1943 | S.removeValNo(SDefVNI); | |||
1944 | } | |||
1945 | } | |||
1946 | } | |||
1947 | ||||
1948 | LI.MergeValueNumberInto(DefVNI, ReadVNI); | |||
1949 | if (PrunedLanes.any()) { | |||
1950 | LLVM_DEBUG(dbgs() << "Pruning undef incoming lanes: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Pruning undef incoming lanes: " << PrunedLanes << '\n'; } } while (false) | |||
1951 | << PrunedLanes << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Pruning undef incoming lanes: " << PrunedLanes << '\n'; } } while (false); | |||
1952 | setUndefOnPrunedSubRegUses(LI, CP.getSrcReg(), PrunedLanes); | |||
1953 | } | |||
1954 | ||||
1955 | LLVM_DEBUG(dbgs() << "\tMerged values: " << LI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tMerged values: " << LI << '\n'; } } while (false); | |||
1956 | } | |||
1957 | deleteInstr(CopyMI); | |||
1958 | return true; | |||
1959 | } | |||
1960 | ||||
1961 | // Enforce policies. | |||
1962 | if (CP.isPhys()) { | |||
1963 | LLVM_DEBUG(dbgs() << "\tConsidering merging "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tConsidering merging " << printReg(CP.getSrcReg(), TRI) << " with " << printReg (CP.getDstReg(), TRI, CP.getSrcIdx()) << '\n'; } } while (false) | |||
1964 | << printReg(CP.getSrcReg(), TRI) << " with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tConsidering merging " << printReg(CP.getSrcReg(), TRI) << " with " << printReg (CP.getDstReg(), TRI, CP.getSrcIdx()) << '\n'; } } while (false) | |||
1965 | << printReg(CP.getDstReg(), TRI, CP.getSrcIdx()) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tConsidering merging " << printReg(CP.getSrcReg(), TRI) << " with " << printReg (CP.getDstReg(), TRI, CP.getSrcIdx()) << '\n'; } } while (false); | |||
1966 | if (!canJoinPhys(CP)) { | |||
1967 | // Before giving up coalescing, if definition of source is defined by | |||
1968 | // trivial computation, try rematerializing it. | |||
1969 | bool IsDefCopy; | |||
1970 | if (reMaterializeTrivialDef(CP, CopyMI, IsDefCopy)) | |||
1971 | return true; | |||
1972 | if (IsDefCopy) | |||
1973 | Again = true; // May be possible to coalesce later. | |||
1974 | return false; | |||
1975 | } | |||
1976 | } else { | |||
1977 | // When possible, let DstReg be the larger interval. | |||
1978 | if (!CP.isPartial() && LIS->getInterval(CP.getSrcReg()).size() > | |||
1979 | LIS->getInterval(CP.getDstReg()).size()) | |||
1980 | CP.flip(); | |||
1981 | ||||
1982 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1983 | dbgs() << "\tConsidering merging to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1984 | << TRI->getRegClassName(CP.getNewRC()) << " with ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1985 | if (CP.getDstIdx() && CP.getSrcIdx())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1986 | dbgs() << printReg(CP.getDstReg()) << " in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1987 | << TRI->getSubRegIndexName(CP.getDstIdx()) << " and "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1988 | << printReg(CP.getSrcReg()) << " in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1989 | << TRI->getSubRegIndexName(CP.getSrcIdx()) << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1990 | elsedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1991 | dbgs() << printReg(CP.getSrcReg(), TRI) << " in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1992 | << printReg(CP.getDstReg(), TRI, CP.getSrcIdx()) << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false) | |||
1993 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tConsidering merging to " << TRI->getRegClassName(CP.getNewRC()) << " with " ; if (CP.getDstIdx() && CP.getSrcIdx()) dbgs() << printReg(CP.getDstReg()) << " in " << TRI->getSubRegIndexName (CP.getDstIdx()) << " and " << printReg(CP.getSrcReg ()) << " in " << TRI->getSubRegIndexName(CP.getSrcIdx ()) << '\n'; else dbgs() << printReg(CP.getSrcReg (), TRI) << " in " << printReg(CP.getDstReg(), TRI , CP.getSrcIdx()) << '\n'; }; } } while (false); | |||
1994 | } | |||
1995 | ||||
1996 | ShrinkMask = LaneBitmask::getNone(); | |||
1997 | ShrinkMainRange = false; | |||
1998 | ||||
1999 | // Okay, attempt to join these two intervals. On failure, this returns false. | |||
2000 | // Otherwise, if one of the intervals being joined is a physreg, this method | |||
2001 | // always canonicalizes DstInt to be it. The output "SrcInt" will not have | |||
2002 | // been modified, so we can use this information below to update aliases. | |||
2003 | if (!joinIntervals(CP)) { | |||
2004 | // Coalescing failed. | |||
2005 | ||||
2006 | // If definition of source is defined by trivial computation, try | |||
2007 | // rematerializing it. | |||
2008 | bool IsDefCopy; | |||
2009 | if (reMaterializeTrivialDef(CP, CopyMI, IsDefCopy)) | |||
2010 | return true; | |||
2011 | ||||
2012 | // If we can eliminate the copy without merging the live segments, do so | |||
2013 | // now. | |||
2014 | if (!CP.isPartial() && !CP.isPhys()) { | |||
2015 | bool Changed = adjustCopiesBackFrom(CP, CopyMI); | |||
2016 | bool Shrink = false; | |||
2017 | if (!Changed) | |||
2018 | std::tie(Changed, Shrink) = removeCopyByCommutingDef(CP, CopyMI); | |||
2019 | if (Changed) { | |||
2020 | deleteInstr(CopyMI); | |||
2021 | if (Shrink) { | |||
2022 | Register DstReg = CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg(); | |||
2023 | LiveInterval &DstLI = LIS->getInterval(DstReg); | |||
2024 | shrinkToUses(&DstLI); | |||
2025 | LLVM_DEBUG(dbgs() << "\t\tshrunk: " << DstLI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tshrunk: " << DstLI << '\n'; } } while (false); | |||
2026 | } | |||
2027 | LLVM_DEBUG(dbgs() << "\tTrivial!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tTrivial!\n"; } } while (false ); | |||
2028 | return true; | |||
2029 | } | |||
2030 | } | |||
2031 | ||||
2032 | // Try and see if we can partially eliminate the copy by moving the copy to | |||
2033 | // its predecessor. | |||
2034 | if (!CP.isPartial() && !CP.isPhys()) | |||
2035 | if (removePartialRedundancy(CP, *CopyMI)) | |||
2036 | return true; | |||
2037 | ||||
2038 | // Otherwise, we are unable to join the intervals. | |||
2039 | LLVM_DEBUG(dbgs() << "\tInterference!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tInterference!\n"; } } while (false); | |||
2040 | Again = true; // May be possible to coalesce later. | |||
2041 | return false; | |||
2042 | } | |||
2043 | ||||
2044 | // Coalescing to a virtual register that is of a sub-register class of the | |||
2045 | // other. Make sure the resulting register is set to the right register class. | |||
2046 | if (CP.isCrossClass()) { | |||
2047 | ++numCrossRCs; | |||
2048 | MRI->setRegClass(CP.getDstReg(), CP.getNewRC()); | |||
2049 | } | |||
2050 | ||||
2051 | // Removing sub-register copies can ease the register class constraints. | |||
2052 | // Make sure we attempt to inflate the register class of DstReg. | |||
2053 | if (!CP.isPhys() && RegClassInfo.isProperSubClass(CP.getNewRC())) | |||
2054 | InflateRegs.push_back(CP.getDstReg()); | |||
2055 | ||||
2056 | // CopyMI has been erased by joinIntervals at this point. Remove it from | |||
2057 | // ErasedInstrs since copyCoalesceWorkList() won't add a successful join back | |||
2058 | // to the work list. This keeps ErasedInstrs from growing needlessly. | |||
2059 | ErasedInstrs.erase(CopyMI); | |||
2060 | ||||
2061 | // Rewrite all SrcReg operands to DstReg. | |||
2062 | // Also update DstReg operands to include DstIdx if it is set. | |||
2063 | if (CP.getDstIdx()) | |||
2064 | updateRegDefsUses(CP.getDstReg(), CP.getDstReg(), CP.getDstIdx()); | |||
2065 | updateRegDefsUses(CP.getSrcReg(), CP.getDstReg(), CP.getSrcIdx()); | |||
2066 | ||||
2067 | // Shrink subregister ranges if necessary. | |||
2068 | if (ShrinkMask.any()) { | |||
2069 | LiveInterval &LI = LIS->getInterval(CP.getDstReg()); | |||
2070 | for (LiveInterval::SubRange &S : LI.subranges()) { | |||
2071 | if ((S.LaneMask & ShrinkMask).none()) | |||
2072 | continue; | |||
2073 | LLVM_DEBUG(dbgs() << "Shrink LaneUses (Lane " << PrintLaneMask(S.LaneMask)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Shrink LaneUses (Lane " << PrintLaneMask(S.LaneMask) << ")\n"; } } while (false) | |||
2074 | << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Shrink LaneUses (Lane " << PrintLaneMask(S.LaneMask) << ")\n"; } } while (false); | |||
2075 | LIS->shrinkToUses(S, LI.reg()); | |||
2076 | } | |||
2077 | LI.removeEmptySubRanges(); | |||
2078 | } | |||
2079 | ||||
2080 | // CP.getSrcReg()'s live interval has been merged into CP.getDstReg's live | |||
2081 | // interval. Since CP.getSrcReg() is in ToBeUpdated set and its live interval | |||
2082 | // is not up-to-date, need to update the merged live interval here. | |||
2083 | if (ToBeUpdated.count(CP.getSrcReg())) | |||
2084 | ShrinkMainRange = true; | |||
2085 | ||||
2086 | if (ShrinkMainRange) { | |||
2087 | LiveInterval &LI = LIS->getInterval(CP.getDstReg()); | |||
2088 | shrinkToUses(&LI); | |||
2089 | } | |||
2090 | ||||
2091 | // SrcReg is guaranteed to be the register whose live interval that is | |||
2092 | // being merged. | |||
2093 | LIS->removeInterval(CP.getSrcReg()); | |||
2094 | ||||
2095 | // Update regalloc hint. | |||
2096 | TRI->updateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *MF); | |||
2097 | ||||
2098 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2099 | dbgs() << "\tSuccess: " << printReg(CP.getSrcReg(), TRI, CP.getSrcIdx())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2100 | << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2101 | dbgs() << "\tResult = ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2102 | if (CP.isPhys())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2103 | dbgs() << printReg(CP.getDstReg(), TRI);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2104 | elsedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2105 | dbgs() << LIS->getInterval(CP.getDstReg());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2106 | dbgs() << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false) | |||
2107 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\tSuccess: " << printReg (CP.getSrcReg(), TRI, CP.getSrcIdx()) << " -> " << printReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n'; dbgs() << "\tResult = "; if (CP.isPhys()) dbgs() << printReg(CP.getDstReg(), TRI); else dbgs() << LIS-> getInterval(CP.getDstReg()); dbgs() << '\n'; }; } } while (false); | |||
2108 | ||||
2109 | ++numJoins; | |||
2110 | return true; | |||
2111 | } | |||
2112 | ||||
2113 | bool RegisterCoalescer::joinReservedPhysReg(CoalescerPair &CP) { | |||
2114 | Register DstReg = CP.getDstReg(); | |||
2115 | Register SrcReg = CP.getSrcReg(); | |||
2116 | assert(CP.isPhys() && "Must be a physreg copy")((CP.isPhys() && "Must be a physreg copy") ? static_cast <void> (0) : __assert_fail ("CP.isPhys() && \"Must be a physreg copy\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2116, __PRETTY_FUNCTION__)); | |||
2117 | assert(MRI->isReserved(DstReg) && "Not a reserved register")((MRI->isReserved(DstReg) && "Not a reserved register" ) ? static_cast<void> (0) : __assert_fail ("MRI->isReserved(DstReg) && \"Not a reserved register\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2117, __PRETTY_FUNCTION__)); | |||
2118 | LiveInterval &RHS = LIS->getInterval(SrcReg); | |||
2119 | LLVM_DEBUG(dbgs() << "\t\tRHS = " << RHS << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tRHS = " << RHS << '\n'; } } while (false); | |||
2120 | ||||
2121 | assert(RHS.containsOneValue() && "Invalid join with reserved register")((RHS.containsOneValue() && "Invalid join with reserved register" ) ? static_cast<void> (0) : __assert_fail ("RHS.containsOneValue() && \"Invalid join with reserved register\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2121, __PRETTY_FUNCTION__)); | |||
2122 | ||||
2123 | // Optimization for reserved registers like ESP. We can only merge with a | |||
2124 | // reserved physreg if RHS has a single value that is a copy of DstReg. | |||
2125 | // The live range of the reserved register will look like a set of dead defs | |||
2126 | // - we don't properly track the live range of reserved registers. | |||
2127 | ||||
2128 | // Deny any overlapping intervals. This depends on all the reserved | |||
2129 | // register live ranges to look like dead defs. | |||
2130 | if (!MRI->isConstantPhysReg(DstReg)) { | |||
2131 | for (MCRegUnitIterator UI(DstReg, TRI); UI.isValid(); ++UI) { | |||
2132 | // Abort if not all the regunits are reserved. | |||
2133 | for (MCRegUnitRootIterator RI(*UI, TRI); RI.isValid(); ++RI) { | |||
2134 | if (!MRI->isReserved(*RI)) | |||
2135 | return false; | |||
2136 | } | |||
2137 | if (RHS.overlaps(LIS->getRegUnit(*UI))) { | |||
2138 | LLVM_DEBUG(dbgs() << "\t\tInterference: " << printRegUnit(*UI, TRI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tInterference: " << printRegUnit(*UI, TRI) << '\n'; } } while (false) | |||
2139 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tInterference: " << printRegUnit(*UI, TRI) << '\n'; } } while (false); | |||
2140 | return false; | |||
2141 | } | |||
2142 | } | |||
2143 | ||||
2144 | // We must also check for overlaps with regmask clobbers. | |||
2145 | BitVector RegMaskUsable; | |||
2146 | if (LIS->checkRegMaskInterference(RHS, RegMaskUsable) && | |||
2147 | !RegMaskUsable.test(DstReg)) { | |||
2148 | LLVM_DEBUG(dbgs() << "\t\tRegMask interference\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tRegMask interference\n"; } } while (false); | |||
2149 | return false; | |||
2150 | } | |||
2151 | } | |||
2152 | ||||
2153 | // Skip any value computations, we are not adding new values to the | |||
2154 | // reserved register. Also skip merging the live ranges, the reserved | |||
2155 | // register live range doesn't need to be accurate as long as all the | |||
2156 | // defs are there. | |||
2157 | ||||
2158 | // Delete the identity copy. | |||
2159 | MachineInstr *CopyMI; | |||
2160 | if (CP.isFlipped()) { | |||
2161 | // Physreg is copied into vreg | |||
2162 | // %y = COPY %physreg_x | |||
2163 | // ... //< no other def of %physreg_x here | |||
2164 | // use %y | |||
2165 | // => | |||
2166 | // ... | |||
2167 | // use %physreg_x | |||
2168 | CopyMI = MRI->getVRegDef(SrcReg); | |||
2169 | } else { | |||
2170 | // VReg is copied into physreg: | |||
2171 | // %y = def | |||
2172 | // ... //< no other def or use of %physreg_x here | |||
2173 | // %physreg_x = COPY %y | |||
2174 | // => | |||
2175 | // %physreg_x = def | |||
2176 | // ... | |||
2177 | if (!MRI->hasOneNonDBGUse(SrcReg)) { | |||
2178 | LLVM_DEBUG(dbgs() << "\t\tMultiple vreg uses!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tMultiple vreg uses!\n"; } } while (false); | |||
2179 | return false; | |||
2180 | } | |||
2181 | ||||
2182 | if (!LIS->intervalIsInOneMBB(RHS)) { | |||
2183 | LLVM_DEBUG(dbgs() << "\t\tComplex control flow!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tComplex control flow!\n" ; } } while (false); | |||
2184 | return false; | |||
2185 | } | |||
2186 | ||||
2187 | MachineInstr &DestMI = *MRI->getVRegDef(SrcReg); | |||
2188 | CopyMI = &*MRI->use_instr_nodbg_begin(SrcReg); | |||
2189 | SlotIndex CopyRegIdx = LIS->getInstructionIndex(*CopyMI).getRegSlot(); | |||
2190 | SlotIndex DestRegIdx = LIS->getInstructionIndex(DestMI).getRegSlot(); | |||
2191 | ||||
2192 | if (!MRI->isConstantPhysReg(DstReg)) { | |||
2193 | // We checked above that there are no interfering defs of the physical | |||
2194 | // register. However, for this case, where we intend to move up the def of | |||
2195 | // the physical register, we also need to check for interfering uses. | |||
2196 | SlotIndexes *Indexes = LIS->getSlotIndexes(); | |||
2197 | for (SlotIndex SI = Indexes->getNextNonNullIndex(DestRegIdx); | |||
2198 | SI != CopyRegIdx; SI = Indexes->getNextNonNullIndex(SI)) { | |||
2199 | MachineInstr *MI = LIS->getInstructionFromIndex(SI); | |||
2200 | if (MI->readsRegister(DstReg, TRI)) { | |||
2201 | LLVM_DEBUG(dbgs() << "\t\tInterference (read): " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tInterference (read): " << *MI; } } while (false); | |||
2202 | return false; | |||
2203 | } | |||
2204 | } | |||
2205 | } | |||
2206 | ||||
2207 | // We're going to remove the copy which defines a physical reserved | |||
2208 | // register, so remove its valno, etc. | |||
2209 | LLVM_DEBUG(dbgs() << "\t\tRemoving phys reg def of "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tRemoving phys reg def of " << printReg(DstReg, TRI) << " at " << CopyRegIdx << "\n"; } } while (false) | |||
2210 | << printReg(DstReg, TRI) << " at " << CopyRegIdx << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tRemoving phys reg def of " << printReg(DstReg, TRI) << " at " << CopyRegIdx << "\n"; } } while (false); | |||
2211 | ||||
2212 | LIS->removePhysRegDefAt(DstReg.asMCReg(), CopyRegIdx); | |||
2213 | // Create a new dead def at the new def location. | |||
2214 | for (MCRegUnitIterator UI(DstReg, TRI); UI.isValid(); ++UI) { | |||
2215 | LiveRange &LR = LIS->getRegUnit(*UI); | |||
2216 | LR.createDeadDef(DestRegIdx, LIS->getVNInfoAllocator()); | |||
2217 | } | |||
2218 | } | |||
2219 | ||||
2220 | deleteInstr(CopyMI); | |||
2221 | ||||
2222 | // We don't track kills for reserved registers. | |||
2223 | MRI->clearKillFlags(CP.getSrcReg()); | |||
2224 | ||||
2225 | return true; | |||
2226 | } | |||
2227 | ||||
2228 | //===----------------------------------------------------------------------===// | |||
2229 | // Interference checking and interval joining | |||
2230 | //===----------------------------------------------------------------------===// | |||
2231 | // | |||
2232 | // In the easiest case, the two live ranges being joined are disjoint, and | |||
2233 | // there is no interference to consider. It is quite common, though, to have | |||
2234 | // overlapping live ranges, and we need to check if the interference can be | |||
2235 | // resolved. | |||
2236 | // | |||
2237 | // The live range of a single SSA value forms a sub-tree of the dominator tree. | |||
2238 | // This means that two SSA values overlap if and only if the def of one value | |||
2239 | // is contained in the live range of the other value. As a special case, the | |||
2240 | // overlapping values can be defined at the same index. | |||
2241 | // | |||
2242 | // The interference from an overlapping def can be resolved in these cases: | |||
2243 | // | |||
2244 | // 1. Coalescable copies. The value is defined by a copy that would become an | |||
2245 | // identity copy after joining SrcReg and DstReg. The copy instruction will | |||
2246 | // be removed, and the value will be merged with the source value. | |||
2247 | // | |||
2248 | // There can be several copies back and forth, causing many values to be | |||
2249 | // merged into one. We compute a list of ultimate values in the joined live | |||
2250 | // range as well as a mappings from the old value numbers. | |||
2251 | // | |||
2252 | // 2. IMPLICIT_DEF. This instruction is only inserted to ensure all PHI | |||
2253 | // predecessors have a live out value. It doesn't cause real interference, | |||
2254 | // and can be merged into the value it overlaps. Like a coalescable copy, it | |||
2255 | // can be erased after joining. | |||
2256 | // | |||
2257 | // 3. Copy of external value. The overlapping def may be a copy of a value that | |||
2258 | // is already in the other register. This is like a coalescable copy, but | |||
2259 | // the live range of the source register must be trimmed after erasing the | |||
2260 | // copy instruction: | |||
2261 | // | |||
2262 | // %src = COPY %ext | |||
2263 | // %dst = COPY %ext <-- Remove this COPY, trim the live range of %ext. | |||
2264 | // | |||
2265 | // 4. Clobbering undefined lanes. Vector registers are sometimes built by | |||
2266 | // defining one lane at a time: | |||
2267 | // | |||
2268 | // %dst:ssub0<def,read-undef> = FOO | |||
2269 | // %src = BAR | |||
2270 | // %dst:ssub1 = COPY %src | |||
2271 | // | |||
2272 | // The live range of %src overlaps the %dst value defined by FOO, but | |||
2273 | // merging %src into %dst:ssub1 is only going to clobber the ssub1 lane | |||
2274 | // which was undef anyway. | |||
2275 | // | |||
2276 | // The value mapping is more complicated in this case. The final live range | |||
2277 | // will have different value numbers for both FOO and BAR, but there is no | |||
2278 | // simple mapping from old to new values. It may even be necessary to add | |||
2279 | // new PHI values. | |||
2280 | // | |||
2281 | // 5. Clobbering dead lanes. A def may clobber a lane of a vector register that | |||
2282 | // is live, but never read. This can happen because we don't compute | |||
2283 | // individual live ranges per lane. | |||
2284 | // | |||
2285 | // %dst = FOO | |||
2286 | // %src = BAR | |||
2287 | // %dst:ssub1 = COPY %src | |||
2288 | // | |||
2289 | // This kind of interference is only resolved locally. If the clobbered | |||
2290 | // lane value escapes the block, the join is aborted. | |||
2291 | ||||
2292 | namespace { | |||
2293 | ||||
2294 | /// Track information about values in a single virtual register about to be | |||
2295 | /// joined. Objects of this class are always created in pairs - one for each | |||
2296 | /// side of the CoalescerPair (or one for each lane of a side of the coalescer | |||
2297 | /// pair) | |||
2298 | class JoinVals { | |||
2299 | /// Live range we work on. | |||
2300 | LiveRange &LR; | |||
2301 | ||||
2302 | /// (Main) register we work on. | |||
2303 | const Register Reg; | |||
2304 | ||||
2305 | /// Reg (and therefore the values in this liverange) will end up as | |||
2306 | /// subregister SubIdx in the coalesced register. Either CP.DstIdx or | |||
2307 | /// CP.SrcIdx. | |||
2308 | const unsigned SubIdx; | |||
2309 | ||||
2310 | /// The LaneMask that this liverange will occupy the coalesced register. May | |||
2311 | /// be smaller than the lanemask produced by SubIdx when merging subranges. | |||
2312 | const LaneBitmask LaneMask; | |||
2313 | ||||
2314 | /// This is true when joining sub register ranges, false when joining main | |||
2315 | /// ranges. | |||
2316 | const bool SubRangeJoin; | |||
2317 | ||||
2318 | /// Whether the current LiveInterval tracks subregister liveness. | |||
2319 | const bool TrackSubRegLiveness; | |||
2320 | ||||
2321 | /// Values that will be present in the final live range. | |||
2322 | SmallVectorImpl<VNInfo*> &NewVNInfo; | |||
2323 | ||||
2324 | const CoalescerPair &CP; | |||
2325 | LiveIntervals *LIS; | |||
2326 | SlotIndexes *Indexes; | |||
2327 | const TargetRegisterInfo *TRI; | |||
2328 | ||||
2329 | /// Value number assignments. Maps value numbers in LI to entries in | |||
2330 | /// NewVNInfo. This is suitable for passing to LiveInterval::join(). | |||
2331 | SmallVector<int, 8> Assignments; | |||
2332 | ||||
2333 | public: | |||
2334 | /// Conflict resolution for overlapping values. | |||
2335 | enum ConflictResolution { | |||
2336 | /// No overlap, simply keep this value. | |||
2337 | CR_Keep, | |||
2338 | ||||
2339 | /// Merge this value into OtherVNI and erase the defining instruction. | |||
2340 | /// Used for IMPLICIT_DEF, coalescable copies, and copies from external | |||
2341 | /// values. | |||
2342 | CR_Erase, | |||
2343 | ||||
2344 | /// Merge this value into OtherVNI but keep the defining instruction. | |||
2345 | /// This is for the special case where OtherVNI is defined by the same | |||
2346 | /// instruction. | |||
2347 | CR_Merge, | |||
2348 | ||||
2349 | /// Keep this value, and have it replace OtherVNI where possible. This | |||
2350 | /// complicates value mapping since OtherVNI maps to two different values | |||
2351 | /// before and after this def. | |||
2352 | /// Used when clobbering undefined or dead lanes. | |||
2353 | CR_Replace, | |||
2354 | ||||
2355 | /// Unresolved conflict. Visit later when all values have been mapped. | |||
2356 | CR_Unresolved, | |||
2357 | ||||
2358 | /// Unresolvable conflict. Abort the join. | |||
2359 | CR_Impossible | |||
2360 | }; | |||
2361 | ||||
2362 | private: | |||
2363 | /// Per-value info for LI. The lane bit masks are all relative to the final | |||
2364 | /// joined register, so they can be compared directly between SrcReg and | |||
2365 | /// DstReg. | |||
2366 | struct Val { | |||
2367 | ConflictResolution Resolution = CR_Keep; | |||
2368 | ||||
2369 | /// Lanes written by this def, 0 for unanalyzed values. | |||
2370 | LaneBitmask WriteLanes; | |||
2371 | ||||
2372 | /// Lanes with defined values in this register. Other lanes are undef and | |||
2373 | /// safe to clobber. | |||
2374 | LaneBitmask ValidLanes; | |||
2375 | ||||
2376 | /// Value in LI being redefined by this def. | |||
2377 | VNInfo *RedefVNI = nullptr; | |||
2378 | ||||
2379 | /// Value in the other live range that overlaps this def, if any. | |||
2380 | VNInfo *OtherVNI = nullptr; | |||
2381 | ||||
2382 | /// Is this value an IMPLICIT_DEF that can be erased? | |||
2383 | /// | |||
2384 | /// IMPLICIT_DEF values should only exist at the end of a basic block that | |||
2385 | /// is a predecessor to a phi-value. These IMPLICIT_DEF instructions can be | |||
2386 | /// safely erased if they are overlapping a live value in the other live | |||
2387 | /// interval. | |||
2388 | /// | |||
2389 | /// Weird control flow graphs and incomplete PHI handling in | |||
2390 | /// ProcessImplicitDefs can very rarely create IMPLICIT_DEF values with | |||
2391 | /// longer live ranges. Such IMPLICIT_DEF values should be treated like | |||
2392 | /// normal values. | |||
2393 | bool ErasableImplicitDef = false; | |||
2394 | ||||
2395 | /// True when the live range of this value will be pruned because of an | |||
2396 | /// overlapping CR_Replace value in the other live range. | |||
2397 | bool Pruned = false; | |||
2398 | ||||
2399 | /// True once Pruned above has been computed. | |||
2400 | bool PrunedComputed = false; | |||
2401 | ||||
2402 | /// True if this value is determined to be identical to OtherVNI | |||
2403 | /// (in valuesIdentical). This is used with CR_Erase where the erased | |||
2404 | /// copy is redundant, i.e. the source value is already the same as | |||
2405 | /// the destination. In such cases the subranges need to be updated | |||
2406 | /// properly. See comment at pruneSubRegValues for more info. | |||
2407 | bool Identical = false; | |||
2408 | ||||
2409 | Val() = default; | |||
2410 | ||||
2411 | bool isAnalyzed() const { return WriteLanes.any(); } | |||
2412 | }; | |||
2413 | ||||
2414 | /// One entry per value number in LI. | |||
2415 | SmallVector<Val, 8> Vals; | |||
2416 | ||||
2417 | /// Compute the bitmask of lanes actually written by DefMI. | |||
2418 | /// Set Redef if there are any partial register definitions that depend on the | |||
2419 | /// previous value of the register. | |||
2420 | LaneBitmask computeWriteLanes(const MachineInstr *DefMI, bool &Redef) const; | |||
2421 | ||||
2422 | /// Find the ultimate value that VNI was copied from. | |||
2423 | std::pair<const VNInfo *, Register> followCopyChain(const VNInfo *VNI) const; | |||
2424 | ||||
2425 | bool valuesIdentical(VNInfo *Value0, VNInfo *Value1, const JoinVals &Other) const; | |||
2426 | ||||
2427 | /// Analyze ValNo in this live range, and set all fields of Vals[ValNo]. | |||
2428 | /// Return a conflict resolution when possible, but leave the hard cases as | |||
2429 | /// CR_Unresolved. | |||
2430 | /// Recursively calls computeAssignment() on this and Other, guaranteeing that | |||
2431 | /// both OtherVNI and RedefVNI have been analyzed and mapped before returning. | |||
2432 | /// The recursion always goes upwards in the dominator tree, making loops | |||
2433 | /// impossible. | |||
2434 | ConflictResolution analyzeValue(unsigned ValNo, JoinVals &Other); | |||
2435 | ||||
2436 | /// Compute the value assignment for ValNo in RI. | |||
2437 | /// This may be called recursively by analyzeValue(), but never for a ValNo on | |||
2438 | /// the stack. | |||
2439 | void computeAssignment(unsigned ValNo, JoinVals &Other); | |||
2440 | ||||
2441 | /// Assuming ValNo is going to clobber some valid lanes in Other.LR, compute | |||
2442 | /// the extent of the tainted lanes in the block. | |||
2443 | /// | |||
2444 | /// Multiple values in Other.LR can be affected since partial redefinitions | |||
2445 | /// can preserve previously tainted lanes. | |||
2446 | /// | |||
2447 | /// 1 %dst = VLOAD <-- Define all lanes in %dst | |||
2448 | /// 2 %src = FOO <-- ValNo to be joined with %dst:ssub0 | |||
2449 | /// 3 %dst:ssub1 = BAR <-- Partial redef doesn't clear taint in ssub0 | |||
2450 | /// 4 %dst:ssub0 = COPY %src <-- Conflict resolved, ssub0 wasn't read | |||
2451 | /// | |||
2452 | /// For each ValNo in Other that is affected, add an (EndIndex, TaintedLanes) | |||
2453 | /// entry to TaintedVals. | |||
2454 | /// | |||
2455 | /// Returns false if the tainted lanes extend beyond the basic block. | |||
2456 | bool | |||
2457 | taintExtent(unsigned ValNo, LaneBitmask TaintedLanes, JoinVals &Other, | |||
2458 | SmallVectorImpl<std::pair<SlotIndex, LaneBitmask>> &TaintExtent); | |||
2459 | ||||
2460 | /// Return true if MI uses any of the given Lanes from Reg. | |||
2461 | /// This does not include partial redefinitions of Reg. | |||
2462 | bool usesLanes(const MachineInstr &MI, Register, unsigned, LaneBitmask) const; | |||
2463 | ||||
2464 | /// Determine if ValNo is a copy of a value number in LR or Other.LR that will | |||
2465 | /// be pruned: | |||
2466 | /// | |||
2467 | /// %dst = COPY %src | |||
2468 | /// %src = COPY %dst <-- This value to be pruned. | |||
2469 | /// %dst = COPY %src <-- This value is a copy of a pruned value. | |||
2470 | bool isPrunedValue(unsigned ValNo, JoinVals &Other); | |||
2471 | ||||
2472 | public: | |||
2473 | JoinVals(LiveRange &LR, Register Reg, unsigned SubIdx, LaneBitmask LaneMask, | |||
2474 | SmallVectorImpl<VNInfo *> &newVNInfo, const CoalescerPair &cp, | |||
2475 | LiveIntervals *lis, const TargetRegisterInfo *TRI, bool SubRangeJoin, | |||
2476 | bool TrackSubRegLiveness) | |||
2477 | : LR(LR), Reg(Reg), SubIdx(SubIdx), LaneMask(LaneMask), | |||
2478 | SubRangeJoin(SubRangeJoin), TrackSubRegLiveness(TrackSubRegLiveness), | |||
2479 | NewVNInfo(newVNInfo), CP(cp), LIS(lis), Indexes(LIS->getSlotIndexes()), | |||
2480 | TRI(TRI), Assignments(LR.getNumValNums(), -1), | |||
2481 | Vals(LR.getNumValNums()) {} | |||
2482 | ||||
2483 | /// Analyze defs in LR and compute a value mapping in NewVNInfo. | |||
2484 | /// Returns false if any conflicts were impossible to resolve. | |||
2485 | bool mapValues(JoinVals &Other); | |||
2486 | ||||
2487 | /// Try to resolve conflicts that require all values to be mapped. | |||
2488 | /// Returns false if any conflicts were impossible to resolve. | |||
2489 | bool resolveConflicts(JoinVals &Other); | |||
2490 | ||||
2491 | /// Prune the live range of values in Other.LR where they would conflict with | |||
2492 | /// CR_Replace values in LR. Collect end points for restoring the live range | |||
2493 | /// after joining. | |||
2494 | void pruneValues(JoinVals &Other, SmallVectorImpl<SlotIndex> &EndPoints, | |||
2495 | bool changeInstrs); | |||
2496 | ||||
2497 | /// Removes subranges starting at copies that get removed. This sometimes | |||
2498 | /// happens when undefined subranges are copied around. These ranges contain | |||
2499 | /// no useful information and can be removed. | |||
2500 | void pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask); | |||
2501 | ||||
2502 | /// Pruning values in subranges can lead to removing segments in these | |||
2503 | /// subranges started by IMPLICIT_DEFs. The corresponding segments in | |||
2504 | /// the main range also need to be removed. This function will mark | |||
2505 | /// the corresponding values in the main range as pruned, so that | |||
2506 | /// eraseInstrs can do the final cleanup. | |||
2507 | /// The parameter @p LI must be the interval whose main range is the | |||
2508 | /// live range LR. | |||
2509 | void pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange); | |||
2510 | ||||
2511 | /// Erase any machine instructions that have been coalesced away. | |||
2512 | /// Add erased instructions to ErasedInstrs. | |||
2513 | /// Add foreign virtual registers to ShrinkRegs if their live range ended at | |||
2514 | /// the erased instrs. | |||
2515 | void eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs, | |||
2516 | SmallVectorImpl<Register> &ShrinkRegs, | |||
2517 | LiveInterval *LI = nullptr); | |||
2518 | ||||
2519 | /// Remove liverange defs at places where implicit defs will be removed. | |||
2520 | void removeImplicitDefs(); | |||
2521 | ||||
2522 | /// Get the value assignments suitable for passing to LiveInterval::join. | |||
2523 | const int *getAssignments() const { return Assignments.data(); } | |||
2524 | ||||
2525 | /// Get the conflict resolution for a value number. | |||
2526 | ConflictResolution getResolution(unsigned Num) const { | |||
2527 | return Vals[Num].Resolution; | |||
2528 | } | |||
2529 | }; | |||
2530 | ||||
2531 | } // end anonymous namespace | |||
2532 | ||||
2533 | LaneBitmask JoinVals::computeWriteLanes(const MachineInstr *DefMI, bool &Redef) | |||
2534 | const { | |||
2535 | LaneBitmask L; | |||
2536 | for (const MachineOperand &MO : DefMI->operands()) { | |||
2537 | if (!MO.isReg() || MO.getReg() != Reg || !MO.isDef()) | |||
2538 | continue; | |||
2539 | L |= TRI->getSubRegIndexLaneMask( | |||
2540 | TRI->composeSubRegIndices(SubIdx, MO.getSubReg())); | |||
2541 | if (MO.readsReg()) | |||
2542 | Redef = true; | |||
2543 | } | |||
2544 | return L; | |||
2545 | } | |||
2546 | ||||
2547 | std::pair<const VNInfo *, Register> | |||
2548 | JoinVals::followCopyChain(const VNInfo *VNI) const { | |||
2549 | Register TrackReg = Reg; | |||
2550 | ||||
2551 | while (!VNI->isPHIDef()) { | |||
| ||||
2552 | SlotIndex Def = VNI->def; | |||
2553 | MachineInstr *MI = Indexes->getInstructionFromIndex(Def); | |||
2554 | assert(MI && "No defining instruction")((MI && "No defining instruction") ? static_cast<void > (0) : __assert_fail ("MI && \"No defining instruction\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2554, __PRETTY_FUNCTION__)); | |||
2555 | if (!MI->isFullCopy()) | |||
2556 | return std::make_pair(VNI, TrackReg); | |||
2557 | Register SrcReg = MI->getOperand(1).getReg(); | |||
2558 | if (!SrcReg.isVirtual()) | |||
2559 | return std::make_pair(VNI, TrackReg); | |||
2560 | ||||
2561 | const LiveInterval &LI = LIS->getInterval(SrcReg); | |||
2562 | const VNInfo *ValueIn; | |||
2563 | // No subrange involved. | |||
2564 | if (!SubRangeJoin || !LI.hasSubRanges()) { | |||
2565 | LiveQueryResult LRQ = LI.Query(Def); | |||
2566 | ValueIn = LRQ.valueIn(); | |||
2567 | } else { | |||
2568 | // Query subranges. Ensure that all matching ones take us to the same def | |||
2569 | // (allowing some of them to be undef). | |||
2570 | ValueIn = nullptr; | |||
2571 | for (const LiveInterval::SubRange &S : LI.subranges()) { | |||
2572 | // Transform lanemask to a mask in the joined live interval. | |||
2573 | LaneBitmask SMask = TRI->composeSubRegIndexLaneMask(SubIdx, S.LaneMask); | |||
2574 | if ((SMask & LaneMask).none()) | |||
2575 | continue; | |||
2576 | LiveQueryResult LRQ = S.Query(Def); | |||
2577 | if (!ValueIn) { | |||
2578 | ValueIn = LRQ.valueIn(); | |||
2579 | continue; | |||
2580 | } | |||
2581 | if (LRQ.valueIn() && ValueIn != LRQ.valueIn()) | |||
2582 | return std::make_pair(VNI, TrackReg); | |||
2583 | } | |||
2584 | } | |||
2585 | if (ValueIn == nullptr) { | |||
2586 | // Reaching an undefined value is legitimate, for example: | |||
2587 | // | |||
2588 | // 1 undef %0.sub1 = ... ;; %0.sub0 == undef | |||
2589 | // 2 %1 = COPY %0 ;; %1 is defined here. | |||
2590 | // 3 %0 = COPY %1 ;; Now %0.sub0 has a definition, | |||
2591 | // ;; but it's equivalent to "undef". | |||
2592 | return std::make_pair(nullptr, SrcReg); | |||
2593 | } | |||
2594 | VNI = ValueIn; | |||
2595 | TrackReg = SrcReg; | |||
2596 | } | |||
2597 | return std::make_pair(VNI, TrackReg); | |||
2598 | } | |||
2599 | ||||
2600 | bool JoinVals::valuesIdentical(VNInfo *Value0, VNInfo *Value1, | |||
2601 | const JoinVals &Other) const { | |||
2602 | const VNInfo *Orig0; | |||
2603 | Register Reg0; | |||
2604 | std::tie(Orig0, Reg0) = followCopyChain(Value0); | |||
| ||||
2605 | if (Orig0 == Value1 && Reg0 == Other.Reg) | |||
2606 | return true; | |||
2607 | ||||
2608 | const VNInfo *Orig1; | |||
2609 | Register Reg1; | |||
2610 | std::tie(Orig1, Reg1) = Other.followCopyChain(Value1); | |||
2611 | // If both values are undefined, and the source registers are the same | |||
2612 | // register, the values are identical. Filter out cases where only one | |||
2613 | // value is defined. | |||
2614 | if (Orig0 == nullptr || Orig1 == nullptr) | |||
2615 | return Orig0 == Orig1 && Reg0 == Reg1; | |||
2616 | ||||
2617 | // The values are equal if they are defined at the same place and use the | |||
2618 | // same register. Note that we cannot compare VNInfos directly as some of | |||
2619 | // them might be from a copy created in mergeSubRangeInto() while the other | |||
2620 | // is from the original LiveInterval. | |||
2621 | return Orig0->def == Orig1->def && Reg0 == Reg1; | |||
2622 | } | |||
2623 | ||||
2624 | JoinVals::ConflictResolution | |||
2625 | JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) { | |||
2626 | Val &V = Vals[ValNo]; | |||
2627 | assert(!V.isAnalyzed() && "Value has already been analyzed!")((!V.isAnalyzed() && "Value has already been analyzed!" ) ? static_cast<void> (0) : __assert_fail ("!V.isAnalyzed() && \"Value has already been analyzed!\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2627, __PRETTY_FUNCTION__)); | |||
2628 | VNInfo *VNI = LR.getValNumInfo(ValNo); | |||
2629 | if (VNI->isUnused()) { | |||
2630 | V.WriteLanes = LaneBitmask::getAll(); | |||
2631 | return CR_Keep; | |||
2632 | } | |||
2633 | ||||
2634 | // Get the instruction defining this value, compute the lanes written. | |||
2635 | const MachineInstr *DefMI = nullptr; | |||
2636 | if (VNI->isPHIDef()) { | |||
2637 | // Conservatively assume that all lanes in a PHI are valid. | |||
2638 | LaneBitmask Lanes = SubRangeJoin ? LaneBitmask::getLane(0) | |||
2639 | : TRI->getSubRegIndexLaneMask(SubIdx); | |||
2640 | V.ValidLanes = V.WriteLanes = Lanes; | |||
2641 | } else { | |||
2642 | DefMI = Indexes->getInstructionFromIndex(VNI->def); | |||
2643 | assert(DefMI != nullptr)((DefMI != nullptr) ? static_cast<void> (0) : __assert_fail ("DefMI != nullptr", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2643, __PRETTY_FUNCTION__)); | |||
2644 | if (SubRangeJoin) { | |||
2645 | // We don't care about the lanes when joining subregister ranges. | |||
2646 | V.WriteLanes = V.ValidLanes = LaneBitmask::getLane(0); | |||
2647 | if (DefMI->isImplicitDef()) { | |||
2648 | V.ValidLanes = LaneBitmask::getNone(); | |||
2649 | V.ErasableImplicitDef = true; | |||
2650 | } | |||
2651 | } else { | |||
2652 | bool Redef = false; | |||
2653 | V.ValidLanes = V.WriteLanes = computeWriteLanes(DefMI, Redef); | |||
2654 | ||||
2655 | // If this is a read-modify-write instruction, there may be more valid | |||
2656 | // lanes than the ones written by this instruction. | |||
2657 | // This only covers partial redef operands. DefMI may have normal use | |||
2658 | // operands reading the register. They don't contribute valid lanes. | |||
2659 | // | |||
2660 | // This adds ssub1 to the set of valid lanes in %src: | |||
2661 | // | |||
2662 | // %src:ssub1 = FOO | |||
2663 | // | |||
2664 | // This leaves only ssub1 valid, making any other lanes undef: | |||
2665 | // | |||
2666 | // %src:ssub1<def,read-undef> = FOO %src:ssub2 | |||
2667 | // | |||
2668 | // The <read-undef> flag on the def operand means that old lane values are | |||
2669 | // not important. | |||
2670 | if (Redef) { | |||
2671 | V.RedefVNI = LR.Query(VNI->def).valueIn(); | |||
2672 | assert((TrackSubRegLiveness || V.RedefVNI) &&(((TrackSubRegLiveness || V.RedefVNI) && "Instruction is reading nonexistent value" ) ? static_cast<void> (0) : __assert_fail ("(TrackSubRegLiveness || V.RedefVNI) && \"Instruction is reading nonexistent value\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2673, __PRETTY_FUNCTION__)) | |||
2673 | "Instruction is reading nonexistent value")(((TrackSubRegLiveness || V.RedefVNI) && "Instruction is reading nonexistent value" ) ? static_cast<void> (0) : __assert_fail ("(TrackSubRegLiveness || V.RedefVNI) && \"Instruction is reading nonexistent value\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2673, __PRETTY_FUNCTION__)); | |||
2674 | if (V.RedefVNI != nullptr) { | |||
2675 | computeAssignment(V.RedefVNI->id, Other); | |||
2676 | V.ValidLanes |= Vals[V.RedefVNI->id].ValidLanes; | |||
2677 | } | |||
2678 | } | |||
2679 | ||||
2680 | // An IMPLICIT_DEF writes undef values. | |||
2681 | if (DefMI->isImplicitDef()) { | |||
2682 | // We normally expect IMPLICIT_DEF values to be live only until the end | |||
2683 | // of their block. If the value is really live longer and gets pruned in | |||
2684 | // another block, this flag is cleared again. | |||
2685 | // | |||
2686 | // Clearing the valid lanes is deferred until it is sure this can be | |||
2687 | // erased. | |||
2688 | V.ErasableImplicitDef = true; | |||
2689 | } | |||
2690 | } | |||
2691 | } | |||
2692 | ||||
2693 | // Find the value in Other that overlaps VNI->def, if any. | |||
2694 | LiveQueryResult OtherLRQ = Other.LR.Query(VNI->def); | |||
2695 | ||||
2696 | // It is possible that both values are defined by the same instruction, or | |||
2697 | // the values are PHIs defined in the same block. When that happens, the two | |||
2698 | // values should be merged into one, but not into any preceding value. | |||
2699 | // The first value defined or visited gets CR_Keep, the other gets CR_Merge. | |||
2700 | if (VNInfo *OtherVNI = OtherLRQ.valueDefined()) { | |||
2701 | assert(SlotIndex::isSameInstr(VNI->def, OtherVNI->def) && "Broken LRQ")((SlotIndex::isSameInstr(VNI->def, OtherVNI->def) && "Broken LRQ") ? static_cast<void> (0) : __assert_fail ( "SlotIndex::isSameInstr(VNI->def, OtherVNI->def) && \"Broken LRQ\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2701, __PRETTY_FUNCTION__)); | |||
2702 | ||||
2703 | // One value stays, the other is merged. Keep the earlier one, or the first | |||
2704 | // one we see. | |||
2705 | if (OtherVNI->def < VNI->def) | |||
2706 | Other.computeAssignment(OtherVNI->id, *this); | |||
2707 | else if (VNI->def < OtherVNI->def && OtherLRQ.valueIn()) { | |||
2708 | // This is an early-clobber def overlapping a live-in value in the other | |||
2709 | // register. Not mergeable. | |||
2710 | V.OtherVNI = OtherLRQ.valueIn(); | |||
2711 | return CR_Impossible; | |||
2712 | } | |||
2713 | V.OtherVNI = OtherVNI; | |||
2714 | Val &OtherV = Other.Vals[OtherVNI->id]; | |||
2715 | // Keep this value, check for conflicts when analyzing OtherVNI. | |||
2716 | if (!OtherV.isAnalyzed()) | |||
2717 | return CR_Keep; | |||
2718 | // Both sides have been analyzed now. | |||
2719 | // Allow overlapping PHI values. Any real interference would show up in a | |||
2720 | // predecessor, the PHI itself can't introduce any conflicts. | |||
2721 | if (VNI->isPHIDef()) | |||
2722 | return CR_Merge; | |||
2723 | if ((V.ValidLanes & OtherV.ValidLanes).any()) | |||
2724 | // Overlapping lanes can't be resolved. | |||
2725 | return CR_Impossible; | |||
2726 | else | |||
2727 | return CR_Merge; | |||
2728 | } | |||
2729 | ||||
2730 | // No simultaneous def. Is Other live at the def? | |||
2731 | V.OtherVNI = OtherLRQ.valueIn(); | |||
2732 | if (!V.OtherVNI) | |||
2733 | // No overlap, no conflict. | |||
2734 | return CR_Keep; | |||
2735 | ||||
2736 | assert(!SlotIndex::isSameInstr(VNI->def, V.OtherVNI->def) && "Broken LRQ")((!SlotIndex::isSameInstr(VNI->def, V.OtherVNI->def) && "Broken LRQ") ? static_cast<void> (0) : __assert_fail ( "!SlotIndex::isSameInstr(VNI->def, V.OtherVNI->def) && \"Broken LRQ\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2736, __PRETTY_FUNCTION__)); | |||
2737 | ||||
2738 | // We have overlapping values, or possibly a kill of Other. | |||
2739 | // Recursively compute assignments up the dominator tree. | |||
2740 | Other.computeAssignment(V.OtherVNI->id, *this); | |||
2741 | Val &OtherV = Other.Vals[V.OtherVNI->id]; | |||
2742 | ||||
2743 | if (OtherV.ErasableImplicitDef) { | |||
2744 | // Check if OtherV is an IMPLICIT_DEF that extends beyond its basic block. | |||
2745 | // This shouldn't normally happen, but ProcessImplicitDefs can leave such | |||
2746 | // IMPLICIT_DEF instructions behind, and there is nothing wrong with it | |||
2747 | // technically. | |||
2748 | // | |||
2749 | // When it happens, treat that IMPLICIT_DEF as a normal value, and don't try | |||
2750 | // to erase the IMPLICIT_DEF instruction. | |||
2751 | if (DefMI && | |||
2752 | DefMI->getParent() != Indexes->getMBBFromIndex(V.OtherVNI->def)) { | |||
2753 | LLVM_DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->defdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def << " extends into " << printMBBReference (*DefMI->getParent()) << ", keeping it.\n"; } } while (false) | |||
2754 | << " extends into "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def << " extends into " << printMBBReference (*DefMI->getParent()) << ", keeping it.\n"; } } while (false) | |||
2755 | << printMBBReference(*DefMI->getParent())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def << " extends into " << printMBBReference (*DefMI->getParent()) << ", keeping it.\n"; } } while (false) | |||
2756 | << ", keeping it.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def << " extends into " << printMBBReference (*DefMI->getParent()) << ", keeping it.\n"; } } while (false); | |||
2757 | OtherV.ErasableImplicitDef = false; | |||
2758 | } else { | |||
2759 | // We deferred clearing these lanes in case we needed to save them | |||
2760 | OtherV.ValidLanes &= ~OtherV.WriteLanes; | |||
2761 | } | |||
2762 | } | |||
2763 | ||||
2764 | // Allow overlapping PHI values. Any real interference would show up in a | |||
2765 | // predecessor, the PHI itself can't introduce any conflicts. | |||
2766 | if (VNI->isPHIDef()) | |||
2767 | return CR_Replace; | |||
2768 | ||||
2769 | // Check for simple erasable conflicts. | |||
2770 | if (DefMI->isImplicitDef()) | |||
2771 | return CR_Erase; | |||
2772 | ||||
2773 | // Include the non-conflict where DefMI is a coalescable copy that kills | |||
2774 | // OtherVNI. We still want the copy erased and value numbers merged. | |||
2775 | if (CP.isCoalescable(DefMI)) { | |||
2776 | // Some of the lanes copied from OtherVNI may be undef, making them undef | |||
2777 | // here too. | |||
2778 | V.ValidLanes &= ~V.WriteLanes | OtherV.ValidLanes; | |||
2779 | return CR_Erase; | |||
2780 | } | |||
2781 | ||||
2782 | // This may not be a real conflict if DefMI simply kills Other and defines | |||
2783 | // VNI. | |||
2784 | if (OtherLRQ.isKill() && OtherLRQ.endPoint() <= VNI->def) | |||
2785 | return CR_Keep; | |||
2786 | ||||
2787 | // Handle the case where VNI and OtherVNI can be proven to be identical: | |||
2788 | // | |||
2789 | // %other = COPY %ext | |||
2790 | // %this = COPY %ext <-- Erase this copy | |||
2791 | // | |||
2792 | if (DefMI->isFullCopy() && !CP.isPartial() && | |||
2793 | valuesIdentical(VNI, V.OtherVNI, Other)) { | |||
2794 | V.Identical = true; | |||
2795 | return CR_Erase; | |||
2796 | } | |||
2797 | ||||
2798 | // The remaining checks apply to the lanes, which aren't tracked here. This | |||
2799 | // was already decided to be OK via the following CR_Replace condition. | |||
2800 | // CR_Replace. | |||
2801 | if (SubRangeJoin) | |||
2802 | return CR_Replace; | |||
2803 | ||||
2804 | // If the lanes written by this instruction were all undef in OtherVNI, it is | |||
2805 | // still safe to join the live ranges. This can't be done with a simple value | |||
2806 | // mapping, though - OtherVNI will map to multiple values: | |||
2807 | // | |||
2808 | // 1 %dst:ssub0 = FOO <-- OtherVNI | |||
2809 | // 2 %src = BAR <-- VNI | |||
2810 | // 3 %dst:ssub1 = COPY killed %src <-- Eliminate this copy. | |||
2811 | // 4 BAZ killed %dst | |||
2812 | // 5 QUUX killed %src | |||
2813 | // | |||
2814 | // Here OtherVNI will map to itself in [1;2), but to VNI in [2;5). CR_Replace | |||
2815 | // handles this complex value mapping. | |||
2816 | if ((V.WriteLanes & OtherV.ValidLanes).none()) | |||
2817 | return CR_Replace; | |||
2818 | ||||
2819 | // If the other live range is killed by DefMI and the live ranges are still | |||
2820 | // overlapping, it must be because we're looking at an early clobber def: | |||
2821 | // | |||
2822 | // %dst<def,early-clobber> = ASM killed %src | |||
2823 | // | |||
2824 | // In this case, it is illegal to merge the two live ranges since the early | |||
2825 | // clobber def would clobber %src before it was read. | |||
2826 | if (OtherLRQ.isKill()) { | |||
2827 | // This case where the def doesn't overlap the kill is handled above. | |||
2828 | assert(VNI->def.isEarlyClobber() &&((VNI->def.isEarlyClobber() && "Only early clobber defs can overlap a kill" ) ? static_cast<void> (0) : __assert_fail ("VNI->def.isEarlyClobber() && \"Only early clobber defs can overlap a kill\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2829, __PRETTY_FUNCTION__)) | |||
2829 | "Only early clobber defs can overlap a kill")((VNI->def.isEarlyClobber() && "Only early clobber defs can overlap a kill" ) ? static_cast<void> (0) : __assert_fail ("VNI->def.isEarlyClobber() && \"Only early clobber defs can overlap a kill\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2829, __PRETTY_FUNCTION__)); | |||
2830 | return CR_Impossible; | |||
2831 | } | |||
2832 | ||||
2833 | // VNI is clobbering live lanes in OtherVNI, but there is still the | |||
2834 | // possibility that no instructions actually read the clobbered lanes. | |||
2835 | // If we're clobbering all the lanes in OtherVNI, at least one must be read. | |||
2836 | // Otherwise Other.RI wouldn't be live here. | |||
2837 | if ((TRI->getSubRegIndexLaneMask(Other.SubIdx) & ~V.WriteLanes).none()) | |||
2838 | return CR_Impossible; | |||
2839 | ||||
2840 | // We need to verify that no instructions are reading the clobbered lanes. To | |||
2841 | // save compile time, we'll only check that locally. Don't allow the tainted | |||
2842 | // value to escape the basic block. | |||
2843 | MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def); | |||
2844 | if (OtherLRQ.endPoint() >= Indexes->getMBBEndIdx(MBB)) | |||
2845 | return CR_Impossible; | |||
2846 | ||||
2847 | // There are still some things that could go wrong besides clobbered lanes | |||
2848 | // being read, for example OtherVNI may be only partially redefined in MBB, | |||
2849 | // and some clobbered lanes could escape the block. Save this analysis for | |||
2850 | // resolveConflicts() when all values have been mapped. We need to know | |||
2851 | // RedefVNI and WriteLanes for any later defs in MBB, and we can't compute | |||
2852 | // that now - the recursive analyzeValue() calls must go upwards in the | |||
2853 | // dominator tree. | |||
2854 | return CR_Unresolved; | |||
2855 | } | |||
2856 | ||||
2857 | void JoinVals::computeAssignment(unsigned ValNo, JoinVals &Other) { | |||
2858 | Val &V = Vals[ValNo]; | |||
2859 | if (V.isAnalyzed()) { | |||
2860 | // Recursion should always move up the dominator tree, so ValNo is not | |||
2861 | // supposed to reappear before it has been assigned. | |||
2862 | assert(Assignments[ValNo] != -1 && "Bad recursion?")((Assignments[ValNo] != -1 && "Bad recursion?") ? static_cast <void> (0) : __assert_fail ("Assignments[ValNo] != -1 && \"Bad recursion?\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2862, __PRETTY_FUNCTION__)); | |||
2863 | return; | |||
2864 | } | |||
2865 | switch ((V.Resolution = analyzeValue(ValNo, Other))) { | |||
2866 | case CR_Erase: | |||
2867 | case CR_Merge: | |||
2868 | // Merge this ValNo into OtherVNI. | |||
2869 | assert(V.OtherVNI && "OtherVNI not assigned, can't merge.")((V.OtherVNI && "OtherVNI not assigned, can't merge." ) ? static_cast<void> (0) : __assert_fail ("V.OtherVNI && \"OtherVNI not assigned, can't merge.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2869, __PRETTY_FUNCTION__)); | |||
2870 | assert(Other.Vals[V.OtherVNI->id].isAnalyzed() && "Missing recursion")((Other.Vals[V.OtherVNI->id].isAnalyzed() && "Missing recursion" ) ? static_cast<void> (0) : __assert_fail ("Other.Vals[V.OtherVNI->id].isAnalyzed() && \"Missing recursion\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2870, __PRETTY_FUNCTION__)); | |||
2871 | Assignments[ValNo] = Other.Assignments[V.OtherVNI->id]; | |||
2872 | LLVM_DEBUG(dbgs() << "\t\tmerge " << printReg(Reg) << ':' << ValNo << '@'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tmerge " << printReg (Reg) << ':' << ValNo << '@' << LR.getValNumInfo (ValNo)->def << " into " << printReg(Other.Reg ) << ':' << V.OtherVNI->id << '@' << V.OtherVNI->def << " --> @" << NewVNInfo[Assignments [ValNo]]->def << '\n'; } } while (false) | |||
2873 | << LR.getValNumInfo(ValNo)->def << " into "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tmerge " << printReg (Reg) << ':' << ValNo << '@' << LR.getValNumInfo (ValNo)->def << " into " << printReg(Other.Reg ) << ':' << V.OtherVNI->id << '@' << V.OtherVNI->def << " --> @" << NewVNInfo[Assignments [ValNo]]->def << '\n'; } } while (false) | |||
2874 | << printReg(Other.Reg) << ':' << V.OtherVNI->id << '@'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tmerge " << printReg (Reg) << ':' << ValNo << '@' << LR.getValNumInfo (ValNo)->def << " into " << printReg(Other.Reg ) << ':' << V.OtherVNI->id << '@' << V.OtherVNI->def << " --> @" << NewVNInfo[Assignments [ValNo]]->def << '\n'; } } while (false) | |||
2875 | << V.OtherVNI->def << " --> @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tmerge " << printReg (Reg) << ':' << ValNo << '@' << LR.getValNumInfo (ValNo)->def << " into " << printReg(Other.Reg ) << ':' << V.OtherVNI->id << '@' << V.OtherVNI->def << " --> @" << NewVNInfo[Assignments [ValNo]]->def << '\n'; } } while (false) | |||
2876 | << NewVNInfo[Assignments[ValNo]]->def << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tmerge " << printReg (Reg) << ':' << ValNo << '@' << LR.getValNumInfo (ValNo)->def << " into " << printReg(Other.Reg ) << ':' << V.OtherVNI->id << '@' << V.OtherVNI->def << " --> @" << NewVNInfo[Assignments [ValNo]]->def << '\n'; } } while (false); | |||
2877 | break; | |||
2878 | case CR_Replace: | |||
2879 | case CR_Unresolved: { | |||
2880 | // The other value is going to be pruned if this join is successful. | |||
2881 | assert(V.OtherVNI && "OtherVNI not assigned, can't prune")((V.OtherVNI && "OtherVNI not assigned, can't prune") ? static_cast<void> (0) : __assert_fail ("V.OtherVNI && \"OtherVNI not assigned, can't prune\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2881, __PRETTY_FUNCTION__)); | |||
2882 | Val &OtherV = Other.Vals[V.OtherVNI->id]; | |||
2883 | // We cannot erase an IMPLICIT_DEF if we don't have valid values for all | |||
2884 | // its lanes. | |||
2885 | if (OtherV.ErasableImplicitDef && | |||
2886 | TrackSubRegLiveness && | |||
2887 | (OtherV.WriteLanes & ~V.ValidLanes).any()) { | |||
2888 | LLVM_DEBUG(dbgs() << "Cannot erase implicit_def with missing values\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Cannot erase implicit_def with missing values\n" ; } } while (false); | |||
2889 | ||||
2890 | OtherV.ErasableImplicitDef = false; | |||
2891 | // The valid lanes written by the implicit_def were speculatively cleared | |||
2892 | // before, so make this more conservative. It may be better to track this, | |||
2893 | // I haven't found a testcase where it matters. | |||
2894 | OtherV.ValidLanes = LaneBitmask::getAll(); | |||
2895 | } | |||
2896 | ||||
2897 | OtherV.Pruned = true; | |||
2898 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | |||
2899 | } | |||
2900 | default: | |||
2901 | // This value number needs to go in the final joined live range. | |||
2902 | Assignments[ValNo] = NewVNInfo.size(); | |||
2903 | NewVNInfo.push_back(LR.getValNumInfo(ValNo)); | |||
2904 | break; | |||
2905 | } | |||
2906 | } | |||
2907 | ||||
2908 | bool JoinVals::mapValues(JoinVals &Other) { | |||
2909 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
2910 | computeAssignment(i, Other); | |||
2911 | if (Vals[i].Resolution == CR_Impossible) { | |||
2912 | LLVM_DEBUG(dbgs() << "\t\tinterference at " << printReg(Reg) << ':' << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tinterference at " << printReg(Reg) << ':' << i << '@' << LR .getValNumInfo(i)->def << '\n'; } } while (false) | |||
2913 | << '@' << LR.getValNumInfo(i)->def << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tinterference at " << printReg(Reg) << ':' << i << '@' << LR .getValNumInfo(i)->def << '\n'; } } while (false); | |||
2914 | return false; | |||
2915 | } | |||
2916 | } | |||
2917 | return true; | |||
2918 | } | |||
2919 | ||||
2920 | bool JoinVals:: | |||
2921 | taintExtent(unsigned ValNo, LaneBitmask TaintedLanes, JoinVals &Other, | |||
2922 | SmallVectorImpl<std::pair<SlotIndex, LaneBitmask>> &TaintExtent) { | |||
2923 | VNInfo *VNI = LR.getValNumInfo(ValNo); | |||
2924 | MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def); | |||
2925 | SlotIndex MBBEnd = Indexes->getMBBEndIdx(MBB); | |||
2926 | ||||
2927 | // Scan Other.LR from VNI.def to MBBEnd. | |||
2928 | LiveInterval::iterator OtherI = Other.LR.find(VNI->def); | |||
2929 | assert(OtherI != Other.LR.end() && "No conflict?")((OtherI != Other.LR.end() && "No conflict?") ? static_cast <void> (0) : __assert_fail ("OtherI != Other.LR.end() && \"No conflict?\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2929, __PRETTY_FUNCTION__)); | |||
2930 | do { | |||
2931 | // OtherI is pointing to a tainted value. Abort the join if the tainted | |||
2932 | // lanes escape the block. | |||
2933 | SlotIndex End = OtherI->end; | |||
2934 | if (End >= MBBEnd) { | |||
2935 | LLVM_DEBUG(dbgs() << "\t\ttaints global " << printReg(Other.Reg) << ':'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttaints global " << printReg(Other.Reg) << ':' << OtherI->valno-> id << '@' << OtherI->start << '\n'; } } while (false) | |||
2936 | << OtherI->valno->id << '@' << OtherI->start << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttaints global " << printReg(Other.Reg) << ':' << OtherI->valno-> id << '@' << OtherI->start << '\n'; } } while (false); | |||
2937 | return false; | |||
2938 | } | |||
2939 | LLVM_DEBUG(dbgs() << "\t\ttaints local " << printReg(Other.Reg) << ':'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttaints local " << printReg (Other.Reg) << ':' << OtherI->valno->id << '@' << OtherI->start << " to " << End << '\n'; } } while (false) | |||
2940 | << OtherI->valno->id << '@' << OtherI->start << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttaints local " << printReg (Other.Reg) << ':' << OtherI->valno->id << '@' << OtherI->start << " to " << End << '\n'; } } while (false) | |||
2941 | << End << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttaints local " << printReg (Other.Reg) << ':' << OtherI->valno->id << '@' << OtherI->start << " to " << End << '\n'; } } while (false); | |||
2942 | // A dead def is not a problem. | |||
2943 | if (End.isDead()) | |||
2944 | break; | |||
2945 | TaintExtent.push_back(std::make_pair(End, TaintedLanes)); | |||
2946 | ||||
2947 | // Check for another def in the MBB. | |||
2948 | if (++OtherI == Other.LR.end() || OtherI->start >= MBBEnd) | |||
2949 | break; | |||
2950 | ||||
2951 | // Lanes written by the new def are no longer tainted. | |||
2952 | const Val &OV = Other.Vals[OtherI->valno->id]; | |||
2953 | TaintedLanes &= ~OV.WriteLanes; | |||
2954 | if (!OV.RedefVNI) | |||
2955 | break; | |||
2956 | } while (TaintedLanes.any()); | |||
2957 | return true; | |||
2958 | } | |||
2959 | ||||
2960 | bool JoinVals::usesLanes(const MachineInstr &MI, Register Reg, unsigned SubIdx, | |||
2961 | LaneBitmask Lanes) const { | |||
2962 | if (MI.isDebugInstr()) | |||
2963 | return false; | |||
2964 | for (const MachineOperand &MO : MI.operands()) { | |||
2965 | if (!MO.isReg() || MO.isDef() || MO.getReg() != Reg) | |||
2966 | continue; | |||
2967 | if (!MO.readsReg()) | |||
2968 | continue; | |||
2969 | unsigned S = TRI->composeSubRegIndices(SubIdx, MO.getSubReg()); | |||
2970 | if ((Lanes & TRI->getSubRegIndexLaneMask(S)).any()) | |||
2971 | return true; | |||
2972 | } | |||
2973 | return false; | |||
2974 | } | |||
2975 | ||||
2976 | bool JoinVals::resolveConflicts(JoinVals &Other) { | |||
2977 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
2978 | Val &V = Vals[i]; | |||
2979 | assert(V.Resolution != CR_Impossible && "Unresolvable conflict")((V.Resolution != CR_Impossible && "Unresolvable conflict" ) ? static_cast<void> (0) : __assert_fail ("V.Resolution != CR_Impossible && \"Unresolvable conflict\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2979, __PRETTY_FUNCTION__)); | |||
2980 | if (V.Resolution != CR_Unresolved) | |||
2981 | continue; | |||
2982 | LLVM_DEBUG(dbgs() << "\t\tconflict at " << printReg(Reg) << ':' << i << '@'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tconflict at " << printReg (Reg) << ':' << i << '@' << LR.getValNumInfo (i)->def << ' ' << PrintLaneMask(LaneMask) << '\n'; } } while (false) | |||
2983 | << LR.getValNumInfo(i)->defdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tconflict at " << printReg (Reg) << ':' << i << '@' << LR.getValNumInfo (i)->def << ' ' << PrintLaneMask(LaneMask) << '\n'; } } while (false) | |||
2984 | << ' ' << PrintLaneMask(LaneMask) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tconflict at " << printReg (Reg) << ':' << i << '@' << LR.getValNumInfo (i)->def << ' ' << PrintLaneMask(LaneMask) << '\n'; } } while (false); | |||
2985 | if (SubRangeJoin) | |||
2986 | return false; | |||
2987 | ||||
2988 | ++NumLaneConflicts; | |||
2989 | assert(V.OtherVNI && "Inconsistent conflict resolution.")((V.OtherVNI && "Inconsistent conflict resolution.") ? static_cast<void> (0) : __assert_fail ("V.OtherVNI && \"Inconsistent conflict resolution.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 2989, __PRETTY_FUNCTION__)); | |||
2990 | VNInfo *VNI = LR.getValNumInfo(i); | |||
2991 | const Val &OtherV = Other.Vals[V.OtherVNI->id]; | |||
2992 | ||||
2993 | // VNI is known to clobber some lanes in OtherVNI. If we go ahead with the | |||
2994 | // join, those lanes will be tainted with a wrong value. Get the extent of | |||
2995 | // the tainted lanes. | |||
2996 | LaneBitmask TaintedLanes = V.WriteLanes & OtherV.ValidLanes; | |||
2997 | SmallVector<std::pair<SlotIndex, LaneBitmask>, 8> TaintExtent; | |||
2998 | if (!taintExtent(i, TaintedLanes, Other, TaintExtent)) | |||
2999 | // Tainted lanes would extend beyond the basic block. | |||
3000 | return false; | |||
3001 | ||||
3002 | assert(!TaintExtent.empty() && "There should be at least one conflict.")((!TaintExtent.empty() && "There should be at least one conflict." ) ? static_cast<void> (0) : __assert_fail ("!TaintExtent.empty() && \"There should be at least one conflict.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3002, __PRETTY_FUNCTION__)); | |||
3003 | ||||
3004 | // Now look at the instructions from VNI->def to TaintExtent (inclusive). | |||
3005 | MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def); | |||
3006 | MachineBasicBlock::iterator MI = MBB->begin(); | |||
3007 | if (!VNI->isPHIDef()) { | |||
3008 | MI = Indexes->getInstructionFromIndex(VNI->def); | |||
3009 | // No need to check the instruction defining VNI for reads. | |||
3010 | ++MI; | |||
3011 | } | |||
3012 | assert(!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) &&((!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first ) && "Interference ends on VNI->def. Should have been handled earlier" ) ? static_cast<void> (0) : __assert_fail ("!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) && \"Interference ends on VNI->def. Should have been handled earlier\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3013, __PRETTY_FUNCTION__)) | |||
3013 | "Interference ends on VNI->def. Should have been handled earlier")((!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first ) && "Interference ends on VNI->def. Should have been handled earlier" ) ? static_cast<void> (0) : __assert_fail ("!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) && \"Interference ends on VNI->def. Should have been handled earlier\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3013, __PRETTY_FUNCTION__)); | |||
3014 | MachineInstr *LastMI = | |||
3015 | Indexes->getInstructionFromIndex(TaintExtent.front().first); | |||
3016 | assert(LastMI && "Range must end at a proper instruction")((LastMI && "Range must end at a proper instruction") ? static_cast<void> (0) : __assert_fail ("LastMI && \"Range must end at a proper instruction\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3016, __PRETTY_FUNCTION__)); | |||
3017 | unsigned TaintNum = 0; | |||
3018 | while (true) { | |||
3019 | assert(MI != MBB->end() && "Bad LastMI")((MI != MBB->end() && "Bad LastMI") ? static_cast< void> (0) : __assert_fail ("MI != MBB->end() && \"Bad LastMI\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3019, __PRETTY_FUNCTION__)); | |||
3020 | if (usesLanes(*MI, Other.Reg, Other.SubIdx, TaintedLanes)) { | |||
3021 | LLVM_DEBUG(dbgs() << "\t\ttainted lanes used by: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\ttainted lanes used by: " << *MI; } } while (false); | |||
3022 | return false; | |||
3023 | } | |||
3024 | // LastMI is the last instruction to use the current value. | |||
3025 | if (&*MI == LastMI) { | |||
3026 | if (++TaintNum == TaintExtent.size()) | |||
3027 | break; | |||
3028 | LastMI = Indexes->getInstructionFromIndex(TaintExtent[TaintNum].first); | |||
3029 | assert(LastMI && "Range must end at a proper instruction")((LastMI && "Range must end at a proper instruction") ? static_cast<void> (0) : __assert_fail ("LastMI && \"Range must end at a proper instruction\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3029, __PRETTY_FUNCTION__)); | |||
3030 | TaintedLanes = TaintExtent[TaintNum].second; | |||
3031 | } | |||
3032 | ++MI; | |||
3033 | } | |||
3034 | ||||
3035 | // The tainted lanes are unused. | |||
3036 | V.Resolution = CR_Replace; | |||
3037 | ++NumLaneResolves; | |||
3038 | } | |||
3039 | return true; | |||
3040 | } | |||
3041 | ||||
3042 | bool JoinVals::isPrunedValue(unsigned ValNo, JoinVals &Other) { | |||
3043 | Val &V = Vals[ValNo]; | |||
3044 | if (V.Pruned || V.PrunedComputed) | |||
3045 | return V.Pruned; | |||
3046 | ||||
3047 | if (V.Resolution != CR_Erase && V.Resolution != CR_Merge) | |||
3048 | return V.Pruned; | |||
3049 | ||||
3050 | // Follow copies up the dominator tree and check if any intermediate value | |||
3051 | // has been pruned. | |||
3052 | V.PrunedComputed = true; | |||
3053 | V.Pruned = Other.isPrunedValue(V.OtherVNI->id, *this); | |||
3054 | return V.Pruned; | |||
3055 | } | |||
3056 | ||||
3057 | void JoinVals::pruneValues(JoinVals &Other, | |||
3058 | SmallVectorImpl<SlotIndex> &EndPoints, | |||
3059 | bool changeInstrs) { | |||
3060 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
3061 | SlotIndex Def = LR.getValNumInfo(i)->def; | |||
3062 | switch (Vals[i].Resolution) { | |||
3063 | case CR_Keep: | |||
3064 | break; | |||
3065 | case CR_Replace: { | |||
3066 | // This value takes precedence over the value in Other.LR. | |||
3067 | LIS->pruneValue(Other.LR, Def, &EndPoints); | |||
3068 | // Check if we're replacing an IMPLICIT_DEF value. The IMPLICIT_DEF | |||
3069 | // instructions are only inserted to provide a live-out value for PHI | |||
3070 | // predecessors, so the instruction should simply go away once its value | |||
3071 | // has been replaced. | |||
3072 | Val &OtherV = Other.Vals[Vals[i].OtherVNI->id]; | |||
3073 | bool EraseImpDef = OtherV.ErasableImplicitDef && | |||
3074 | OtherV.Resolution == CR_Keep; | |||
3075 | if (!Def.isBlock()) { | |||
3076 | if (changeInstrs) { | |||
3077 | // Remove <def,read-undef> flags. This def is now a partial redef. | |||
3078 | // Also remove dead flags since the joined live range will | |||
3079 | // continue past this instruction. | |||
3080 | for (MachineOperand &MO : | |||
3081 | Indexes->getInstructionFromIndex(Def)->operands()) { | |||
3082 | if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) { | |||
3083 | if (MO.getSubReg() != 0 && MO.isUndef() && !EraseImpDef) | |||
3084 | MO.setIsUndef(false); | |||
3085 | MO.setIsDead(false); | |||
3086 | } | |||
3087 | } | |||
3088 | } | |||
3089 | // This value will reach instructions below, but we need to make sure | |||
3090 | // the live range also reaches the instruction at Def. | |||
3091 | if (!EraseImpDef) | |||
3092 | EndPoints.push_back(Def); | |||
3093 | } | |||
3094 | LLVM_DEBUG(dbgs() << "\t\tpruned " << printReg(Other.Reg) << " at " << Defdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tpruned " << printReg (Other.Reg) << " at " << Def << ": " << Other.LR << '\n'; } } while (false) | |||
3095 | << ": " << Other.LR << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tpruned " << printReg (Other.Reg) << " at " << Def << ": " << Other.LR << '\n'; } } while (false); | |||
3096 | break; | |||
3097 | } | |||
3098 | case CR_Erase: | |||
3099 | case CR_Merge: | |||
3100 | if (isPrunedValue(i, Other)) { | |||
3101 | // This value is ultimately a copy of a pruned value in LR or Other.LR. | |||
3102 | // We can no longer trust the value mapping computed by | |||
3103 | // computeAssignment(), the value that was originally copied could have | |||
3104 | // been replaced. | |||
3105 | LIS->pruneValue(LR, Def, &EndPoints); | |||
3106 | LLVM_DEBUG(dbgs() << "\t\tpruned all of " << printReg(Reg) << " at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tpruned all of " << printReg(Reg) << " at " << Def << ": " << LR << '\n'; } } while (false) | |||
3107 | << Def << ": " << LR << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tpruned all of " << printReg(Reg) << " at " << Def << ": " << LR << '\n'; } } while (false); | |||
3108 | } | |||
3109 | break; | |||
3110 | case CR_Unresolved: | |||
3111 | case CR_Impossible: | |||
3112 | llvm_unreachable("Unresolved conflicts")::llvm::llvm_unreachable_internal("Unresolved conflicts", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3112); | |||
3113 | } | |||
3114 | } | |||
3115 | } | |||
3116 | ||||
3117 | /// Consider the following situation when coalescing the copy between | |||
3118 | /// %31 and %45 at 800. (The vertical lines represent live range segments.) | |||
3119 | /// | |||
3120 | /// Main range Subrange 0004 (sub2) | |||
3121 | /// %31 %45 %31 %45 | |||
3122 | /// 544 %45 = COPY %28 + + | |||
3123 | /// | v1 | v1 | |||
3124 | /// 560B bb.1: + + | |||
3125 | /// 624 = %45.sub2 | v2 | v2 | |||
3126 | /// 800 %31 = COPY %45 + + + + | |||
3127 | /// | v0 | v0 | |||
3128 | /// 816 %31.sub1 = ... + | | |||
3129 | /// 880 %30 = COPY %31 | v1 + | |||
3130 | /// 928 %45 = COPY %30 | + + | |||
3131 | /// | | v0 | v0 <--+ | |||
3132 | /// 992B ; backedge -> bb.1 | + + | | |||
3133 | /// 1040 = %31.sub0 + | | |||
3134 | /// This value must remain | |||
3135 | /// live-out! | |||
3136 | /// | |||
3137 | /// Assuming that %31 is coalesced into %45, the copy at 928 becomes | |||
3138 | /// redundant, since it copies the value from %45 back into it. The | |||
3139 | /// conflict resolution for the main range determines that %45.v0 is | |||
3140 | /// to be erased, which is ok since %31.v1 is identical to it. | |||
3141 | /// The problem happens with the subrange for sub2: it has to be live | |||
3142 | /// on exit from the block, but since 928 was actually a point of | |||
3143 | /// definition of %45.sub2, %45.sub2 was not live immediately prior | |||
3144 | /// to that definition. As a result, when 928 was erased, the value v0 | |||
3145 | /// for %45.sub2 was pruned in pruneSubRegValues. Consequently, an | |||
3146 | /// IMPLICIT_DEF was inserted as a "backedge" definition for %45.sub2, | |||
3147 | /// providing an incorrect value to the use at 624. | |||
3148 | /// | |||
3149 | /// Since the main-range values %31.v1 and %45.v0 were proved to be | |||
3150 | /// identical, the corresponding values in subranges must also be the | |||
3151 | /// same. A redundant copy is removed because it's not needed, and not | |||
3152 | /// because it copied an undefined value, so any liveness that originated | |||
3153 | /// from that copy cannot disappear. When pruning a value that started | |||
3154 | /// at the removed copy, the corresponding identical value must be | |||
3155 | /// extended to replace it. | |||
3156 | void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) { | |||
3157 | // Look for values being erased. | |||
3158 | bool DidPrune = false; | |||
3159 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
3160 | Val &V = Vals[i]; | |||
3161 | // We should trigger in all cases in which eraseInstrs() does something. | |||
3162 | // match what eraseInstrs() is doing, print a message so | |||
3163 | if (V.Resolution != CR_Erase && | |||
3164 | (V.Resolution != CR_Keep || !V.ErasableImplicitDef || !V.Pruned)) | |||
3165 | continue; | |||
3166 | ||||
3167 | // Check subranges at the point where the copy will be removed. | |||
3168 | SlotIndex Def = LR.getValNumInfo(i)->def; | |||
3169 | SlotIndex OtherDef; | |||
3170 | if (V.Identical) | |||
3171 | OtherDef = V.OtherVNI->def; | |||
3172 | ||||
3173 | // Print message so mismatches with eraseInstrs() can be diagnosed. | |||
3174 | LLVM_DEBUG(dbgs() << "\t\tExpecting instruction removal at " << Defdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tExpecting instruction removal at " << Def << '\n'; } } while (false) | |||
3175 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tExpecting instruction removal at " << Def << '\n'; } } while (false); | |||
3176 | for (LiveInterval::SubRange &S : LI.subranges()) { | |||
3177 | LiveQueryResult Q = S.Query(Def); | |||
3178 | ||||
3179 | // If a subrange starts at the copy then an undefined value has been | |||
3180 | // copied and we must remove that subrange value as well. | |||
3181 | VNInfo *ValueOut = Q.valueOutOrDead(); | |||
3182 | if (ValueOut != nullptr && (Q.valueIn() == nullptr || | |||
3183 | (V.Identical && V.Resolution == CR_Erase && | |||
3184 | ValueOut->def == Def))) { | |||
3185 | LLVM_DEBUG(dbgs() << "\t\tPrune sublane " << PrintLaneMask(S.LaneMask)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tPrune sublane " << PrintLaneMask(S.LaneMask) << " at " << Def << "\n"; } } while (false) | |||
3186 | << " at " << Def << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tPrune sublane " << PrintLaneMask(S.LaneMask) << " at " << Def << "\n"; } } while (false); | |||
3187 | SmallVector<SlotIndex,8> EndPoints; | |||
3188 | LIS->pruneValue(S, Def, &EndPoints); | |||
3189 | DidPrune = true; | |||
3190 | // Mark value number as unused. | |||
3191 | ValueOut->markUnused(); | |||
3192 | ||||
3193 | if (V.Identical && S.Query(OtherDef).valueOutOrDead()) { | |||
3194 | // If V is identical to V.OtherVNI (and S was live at OtherDef), | |||
3195 | // then we can't simply prune V from S. V needs to be replaced | |||
3196 | // with V.OtherVNI. | |||
3197 | LIS->extendToIndices(S, EndPoints); | |||
3198 | } | |||
3199 | continue; | |||
3200 | } | |||
3201 | // If a subrange ends at the copy, then a value was copied but only | |||
3202 | // partially used later. Shrink the subregister range appropriately. | |||
3203 | if (Q.valueIn() != nullptr && Q.valueOut() == nullptr) { | |||
3204 | LLVM_DEBUG(dbgs() << "\t\tDead uses at sublane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tDead uses at sublane " << PrintLaneMask(S.LaneMask) << " at " << Def << "\n"; } } while (false) | |||
3205 | << PrintLaneMask(S.LaneMask) << " at " << Defdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tDead uses at sublane " << PrintLaneMask(S.LaneMask) << " at " << Def << "\n"; } } while (false) | |||
3206 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tDead uses at sublane " << PrintLaneMask(S.LaneMask) << " at " << Def << "\n"; } } while (false); | |||
3207 | ShrinkMask |= S.LaneMask; | |||
3208 | } | |||
3209 | } | |||
3210 | } | |||
3211 | if (DidPrune) | |||
3212 | LI.removeEmptySubRanges(); | |||
3213 | } | |||
3214 | ||||
3215 | /// Check if any of the subranges of @p LI contain a definition at @p Def. | |||
3216 | static bool isDefInSubRange(LiveInterval &LI, SlotIndex Def) { | |||
3217 | for (LiveInterval::SubRange &SR : LI.subranges()) { | |||
3218 | if (VNInfo *VNI = SR.Query(Def).valueOutOrDead()) | |||
3219 | if (VNI->def == Def) | |||
3220 | return true; | |||
3221 | } | |||
3222 | return false; | |||
3223 | } | |||
3224 | ||||
3225 | void JoinVals::pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange) { | |||
3226 | assert(&static_cast<LiveRange&>(LI) == &LR)((&static_cast<LiveRange&>(LI) == &LR) ? static_cast <void> (0) : __assert_fail ("&static_cast<LiveRange&>(LI) == &LR" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3226, __PRETTY_FUNCTION__)); | |||
3227 | ||||
3228 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
3229 | if (Vals[i].Resolution != CR_Keep) | |||
3230 | continue; | |||
3231 | VNInfo *VNI = LR.getValNumInfo(i); | |||
3232 | if (VNI->isUnused() || VNI->isPHIDef() || isDefInSubRange(LI, VNI->def)) | |||
3233 | continue; | |||
3234 | Vals[i].Pruned = true; | |||
3235 | ShrinkMainRange = true; | |||
3236 | } | |||
3237 | } | |||
3238 | ||||
3239 | void JoinVals::removeImplicitDefs() { | |||
3240 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
3241 | Val &V = Vals[i]; | |||
3242 | if (V.Resolution != CR_Keep || !V.ErasableImplicitDef || !V.Pruned) | |||
3243 | continue; | |||
3244 | ||||
3245 | VNInfo *VNI = LR.getValNumInfo(i); | |||
3246 | VNI->markUnused(); | |||
3247 | LR.removeValNo(VNI); | |||
3248 | } | |||
3249 | } | |||
3250 | ||||
3251 | void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs, | |||
3252 | SmallVectorImpl<Register> &ShrinkRegs, | |||
3253 | LiveInterval *LI) { | |||
3254 | for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) { | |||
3255 | // Get the def location before markUnused() below invalidates it. | |||
3256 | VNInfo *VNI = LR.getValNumInfo(i); | |||
3257 | SlotIndex Def = VNI->def; | |||
3258 | switch (Vals[i].Resolution) { | |||
3259 | case CR_Keep: { | |||
3260 | // If an IMPLICIT_DEF value is pruned, it doesn't serve a purpose any | |||
3261 | // longer. The IMPLICIT_DEF instructions are only inserted by | |||
3262 | // PHIElimination to guarantee that all PHI predecessors have a value. | |||
3263 | if (!Vals[i].ErasableImplicitDef || !Vals[i].Pruned) | |||
3264 | break; | |||
3265 | // Remove value number i from LR. | |||
3266 | // For intervals with subranges, removing a segment from the main range | |||
3267 | // may require extending the previous segment: for each definition of | |||
3268 | // a subregister, there will be a corresponding def in the main range. | |||
3269 | // That def may fall in the middle of a segment from another subrange. | |||
3270 | // In such cases, removing this def from the main range must be | |||
3271 | // complemented by extending the main range to account for the liveness | |||
3272 | // of the other subrange. | |||
3273 | // The new end point of the main range segment to be extended. | |||
3274 | SlotIndex NewEnd; | |||
3275 | if (LI != nullptr) { | |||
3276 | LiveRange::iterator I = LR.FindSegmentContaining(Def); | |||
3277 | assert(I != LR.end())((I != LR.end()) ? static_cast<void> (0) : __assert_fail ("I != LR.end()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3277, __PRETTY_FUNCTION__)); | |||
3278 | // Do not extend beyond the end of the segment being removed. | |||
3279 | // The segment may have been pruned in preparation for joining | |||
3280 | // live ranges. | |||
3281 | NewEnd = I->end; | |||
3282 | } | |||
3283 | ||||
3284 | LR.removeValNo(VNI); | |||
3285 | // Note that this VNInfo is reused and still referenced in NewVNInfo, | |||
3286 | // make it appear like an unused value number. | |||
3287 | VNI->markUnused(); | |||
3288 | ||||
3289 | if (LI != nullptr && LI->hasSubRanges()) { | |||
3290 | assert(static_cast<LiveRange*>(LI) == &LR)((static_cast<LiveRange*>(LI) == &LR) ? static_cast <void> (0) : __assert_fail ("static_cast<LiveRange*>(LI) == &LR" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3290, __PRETTY_FUNCTION__)); | |||
3291 | // Determine the end point based on the subrange information: | |||
3292 | // minimum of (earliest def of next segment, | |||
3293 | // latest end point of containing segment) | |||
3294 | SlotIndex ED, LE; | |||
3295 | for (LiveInterval::SubRange &SR : LI->subranges()) { | |||
3296 | LiveRange::iterator I = SR.find(Def); | |||
3297 | if (I == SR.end()) | |||
3298 | continue; | |||
3299 | if (I->start > Def) | |||
3300 | ED = ED.isValid() ? std::min(ED, I->start) : I->start; | |||
3301 | else | |||
3302 | LE = LE.isValid() ? std::max(LE, I->end) : I->end; | |||
3303 | } | |||
3304 | if (LE.isValid()) | |||
3305 | NewEnd = std::min(NewEnd, LE); | |||
3306 | if (ED.isValid()) | |||
3307 | NewEnd = std::min(NewEnd, ED); | |||
3308 | ||||
3309 | // We only want to do the extension if there was a subrange that | |||
3310 | // was live across Def. | |||
3311 | if (LE.isValid()) { | |||
3312 | LiveRange::iterator S = LR.find(Def); | |||
3313 | if (S != LR.begin()) | |||
3314 | std::prev(S)->end = NewEnd; | |||
3315 | } | |||
3316 | } | |||
3317 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n'; if (LI != nullptr) dbgs() << "\t\t LHS = " << *LI << '\n'; }; } } while (false) | |||
3318 | dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n'; if (LI != nullptr) dbgs() << "\t\t LHS = " << *LI << '\n'; }; } } while (false) | |||
3319 | if (LI != nullptr)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n'; if (LI != nullptr) dbgs() << "\t\t LHS = " << *LI << '\n'; }; } } while (false) | |||
3320 | dbgs() << "\t\t LHS = " << *LI << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n'; if (LI != nullptr) dbgs() << "\t\t LHS = " << *LI << '\n'; }; } } while (false) | |||
3321 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n'; if (LI != nullptr) dbgs() << "\t\t LHS = " << *LI << '\n'; }; } } while (false); | |||
3322 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | |||
3323 | } | |||
3324 | ||||
3325 | case CR_Erase: { | |||
3326 | MachineInstr *MI = Indexes->getInstructionFromIndex(Def); | |||
3327 | assert(MI && "No instruction to erase")((MI && "No instruction to erase") ? static_cast<void > (0) : __assert_fail ("MI && \"No instruction to erase\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3327, __PRETTY_FUNCTION__)); | |||
3328 | if (MI->isCopy()) { | |||
3329 | Register Reg = MI->getOperand(1).getReg(); | |||
3330 | if (Register::isVirtualRegister(Reg) && Reg != CP.getSrcReg() && | |||
3331 | Reg != CP.getDstReg()) | |||
3332 | ShrinkRegs.push_back(Reg); | |||
3333 | } | |||
3334 | ErasedInstrs.insert(MI); | |||
3335 | LLVM_DEBUG(dbgs() << "\t\terased:\t" << Def << '\t' << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\terased:\t" << Def << '\t' << *MI; } } while (false); | |||
3336 | LIS->RemoveMachineInstrFromMaps(*MI); | |||
3337 | MI->eraseFromParent(); | |||
3338 | break; | |||
3339 | } | |||
3340 | default: | |||
3341 | break; | |||
3342 | } | |||
3343 | } | |||
3344 | } | |||
3345 | ||||
3346 | void RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange, | |||
3347 | LaneBitmask LaneMask, | |||
3348 | const CoalescerPair &CP) { | |||
3349 | SmallVector<VNInfo*, 16> NewVNInfo; | |||
3350 | JoinVals RHSVals(RRange, CP.getSrcReg(), CP.getSrcIdx(), LaneMask, | |||
3351 | NewVNInfo, CP, LIS, TRI, true, true); | |||
3352 | JoinVals LHSVals(LRange, CP.getDstReg(), CP.getDstIdx(), LaneMask, | |||
3353 | NewVNInfo, CP, LIS, TRI, true, true); | |||
3354 | ||||
3355 | // Compute NewVNInfo and resolve conflicts (see also joinVirtRegs()) | |||
3356 | // We should be able to resolve all conflicts here as we could successfully do | |||
3357 | // it on the mainrange already. There is however a problem when multiple | |||
3358 | // ranges get mapped to the "overflow" lane mask bit which creates unexpected | |||
3359 | // interferences. | |||
3360 | if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals)) { | |||
3361 | // We already determined that it is legal to merge the intervals, so this | |||
3362 | // should never fail. | |||
3363 | llvm_unreachable("*** Couldn't join subrange!\n")::llvm::llvm_unreachable_internal("*** Couldn't join subrange!\n" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3363); | |||
3364 | } | |||
3365 | if (!LHSVals.resolveConflicts(RHSVals) || | |||
3366 | !RHSVals.resolveConflicts(LHSVals)) { | |||
3367 | // We already determined that it is legal to merge the intervals, so this | |||
3368 | // should never fail. | |||
3369 | llvm_unreachable("*** Couldn't join subrange!\n")::llvm::llvm_unreachable_internal("*** Couldn't join subrange!\n" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3369); | |||
3370 | } | |||
3371 | ||||
3372 | // The merging algorithm in LiveInterval::join() can't handle conflicting | |||
3373 | // value mappings, so we need to remove any live ranges that overlap a | |||
3374 | // CR_Replace resolution. Collect a set of end points that can be used to | |||
3375 | // restore the live range after joining. | |||
3376 | SmallVector<SlotIndex, 8> EndPoints; | |||
3377 | LHSVals.pruneValues(RHSVals, EndPoints, false); | |||
3378 | RHSVals.pruneValues(LHSVals, EndPoints, false); | |||
3379 | ||||
3380 | LHSVals.removeImplicitDefs(); | |||
3381 | RHSVals.removeImplicitDefs(); | |||
3382 | ||||
3383 | LRange.verify(); | |||
3384 | RRange.verify(); | |||
3385 | ||||
3386 | // Join RRange into LHS. | |||
3387 | LRange.join(RRange, LHSVals.getAssignments(), RHSVals.getAssignments(), | |||
3388 | NewVNInfo); | |||
3389 | ||||
3390 | LLVM_DEBUG(dbgs() << "\t\tjoined lanes: " << PrintLaneMask(LaneMask)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tjoined lanes: " << PrintLaneMask(LaneMask) << ' ' << LRange << "\n"; } } while (false) | |||
3391 | << ' ' << LRange << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tjoined lanes: " << PrintLaneMask(LaneMask) << ' ' << LRange << "\n"; } } while (false); | |||
3392 | if (EndPoints.empty()) | |||
3393 | return; | |||
3394 | ||||
3395 | // Recompute the parts of the live range we had to remove because of | |||
3396 | // CR_Replace conflicts. | |||
3397 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3398 | dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3399 | for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3400 | dbgs() << EndPoints[i];do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3401 | if (i != n-1)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3402 | dbgs() << ',';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3403 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3404 | dbgs() << ": " << LRange << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false) | |||
3405 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LRange << '\n'; }; } } while (false); | |||
3406 | LIS->extendToIndices(LRange, EndPoints); | |||
3407 | } | |||
3408 | ||||
3409 | void RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI, | |||
3410 | const LiveRange &ToMerge, | |||
3411 | LaneBitmask LaneMask, | |||
3412 | CoalescerPair &CP, | |||
3413 | unsigned ComposeSubRegIdx) { | |||
3414 | BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); | |||
3415 | LI.refineSubRanges( | |||
3416 | Allocator, LaneMask, | |||
3417 | [this, &Allocator, &ToMerge, &CP](LiveInterval::SubRange &SR) { | |||
3418 | if (SR.empty()) { | |||
3419 | SR.assign(ToMerge, Allocator); | |||
3420 | } else { | |||
3421 | // joinSubRegRange() destroys the merged range, so we need a copy. | |||
3422 | LiveRange RangeCopy(ToMerge, Allocator); | |||
3423 | joinSubRegRanges(SR, RangeCopy, SR.LaneMask, CP); | |||
3424 | } | |||
3425 | }, | |||
3426 | *LIS->getSlotIndexes(), *TRI, ComposeSubRegIdx); | |||
3427 | } | |||
3428 | ||||
3429 | bool RegisterCoalescer::isHighCostLiveInterval(LiveInterval &LI) { | |||
3430 | if (LI.valnos.size() < LargeIntervalSizeThreshold) | |||
3431 | return false; | |||
3432 | auto &Counter = LargeLIVisitCounter[LI.reg()]; | |||
3433 | if (Counter < LargeIntervalFreqThreshold) { | |||
3434 | Counter++; | |||
3435 | return false; | |||
3436 | } | |||
3437 | return true; | |||
3438 | } | |||
3439 | ||||
3440 | bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) { | |||
3441 | SmallVector<VNInfo*, 16> NewVNInfo; | |||
3442 | LiveInterval &RHS = LIS->getInterval(CP.getSrcReg()); | |||
3443 | LiveInterval &LHS = LIS->getInterval(CP.getDstReg()); | |||
3444 | bool TrackSubRegLiveness = MRI->shouldTrackSubRegLiveness(*CP.getNewRC()); | |||
3445 | JoinVals RHSVals(RHS, CP.getSrcReg(), CP.getSrcIdx(), LaneBitmask::getNone(), | |||
3446 | NewVNInfo, CP, LIS, TRI, false, TrackSubRegLiveness); | |||
3447 | JoinVals LHSVals(LHS, CP.getDstReg(), CP.getDstIdx(), LaneBitmask::getNone(), | |||
3448 | NewVNInfo, CP, LIS, TRI, false, TrackSubRegLiveness); | |||
3449 | ||||
3450 | LLVM_DEBUG(dbgs() << "\t\tRHS = " << RHS << "\n\t\tLHS = " << LHS << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tRHS = " << RHS << "\n\t\tLHS = " << LHS << '\n'; } } while (false); | |||
3451 | ||||
3452 | if (isHighCostLiveInterval(LHS) || isHighCostLiveInterval(RHS)) | |||
3453 | return false; | |||
3454 | ||||
3455 | // First compute NewVNInfo and the simple value mappings. | |||
3456 | // Detect impossible conflicts early. | |||
3457 | if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals)) | |||
3458 | return false; | |||
3459 | ||||
3460 | // Some conflicts can only be resolved after all values have been mapped. | |||
3461 | if (!LHSVals.resolveConflicts(RHSVals) || !RHSVals.resolveConflicts(LHSVals)) | |||
3462 | return false; | |||
3463 | ||||
3464 | // All clear, the live ranges can be merged. | |||
3465 | if (RHS.hasSubRanges() || LHS.hasSubRanges()) { | |||
3466 | BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); | |||
3467 | ||||
3468 | // Transform lanemasks from the LHS to masks in the coalesced register and | |||
3469 | // create initial subranges if necessary. | |||
3470 | unsigned DstIdx = CP.getDstIdx(); | |||
3471 | if (!LHS.hasSubRanges()) { | |||
3472 | LaneBitmask Mask = DstIdx == 0 ? CP.getNewRC()->getLaneMask() | |||
3473 | : TRI->getSubRegIndexLaneMask(DstIdx); | |||
3474 | // LHS must support subregs or we wouldn't be in this codepath. | |||
3475 | assert(Mask.any())((Mask.any()) ? static_cast<void> (0) : __assert_fail ( "Mask.any()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3475, __PRETTY_FUNCTION__)); | |||
3476 | LHS.createSubRangeFrom(Allocator, Mask, LHS); | |||
3477 | } else if (DstIdx != 0) { | |||
3478 | // Transform LHS lanemasks to new register class if necessary. | |||
3479 | for (LiveInterval::SubRange &R : LHS.subranges()) { | |||
3480 | LaneBitmask Mask = TRI->composeSubRegIndexLaneMask(DstIdx, R.LaneMask); | |||
3481 | R.LaneMask = Mask; | |||
3482 | } | |||
3483 | } | |||
3484 | LLVM_DEBUG(dbgs() << "\t\tLHST = " << printReg(CP.getDstReg()) << ' ' << LHSdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tLHST = " << printReg (CP.getDstReg()) << ' ' << LHS << '\n'; } } while (false) | |||
3485 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t\tLHST = " << printReg (CP.getDstReg()) << ' ' << LHS << '\n'; } } while (false); | |||
3486 | ||||
3487 | // Determine lanemasks of RHS in the coalesced register and merge subranges. | |||
3488 | unsigned SrcIdx = CP.getSrcIdx(); | |||
3489 | if (!RHS.hasSubRanges()) { | |||
3490 | LaneBitmask Mask = SrcIdx == 0 ? CP.getNewRC()->getLaneMask() | |||
3491 | : TRI->getSubRegIndexLaneMask(SrcIdx); | |||
3492 | mergeSubRangeInto(LHS, RHS, Mask, CP, DstIdx); | |||
3493 | } else { | |||
3494 | // Pair up subranges and merge. | |||
3495 | for (LiveInterval::SubRange &R : RHS.subranges()) { | |||
3496 | LaneBitmask Mask = TRI->composeSubRegIndexLaneMask(SrcIdx, R.LaneMask); | |||
3497 | mergeSubRangeInto(LHS, R, Mask, CP, DstIdx); | |||
3498 | } | |||
3499 | } | |||
3500 | LLVM_DEBUG(dbgs() << "\tJoined SubRanges " << LHS << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tJoined SubRanges " << LHS << "\n"; } } while (false); | |||
3501 | ||||
3502 | // Pruning implicit defs from subranges may result in the main range | |||
3503 | // having stale segments. | |||
3504 | LHSVals.pruneMainSegments(LHS, ShrinkMainRange); | |||
3505 | ||||
3506 | LHSVals.pruneSubRegValues(LHS, ShrinkMask); | |||
3507 | RHSVals.pruneSubRegValues(LHS, ShrinkMask); | |||
3508 | } | |||
3509 | ||||
3510 | // The merging algorithm in LiveInterval::join() can't handle conflicting | |||
3511 | // value mappings, so we need to remove any live ranges that overlap a | |||
3512 | // CR_Replace resolution. Collect a set of end points that can be used to | |||
3513 | // restore the live range after joining. | |||
3514 | SmallVector<SlotIndex, 8> EndPoints; | |||
3515 | LHSVals.pruneValues(RHSVals, EndPoints, true); | |||
3516 | RHSVals.pruneValues(LHSVals, EndPoints, true); | |||
3517 | ||||
3518 | // Erase COPY and IMPLICIT_DEF instructions. This may cause some external | |||
3519 | // registers to require trimming. | |||
3520 | SmallVector<Register, 8> ShrinkRegs; | |||
3521 | LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs, &LHS); | |||
3522 | RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs); | |||
3523 | while (!ShrinkRegs.empty()) | |||
3524 | shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val())); | |||
3525 | ||||
3526 | // Scan and mark undef any DBG_VALUEs that would refer to a different value. | |||
3527 | checkMergingChangesDbgValues(CP, LHS, LHSVals, RHS, RHSVals); | |||
3528 | ||||
3529 | // Join RHS into LHS. | |||
3530 | LHS.join(RHS, LHSVals.getAssignments(), RHSVals.getAssignments(), NewVNInfo); | |||
3531 | ||||
3532 | // Kill flags are going to be wrong if the live ranges were overlapping. | |||
3533 | // Eventually, we should simply clear all kill flags when computing live | |||
3534 | // ranges. They are reinserted after register allocation. | |||
3535 | MRI->clearKillFlags(LHS.reg()); | |||
3536 | MRI->clearKillFlags(RHS.reg()); | |||
3537 | ||||
3538 | if (!EndPoints.empty()) { | |||
3539 | // Recompute the parts of the live range we had to remove because of | |||
3540 | // CR_Replace conflicts. | |||
3541 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3542 | dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3543 | for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3544 | dbgs() << EndPoints[i];do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3545 | if (i != n-1)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3546 | dbgs() << ',';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3547 | }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3548 | dbgs() << ": " << LHS << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false) | |||
3549 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\t\trestoring liveness to " << EndPoints.size() << " points: "; for (unsigned i = 0, n = EndPoints.size(); i != n; ++i) { dbgs() << EndPoints [i]; if (i != n-1) dbgs() << ','; } dbgs() << ": " << LHS << '\n'; }; } } while (false); | |||
3550 | LIS->extendToIndices((LiveRange&)LHS, EndPoints); | |||
3551 | } | |||
3552 | ||||
3553 | return true; | |||
3554 | } | |||
3555 | ||||
3556 | bool RegisterCoalescer::joinIntervals(CoalescerPair &CP) { | |||
3557 | return CP.isPhys() ? joinReservedPhysReg(CP) : joinVirtRegs(CP); | |||
3558 | } | |||
3559 | ||||
3560 | void RegisterCoalescer::buildVRegToDbgValueMap(MachineFunction &MF) | |||
3561 | { | |||
3562 | const SlotIndexes &Slots = *LIS->getSlotIndexes(); | |||
3563 | SmallVector<MachineInstr *, 8> ToInsert; | |||
3564 | ||||
3565 | // After collecting a block of DBG_VALUEs into ToInsert, enter them into the | |||
3566 | // vreg => DbgValueLoc map. | |||
3567 | auto CloseNewDVRange = [this, &ToInsert](SlotIndex Slot) { | |||
3568 | for (auto *X : ToInsert) | |||
3569 | DbgVRegToValues[X->getDebugOperand(0).getReg()].push_back({Slot, X}); | |||
3570 | ||||
3571 | ToInsert.clear(); | |||
3572 | }; | |||
3573 | ||||
3574 | // Iterate over all instructions, collecting them into the ToInsert vector. | |||
3575 | // Once a non-debug instruction is found, record the slot index of the | |||
3576 | // collected DBG_VALUEs. | |||
3577 | for (auto &MBB : MF) { | |||
3578 | SlotIndex CurrentSlot = Slots.getMBBStartIdx(&MBB); | |||
3579 | ||||
3580 | for (auto &MI : MBB) { | |||
3581 | if (MI.isDebugValue() && MI.getDebugOperand(0).isReg() && | |||
3582 | MI.getDebugOperand(0).getReg().isVirtual()) { | |||
3583 | ToInsert.push_back(&MI); | |||
3584 | } else if (!MI.isDebugInstr()) { | |||
3585 | CurrentSlot = Slots.getInstructionIndex(MI); | |||
3586 | CloseNewDVRange(CurrentSlot); | |||
3587 | } | |||
3588 | } | |||
3589 | ||||
3590 | // Close range of DBG_VALUEs at the end of blocks. | |||
3591 | CloseNewDVRange(Slots.getMBBEndIdx(&MBB)); | |||
3592 | } | |||
3593 | ||||
3594 | // Sort all DBG_VALUEs we've seen by slot number. | |||
3595 | for (auto &Pair : DbgVRegToValues) | |||
3596 | llvm::sort(Pair.second); | |||
3597 | } | |||
3598 | ||||
3599 | void RegisterCoalescer::checkMergingChangesDbgValues(CoalescerPair &CP, | |||
3600 | LiveRange &LHS, | |||
3601 | JoinVals &LHSVals, | |||
3602 | LiveRange &RHS, | |||
3603 | JoinVals &RHSVals) { | |||
3604 | auto ScanForDstReg = [&](Register Reg) { | |||
3605 | checkMergingChangesDbgValuesImpl(Reg, RHS, LHS, LHSVals); | |||
3606 | }; | |||
3607 | ||||
3608 | auto ScanForSrcReg = [&](Register Reg) { | |||
3609 | checkMergingChangesDbgValuesImpl(Reg, LHS, RHS, RHSVals); | |||
3610 | }; | |||
3611 | ||||
3612 | // Scan for potentially unsound DBG_VALUEs: examine first the register number | |||
3613 | // Reg, and then any other vregs that may have been merged into it. | |||
3614 | auto PerformScan = [this](Register Reg, std::function<void(Register)> Func) { | |||
3615 | Func(Reg); | |||
3616 | if (DbgMergedVRegNums.count(Reg)) | |||
3617 | for (Register X : DbgMergedVRegNums[Reg]) | |||
3618 | Func(X); | |||
3619 | }; | |||
3620 | ||||
3621 | // Scan for unsound updates of both the source and destination register. | |||
3622 | PerformScan(CP.getSrcReg(), ScanForSrcReg); | |||
3623 | PerformScan(CP.getDstReg(), ScanForDstReg); | |||
3624 | } | |||
3625 | ||||
3626 | void RegisterCoalescer::checkMergingChangesDbgValuesImpl(Register Reg, | |||
3627 | LiveRange &OtherLR, | |||
3628 | LiveRange &RegLR, | |||
3629 | JoinVals &RegVals) { | |||
3630 | // Are there any DBG_VALUEs to examine? | |||
3631 | auto VRegMapIt = DbgVRegToValues.find(Reg); | |||
3632 | if (VRegMapIt == DbgVRegToValues.end()) | |||
3633 | return; | |||
3634 | ||||
3635 | auto &DbgValueSet = VRegMapIt->second; | |||
3636 | auto DbgValueSetIt = DbgValueSet.begin(); | |||
3637 | auto SegmentIt = OtherLR.begin(); | |||
3638 | ||||
3639 | bool LastUndefResult = false; | |||
3640 | SlotIndex LastUndefIdx; | |||
3641 | ||||
3642 | // If the "Other" register is live at a slot Idx, test whether Reg can | |||
3643 | // safely be merged with it, or should be marked undef. | |||
3644 | auto ShouldUndef = [&RegVals, &RegLR, &LastUndefResult, | |||
3645 | &LastUndefIdx](SlotIndex Idx) -> bool { | |||
3646 | // Our worst-case performance typically happens with asan, causing very | |||
3647 | // many DBG_VALUEs of the same location. Cache a copy of the most recent | |||
3648 | // result for this edge-case. | |||
3649 | if (LastUndefIdx == Idx) | |||
3650 | return LastUndefResult; | |||
3651 | ||||
3652 | // If the other range was live, and Reg's was not, the register coalescer | |||
3653 | // will not have tried to resolve any conflicts. We don't know whether | |||
3654 | // the DBG_VALUE will refer to the same value number, so it must be made | |||
3655 | // undef. | |||
3656 | auto OtherIt = RegLR.find(Idx); | |||
3657 | if (OtherIt == RegLR.end()) | |||
3658 | return true; | |||
3659 | ||||
3660 | // Both the registers were live: examine the conflict resolution record for | |||
3661 | // the value number Reg refers to. CR_Keep meant that this value number | |||
3662 | // "won" and the merged register definitely refers to that value. CR_Erase | |||
3663 | // means the value number was a redundant copy of the other value, which | |||
3664 | // was coalesced and Reg deleted. It's safe to refer to the other register | |||
3665 | // (which will be the source of the copy). | |||
3666 | auto Resolution = RegVals.getResolution(OtherIt->valno->id); | |||
3667 | LastUndefResult = Resolution != JoinVals::CR_Keep && | |||
3668 | Resolution != JoinVals::CR_Erase; | |||
3669 | LastUndefIdx = Idx; | |||
3670 | return LastUndefResult; | |||
3671 | }; | |||
3672 | ||||
3673 | // Iterate over both the live-range of the "Other" register, and the set of | |||
3674 | // DBG_VALUEs for Reg at the same time. Advance whichever one has the lowest | |||
3675 | // slot index. This relies on the DbgValueSet being ordered. | |||
3676 | while (DbgValueSetIt != DbgValueSet.end() && SegmentIt != OtherLR.end()) { | |||
3677 | if (DbgValueSetIt->first < SegmentIt->end) { | |||
3678 | // "Other" is live and there is a DBG_VALUE of Reg: test if we should | |||
3679 | // set it undef. | |||
3680 | if (DbgValueSetIt->first >= SegmentIt->start && | |||
3681 | DbgValueSetIt->second->getDebugOperand(0).getReg() != 0 && | |||
3682 | ShouldUndef(DbgValueSetIt->first)) { | |||
3683 | // Mark undef, erase record of this DBG_VALUE to avoid revisiting. | |||
3684 | DbgValueSetIt->second->setDebugValueUndef(); | |||
3685 | continue; | |||
3686 | } | |||
3687 | ++DbgValueSetIt; | |||
3688 | } else { | |||
3689 | ++SegmentIt; | |||
3690 | } | |||
3691 | } | |||
3692 | } | |||
3693 | ||||
3694 | namespace { | |||
3695 | ||||
3696 | /// Information concerning MBB coalescing priority. | |||
3697 | struct MBBPriorityInfo { | |||
3698 | MachineBasicBlock *MBB; | |||
3699 | unsigned Depth; | |||
3700 | bool IsSplit; | |||
3701 | ||||
3702 | MBBPriorityInfo(MachineBasicBlock *mbb, unsigned depth, bool issplit) | |||
3703 | : MBB(mbb), Depth(depth), IsSplit(issplit) {} | |||
3704 | }; | |||
3705 | ||||
3706 | } // end anonymous namespace | |||
3707 | ||||
3708 | /// C-style comparator that sorts first based on the loop depth of the basic | |||
3709 | /// block (the unsigned), and then on the MBB number. | |||
3710 | /// | |||
3711 | /// EnableGlobalCopies assumes that the primary sort key is loop depth. | |||
3712 | static int compareMBBPriority(const MBBPriorityInfo *LHS, | |||
3713 | const MBBPriorityInfo *RHS) { | |||
3714 | // Deeper loops first | |||
3715 | if (LHS->Depth != RHS->Depth) | |||
3716 | return LHS->Depth > RHS->Depth ? -1 : 1; | |||
3717 | ||||
3718 | // Try to unsplit critical edges next. | |||
3719 | if (LHS->IsSplit != RHS->IsSplit) | |||
3720 | return LHS->IsSplit ? -1 : 1; | |||
3721 | ||||
3722 | // Prefer blocks that are more connected in the CFG. This takes care of | |||
3723 | // the most difficult copies first while intervals are short. | |||
3724 | unsigned cl = LHS->MBB->pred_size() + LHS->MBB->succ_size(); | |||
3725 | unsigned cr = RHS->MBB->pred_size() + RHS->MBB->succ_size(); | |||
3726 | if (cl != cr) | |||
3727 | return cl > cr ? -1 : 1; | |||
3728 | ||||
3729 | // As a last resort, sort by block number. | |||
3730 | return LHS->MBB->getNumber() < RHS->MBB->getNumber() ? -1 : 1; | |||
3731 | } | |||
3732 | ||||
3733 | /// \returns true if the given copy uses or defines a local live range. | |||
3734 | static bool isLocalCopy(MachineInstr *Copy, const LiveIntervals *LIS) { | |||
3735 | if (!Copy->isCopy()) | |||
3736 | return false; | |||
3737 | ||||
3738 | if (Copy->getOperand(1).isUndef()) | |||
3739 | return false; | |||
3740 | ||||
3741 | Register SrcReg = Copy->getOperand(1).getReg(); | |||
3742 | Register DstReg = Copy->getOperand(0).getReg(); | |||
3743 | if (Register::isPhysicalRegister(SrcReg) || | |||
3744 | Register::isPhysicalRegister(DstReg)) | |||
3745 | return false; | |||
3746 | ||||
3747 | return LIS->intervalIsInOneMBB(LIS->getInterval(SrcReg)) | |||
3748 | || LIS->intervalIsInOneMBB(LIS->getInterval(DstReg)); | |||
3749 | } | |||
3750 | ||||
3751 | void RegisterCoalescer::lateLiveIntervalUpdate() { | |||
3752 | for (Register reg : ToBeUpdated) { | |||
3753 | if (!LIS->hasInterval(reg)) | |||
3754 | continue; | |||
3755 | LiveInterval &LI = LIS->getInterval(reg); | |||
3756 | shrinkToUses(&LI, &DeadDefs); | |||
3757 | if (!DeadDefs.empty()) | |||
3758 | eliminateDeadDefs(); | |||
3759 | } | |||
3760 | ToBeUpdated.clear(); | |||
3761 | } | |||
3762 | ||||
3763 | bool RegisterCoalescer:: | |||
3764 | copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList) { | |||
3765 | bool Progress = false; | |||
3766 | for (unsigned i = 0, e = CurrList.size(); i != e; ++i) { | |||
3767 | if (!CurrList[i]) | |||
3768 | continue; | |||
3769 | // Skip instruction pointers that have already been erased, for example by | |||
3770 | // dead code elimination. | |||
3771 | if (ErasedInstrs.count(CurrList[i])) { | |||
3772 | CurrList[i] = nullptr; | |||
3773 | continue; | |||
3774 | } | |||
3775 | bool Again = false; | |||
3776 | bool Success = joinCopy(CurrList[i], Again); | |||
3777 | Progress |= Success; | |||
3778 | if (Success || !Again) | |||
3779 | CurrList[i] = nullptr; | |||
3780 | } | |||
3781 | return Progress; | |||
3782 | } | |||
3783 | ||||
3784 | /// Check if DstReg is a terminal node. | |||
3785 | /// I.e., it does not have any affinity other than \p Copy. | |||
3786 | static bool isTerminalReg(Register DstReg, const MachineInstr &Copy, | |||
3787 | const MachineRegisterInfo *MRI) { | |||
3788 | assert(Copy.isCopyLike())((Copy.isCopyLike()) ? static_cast<void> (0) : __assert_fail ("Copy.isCopyLike()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3788, __PRETTY_FUNCTION__)); | |||
3789 | // Check if the destination of this copy as any other affinity. | |||
3790 | for (const MachineInstr &MI : MRI->reg_nodbg_instructions(DstReg)) | |||
3791 | if (&MI != &Copy && MI.isCopyLike()) | |||
3792 | return false; | |||
3793 | return true; | |||
3794 | } | |||
3795 | ||||
3796 | bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const { | |||
3797 | assert(Copy.isCopyLike())((Copy.isCopyLike()) ? static_cast<void> (0) : __assert_fail ("Copy.isCopyLike()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3797, __PRETTY_FUNCTION__)); | |||
3798 | if (!UseTerminalRule) | |||
3799 | return false; | |||
3800 | Register SrcReg, DstReg; | |||
3801 | unsigned SrcSubReg, DstSubReg; | |||
3802 | if (!isMoveInstr(*TRI, &Copy, SrcReg, DstReg, SrcSubReg, DstSubReg)) | |||
3803 | return false; | |||
3804 | // Check if the destination of this copy has any other affinity. | |||
3805 | if (DstReg.isPhysical() || | |||
3806 | // If SrcReg is a physical register, the copy won't be coalesced. | |||
3807 | // Ignoring it may have other side effect (like missing | |||
3808 | // rematerialization). So keep it. | |||
3809 | SrcReg.isPhysical() || !isTerminalReg(DstReg, Copy, MRI)) | |||
3810 | return false; | |||
3811 | ||||
3812 | // DstReg is a terminal node. Check if it interferes with any other | |||
3813 | // copy involving SrcReg. | |||
3814 | const MachineBasicBlock *OrigBB = Copy.getParent(); | |||
3815 | const LiveInterval &DstLI = LIS->getInterval(DstReg); | |||
3816 | for (const MachineInstr &MI : MRI->reg_nodbg_instructions(SrcReg)) { | |||
3817 | // Technically we should check if the weight of the new copy is | |||
3818 | // interesting compared to the other one and update the weight | |||
3819 | // of the copies accordingly. However, this would only work if | |||
3820 | // we would gather all the copies first then coalesce, whereas | |||
3821 | // right now we interleave both actions. | |||
3822 | // For now, just consider the copies that are in the same block. | |||
3823 | if (&MI == &Copy || !MI.isCopyLike() || MI.getParent() != OrigBB) | |||
3824 | continue; | |||
3825 | Register OtherSrcReg, OtherReg; | |||
3826 | unsigned OtherSrcSubReg, OtherSubReg; | |||
3827 | if (!isMoveInstr(*TRI, &Copy, OtherSrcReg, OtherReg, OtherSrcSubReg, | |||
3828 | OtherSubReg)) | |||
3829 | return false; | |||
3830 | if (OtherReg == SrcReg) | |||
3831 | OtherReg = OtherSrcReg; | |||
3832 | // Check if OtherReg is a non-terminal. | |||
3833 | if (Register::isPhysicalRegister(OtherReg) || | |||
3834 | isTerminalReg(OtherReg, MI, MRI)) | |||
3835 | continue; | |||
3836 | // Check that OtherReg interfere with DstReg. | |||
3837 | if (LIS->getInterval(OtherReg).overlaps(DstLI)) { | |||
3838 | LLVM_DEBUG(dbgs() << "Apply terminal rule for: " << printReg(DstReg)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Apply terminal rule for: " << printReg(DstReg) << '\n'; } } while (false) | |||
3839 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Apply terminal rule for: " << printReg(DstReg) << '\n'; } } while (false); | |||
3840 | return true; | |||
3841 | } | |||
3842 | } | |||
3843 | return false; | |||
3844 | } | |||
3845 | ||||
3846 | void | |||
3847 | RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) { | |||
3848 | LLVM_DEBUG(dbgs() << MBB->getName() << ":\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << MBB->getName() << ":\n" ; } } while (false); | |||
3849 | ||||
3850 | // Collect all copy-like instructions in MBB. Don't start coalescing anything | |||
3851 | // yet, it might invalidate the iterator. | |||
3852 | const unsigned PrevSize = WorkList.size(); | |||
3853 | if (JoinGlobalCopies) { | |||
3854 | SmallVector<MachineInstr*, 2> LocalTerminals; | |||
3855 | SmallVector<MachineInstr*, 2> GlobalTerminals; | |||
3856 | // Coalesce copies bottom-up to coalesce local defs before local uses. They | |||
3857 | // are not inherently easier to resolve, but slightly preferable until we | |||
3858 | // have local live range splitting. In particular this is required by | |||
3859 | // cmp+jmp macro fusion. | |||
3860 | for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); | |||
3861 | MII != E; ++MII) { | |||
3862 | if (!MII->isCopyLike()) | |||
3863 | continue; | |||
3864 | bool ApplyTerminalRule = applyTerminalRule(*MII); | |||
3865 | if (isLocalCopy(&(*MII), LIS)) { | |||
3866 | if (ApplyTerminalRule) | |||
3867 | LocalTerminals.push_back(&(*MII)); | |||
3868 | else | |||
3869 | LocalWorkList.push_back(&(*MII)); | |||
3870 | } else { | |||
3871 | if (ApplyTerminalRule) | |||
3872 | GlobalTerminals.push_back(&(*MII)); | |||
3873 | else | |||
3874 | WorkList.push_back(&(*MII)); | |||
3875 | } | |||
3876 | } | |||
3877 | // Append the copies evicted by the terminal rule at the end of the list. | |||
3878 | LocalWorkList.append(LocalTerminals.begin(), LocalTerminals.end()); | |||
3879 | WorkList.append(GlobalTerminals.begin(), GlobalTerminals.end()); | |||
3880 | } | |||
3881 | else { | |||
3882 | SmallVector<MachineInstr*, 2> Terminals; | |||
3883 | for (MachineInstr &MII : *MBB) | |||
3884 | if (MII.isCopyLike()) { | |||
3885 | if (applyTerminalRule(MII)) | |||
3886 | Terminals.push_back(&MII); | |||
3887 | else | |||
3888 | WorkList.push_back(&MII); | |||
3889 | } | |||
3890 | // Append the copies evicted by the terminal rule at the end of the list. | |||
3891 | WorkList.append(Terminals.begin(), Terminals.end()); | |||
3892 | } | |||
3893 | // Try coalescing the collected copies immediately, and remove the nulls. | |||
3894 | // This prevents the WorkList from getting too large since most copies are | |||
3895 | // joinable on the first attempt. | |||
3896 | MutableArrayRef<MachineInstr*> | |||
3897 | CurrList(WorkList.begin() + PrevSize, WorkList.end()); | |||
3898 | if (copyCoalesceWorkList(CurrList)) | |||
3899 | WorkList.erase(std::remove(WorkList.begin() + PrevSize, WorkList.end(), | |||
3900 | nullptr), WorkList.end()); | |||
3901 | } | |||
3902 | ||||
3903 | void RegisterCoalescer::coalesceLocals() { | |||
3904 | copyCoalesceWorkList(LocalWorkList); | |||
3905 | for (unsigned j = 0, je = LocalWorkList.size(); j != je; ++j) { | |||
3906 | if (LocalWorkList[j]) | |||
3907 | WorkList.push_back(LocalWorkList[j]); | |||
3908 | } | |||
3909 | LocalWorkList.clear(); | |||
3910 | } | |||
3911 | ||||
3912 | void RegisterCoalescer::joinAllIntervals() { | |||
3913 | LLVM_DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "********** JOINING INTERVALS ***********\n" ; } } while (false); | |||
3914 | assert(WorkList.empty() && LocalWorkList.empty() && "Old data still around.")((WorkList.empty() && LocalWorkList.empty() && "Old data still around.") ? static_cast<void> (0) : __assert_fail ("WorkList.empty() && LocalWorkList.empty() && \"Old data still around.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 3914, __PRETTY_FUNCTION__)); | |||
3915 | ||||
3916 | std::vector<MBBPriorityInfo> MBBs; | |||
3917 | MBBs.reserve(MF->size()); | |||
3918 | for (MachineFunction::iterator I = MF->begin(), E = MF->end(); I != E; ++I) { | |||
3919 | MachineBasicBlock *MBB = &*I; | |||
3920 | MBBs.push_back(MBBPriorityInfo(MBB, Loops->getLoopDepth(MBB), | |||
3921 | JoinSplitEdges && isSplitEdge(MBB))); | |||
3922 | } | |||
3923 | array_pod_sort(MBBs.begin(), MBBs.end(), compareMBBPriority); | |||
3924 | ||||
3925 | // Coalesce intervals in MBB priority order. | |||
3926 | unsigned CurrDepth = std::numeric_limits<unsigned>::max(); | |||
3927 | for (unsigned i = 0, e = MBBs.size(); i != e; ++i) { | |||
3928 | // Try coalescing the collected local copies for deeper loops. | |||
3929 | if (JoinGlobalCopies && MBBs[i].Depth < CurrDepth) { | |||
3930 | coalesceLocals(); | |||
3931 | CurrDepth = MBBs[i].Depth; | |||
3932 | } | |||
3933 | copyCoalesceInMBB(MBBs[i].MBB); | |||
3934 | } | |||
3935 | lateLiveIntervalUpdate(); | |||
3936 | coalesceLocals(); | |||
3937 | ||||
3938 | // Joining intervals can allow other intervals to be joined. Iteratively join | |||
3939 | // until we make no progress. | |||
3940 | while (copyCoalesceWorkList(WorkList)) | |||
3941 | /* empty */ ; | |||
3942 | lateLiveIntervalUpdate(); | |||
3943 | } | |||
3944 | ||||
3945 | void RegisterCoalescer::releaseMemory() { | |||
3946 | ErasedInstrs.clear(); | |||
3947 | WorkList.clear(); | |||
3948 | DeadDefs.clear(); | |||
3949 | InflateRegs.clear(); | |||
3950 | LargeLIVisitCounter.clear(); | |||
3951 | } | |||
3952 | ||||
3953 | bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { | |||
3954 | LLVM_DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "********** SIMPLE REGISTER COALESCING **********\n" << "********** Function: " << fn.getName() << '\n'; } } while (false) | |||
3955 | << "********** Function: " << fn.getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "********** SIMPLE REGISTER COALESCING **********\n" << "********** Function: " << fn.getName() << '\n'; } } while (false); | |||
3956 | ||||
3957 | // Variables changed between a setjmp and a longjump can have undefined value | |||
3958 | // after the longjmp. This behaviour can be observed if such a variable is | |||
3959 | // spilled, so longjmp won't restore the value in the spill slot. | |||
3960 | // RegisterCoalescer should not run in functions with a setjmp to avoid | |||
3961 | // merging such undefined variables with predictable ones. | |||
3962 | // | |||
3963 | // TODO: Could specifically disable coalescing registers live across setjmp | |||
3964 | // calls | |||
3965 | if (fn.exposesReturnsTwice()) { | |||
3966 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "* Skipped as it exposes funcions that returns twice.\n" ; } } while (false) | |||
3967 | dbgs() << "* Skipped as it exposes funcions that returns twice.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "* Skipped as it exposes funcions that returns twice.\n" ; } } while (false); | |||
3968 | return false; | |||
3969 | } | |||
3970 | ||||
3971 | MF = &fn; | |||
3972 | MRI = &fn.getRegInfo(); | |||
3973 | const TargetSubtargetInfo &STI = fn.getSubtarget(); | |||
3974 | TRI = STI.getRegisterInfo(); | |||
3975 | TII = STI.getInstrInfo(); | |||
3976 | LIS = &getAnalysis<LiveIntervals>(); | |||
3977 | AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); | |||
3978 | Loops = &getAnalysis<MachineLoopInfo>(); | |||
3979 | if (EnableGlobalCopies == cl::BOU_UNSET) | |||
3980 | JoinGlobalCopies = STI.enableJoinGlobalCopies(); | |||
3981 | else | |||
3982 | JoinGlobalCopies = (EnableGlobalCopies == cl::BOU_TRUE); | |||
3983 | ||||
3984 | // The MachineScheduler does not currently require JoinSplitEdges. This will | |||
3985 | // either be enabled unconditionally or replaced by a more general live range | |||
3986 | // splitting optimization. | |||
3987 | JoinSplitEdges = EnableJoinSplits; | |||
3988 | ||||
3989 | if (VerifyCoalescing) | |||
3990 | MF->verify(this, "Before register coalescing"); | |||
3991 | ||||
3992 | DbgVRegToValues.clear(); | |||
3993 | DbgMergedVRegNums.clear(); | |||
3994 | buildVRegToDbgValueMap(fn); | |||
3995 | ||||
3996 | RegClassInfo.runOnMachineFunction(fn); | |||
3997 | ||||
3998 | // Join (coalesce) intervals if requested. | |||
3999 | if (EnableJoining) | |||
4000 | joinAllIntervals(); | |||
4001 | ||||
4002 | // After deleting a lot of copies, register classes may be less constrained. | |||
4003 | // Removing sub-register operands may allow GR32_ABCD -> GR32 and DPR_VFP2 -> | |||
4004 | // DPR inflation. | |||
4005 | array_pod_sort(InflateRegs.begin(), InflateRegs.end()); | |||
4006 | InflateRegs.erase(std::unique(InflateRegs.begin(), InflateRegs.end()), | |||
4007 | InflateRegs.end()); | |||
4008 | LLVM_DEBUG(dbgs() << "Trying to inflate " << InflateRegs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Trying to inflate " << InflateRegs.size() << " regs.\n"; } } while (false) | |||
4009 | << " regs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Trying to inflate " << InflateRegs.size() << " regs.\n"; } } while (false); | |||
4010 | for (unsigned i = 0, e = InflateRegs.size(); i != e; ++i) { | |||
4011 | Register Reg = InflateRegs[i]; | |||
4012 | if (MRI->reg_nodbg_empty(Reg)) | |||
4013 | continue; | |||
4014 | if (MRI->recomputeRegClass(Reg)) { | |||
4015 | LLVM_DEBUG(dbgs() << printReg(Reg) << " inflated to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << printReg(Reg) << " inflated to " << TRI->getRegClassName(MRI->getRegClass(Reg)) << '\n'; } } while (false) | |||
4016 | << TRI->getRegClassName(MRI->getRegClass(Reg)) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << printReg(Reg) << " inflated to " << TRI->getRegClassName(MRI->getRegClass(Reg)) << '\n'; } } while (false); | |||
4017 | ++NumInflated; | |||
4018 | ||||
4019 | LiveInterval &LI = LIS->getInterval(Reg); | |||
4020 | if (LI.hasSubRanges()) { | |||
4021 | // If the inflated register class does not support subregisters anymore | |||
4022 | // remove the subranges. | |||
4023 | if (!MRI->shouldTrackSubRegLiveness(Reg)) { | |||
4024 | LI.clearSubRanges(); | |||
4025 | } else { | |||
4026 | #ifndef NDEBUG | |||
4027 | LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg); | |||
4028 | // If subranges are still supported, then the same subregs | |||
4029 | // should still be supported. | |||
4030 | for (LiveInterval::SubRange &S : LI.subranges()) { | |||
4031 | assert((S.LaneMask & ~MaxMask).none())(((S.LaneMask & ~MaxMask).none()) ? static_cast<void> (0) : __assert_fail ("(S.LaneMask & ~MaxMask).none()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/lib/CodeGen/RegisterCoalescer.cpp" , 4031, __PRETTY_FUNCTION__)); | |||
4032 | } | |||
4033 | #endif | |||
4034 | } | |||
4035 | } | |||
4036 | } | |||
4037 | } | |||
4038 | ||||
4039 | LLVM_DEBUG(dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dump(); } } while (false); | |||
4040 | if (VerifyCoalescing) | |||
4041 | MF->verify(this, "After register coalescing"); | |||
4042 | return true; | |||
4043 | } | |||
4044 | ||||
4045 | void RegisterCoalescer::print(raw_ostream &O, const Module* m) const { | |||
4046 | LIS->print(O, m); | |||
4047 | } |
1 | //===- llvm/CodeGen/LiveInterval.h - Interval representation ----*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the LiveRange and LiveInterval classes. Given some |
10 | // numbering of each the machine instructions an interval [i, j) is said to be a |
11 | // live range for register v if there is no instruction with number j' >= j |
12 | // such that v is live at j' and there is no instruction with number i' < i such |
13 | // that v is live at i'. In this implementation ranges can have holes, |
14 | // i.e. a range might look like [1,20), [50,65), [1000,1001). Each |
15 | // individual segment is represented as an instance of LiveRange::Segment, |
16 | // and the whole range is represented as an instance of LiveRange. |
17 | // |
18 | //===----------------------------------------------------------------------===// |
19 | |
20 | #ifndef LLVM_CODEGEN_LIVEINTERVAL_H |
21 | #define LLVM_CODEGEN_LIVEINTERVAL_H |
22 | |
23 | #include "llvm/ADT/ArrayRef.h" |
24 | #include "llvm/ADT/IntEqClasses.h" |
25 | #include "llvm/ADT/STLExtras.h" |
26 | #include "llvm/ADT/SmallVector.h" |
27 | #include "llvm/ADT/iterator_range.h" |
28 | #include "llvm/CodeGen/Register.h" |
29 | #include "llvm/CodeGen/SlotIndexes.h" |
30 | #include "llvm/MC/LaneBitmask.h" |
31 | #include "llvm/Support/Allocator.h" |
32 | #include "llvm/Support/MathExtras.h" |
33 | #include <algorithm> |
34 | #include <cassert> |
35 | #include <cstddef> |
36 | #include <functional> |
37 | #include <memory> |
38 | #include <set> |
39 | #include <tuple> |
40 | #include <utility> |
41 | |
42 | namespace llvm { |
43 | |
44 | class CoalescerPair; |
45 | class LiveIntervals; |
46 | class MachineRegisterInfo; |
47 | class raw_ostream; |
48 | |
49 | /// VNInfo - Value Number Information. |
50 | /// This class holds information about a machine level values, including |
51 | /// definition and use points. |
52 | /// |
53 | class VNInfo { |
54 | public: |
55 | using Allocator = BumpPtrAllocator; |
56 | |
57 | /// The ID number of this value. |
58 | unsigned id; |
59 | |
60 | /// The index of the defining instruction. |
61 | SlotIndex def; |
62 | |
63 | /// VNInfo constructor. |
64 | VNInfo(unsigned i, SlotIndex d) : id(i), def(d) {} |
65 | |
66 | /// VNInfo constructor, copies values from orig, except for the value number. |
67 | VNInfo(unsigned i, const VNInfo &orig) : id(i), def(orig.def) {} |
68 | |
69 | /// Copy from the parameter into this VNInfo. |
70 | void copyFrom(VNInfo &src) { |
71 | def = src.def; |
72 | } |
73 | |
74 | /// Returns true if this value is defined by a PHI instruction (or was, |
75 | /// PHI instructions may have been eliminated). |
76 | /// PHI-defs begin at a block boundary, all other defs begin at register or |
77 | /// EC slots. |
78 | bool isPHIDef() const { return def.isBlock(); } |
79 | |
80 | /// Returns true if this value is unused. |
81 | bool isUnused() const { return !def.isValid(); } |
82 | |
83 | /// Mark this value as unused. |
84 | void markUnused() { def = SlotIndex(); } |
85 | }; |
86 | |
87 | /// Result of a LiveRange query. This class hides the implementation details |
88 | /// of live ranges, and it should be used as the primary interface for |
89 | /// examining live ranges around instructions. |
90 | class LiveQueryResult { |
91 | VNInfo *const EarlyVal; |
92 | VNInfo *const LateVal; |
93 | const SlotIndex EndPoint; |
94 | const bool Kill; |
95 | |
96 | public: |
97 | LiveQueryResult(VNInfo *EarlyVal, VNInfo *LateVal, SlotIndex EndPoint, |
98 | bool Kill) |
99 | : EarlyVal(EarlyVal), LateVal(LateVal), EndPoint(EndPoint), Kill(Kill) |
100 | {} |
101 | |
102 | /// Return the value that is live-in to the instruction. This is the value |
103 | /// that will be read by the instruction's use operands. Return NULL if no |
104 | /// value is live-in. |
105 | VNInfo *valueIn() const { |
106 | return EarlyVal; |
107 | } |
108 | |
109 | /// Return true if the live-in value is killed by this instruction. This |
110 | /// means that either the live range ends at the instruction, or it changes |
111 | /// value. |
112 | bool isKill() const { |
113 | return Kill; |
114 | } |
115 | |
116 | /// Return true if this instruction has a dead def. |
117 | bool isDeadDef() const { |
118 | return EndPoint.isDead(); |
119 | } |
120 | |
121 | /// Return the value leaving the instruction, if any. This can be a |
122 | /// live-through value, or a live def. A dead def returns NULL. |
123 | VNInfo *valueOut() const { |
124 | return isDeadDef() ? nullptr : LateVal; |
125 | } |
126 | |
127 | /// Returns the value alive at the end of the instruction, if any. This can |
128 | /// be a live-through value, a live def or a dead def. |
129 | VNInfo *valueOutOrDead() const { |
130 | return LateVal; |
131 | } |
132 | |
133 | /// Return the value defined by this instruction, if any. This includes |
134 | /// dead defs, it is the value created by the instruction's def operands. |
135 | VNInfo *valueDefined() const { |
136 | return EarlyVal == LateVal ? nullptr : LateVal; |
137 | } |
138 | |
139 | /// Return the end point of the last live range segment to interact with |
140 | /// the instruction, if any. |
141 | /// |
142 | /// The end point is an invalid SlotIndex only if the live range doesn't |
143 | /// intersect the instruction at all. |
144 | /// |
145 | /// The end point may be at or past the end of the instruction's basic |
146 | /// block. That means the value was live out of the block. |
147 | SlotIndex endPoint() const { |
148 | return EndPoint; |
149 | } |
150 | }; |
151 | |
152 | /// This class represents the liveness of a register, stack slot, etc. |
153 | /// It manages an ordered list of Segment objects. |
154 | /// The Segments are organized in a static single assignment form: At places |
155 | /// where a new value is defined or different values reach a CFG join a new |
156 | /// segment with a new value number is used. |
157 | class LiveRange { |
158 | public: |
159 | /// This represents a simple continuous liveness interval for a value. |
160 | /// The start point is inclusive, the end point exclusive. These intervals |
161 | /// are rendered as [start,end). |
162 | struct Segment { |
163 | SlotIndex start; // Start point of the interval (inclusive) |
164 | SlotIndex end; // End point of the interval (exclusive) |
165 | VNInfo *valno = nullptr; // identifier for the value contained in this |
166 | // segment. |
167 | |
168 | Segment() = default; |
169 | |
170 | Segment(SlotIndex S, SlotIndex E, VNInfo *V) |
171 | : start(S), end(E), valno(V) { |
172 | assert(S < E && "Cannot create empty or backwards segment")((S < E && "Cannot create empty or backwards segment" ) ? static_cast<void> (0) : __assert_fail ("S < E && \"Cannot create empty or backwards segment\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 172, __PRETTY_FUNCTION__)); |
173 | } |
174 | |
175 | /// Return true if the index is covered by this segment. |
176 | bool contains(SlotIndex I) const { |
177 | return start <= I && I < end; |
178 | } |
179 | |
180 | /// Return true if the given interval, [S, E), is covered by this segment. |
181 | bool containsInterval(SlotIndex S, SlotIndex E) const { |
182 | assert((S < E) && "Backwards interval?")(((S < E) && "Backwards interval?") ? static_cast< void> (0) : __assert_fail ("(S < E) && \"Backwards interval?\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 182, __PRETTY_FUNCTION__)); |
183 | return (start <= S && S < end) && (start < E && E <= end); |
184 | } |
185 | |
186 | bool operator<(const Segment &Other) const { |
187 | return std::tie(start, end) < std::tie(Other.start, Other.end); |
188 | } |
189 | bool operator==(const Segment &Other) const { |
190 | return start == Other.start && end == Other.end; |
191 | } |
192 | |
193 | bool operator!=(const Segment &Other) const { |
194 | return !(*this == Other); |
195 | } |
196 | |
197 | void dump() const; |
198 | }; |
199 | |
200 | using Segments = SmallVector<Segment, 2>; |
201 | using VNInfoList = SmallVector<VNInfo *, 2>; |
202 | |
203 | Segments segments; // the liveness segments |
204 | VNInfoList valnos; // value#'s |
205 | |
206 | // The segment set is used temporarily to accelerate initial computation |
207 | // of live ranges of physical registers in computeRegUnitRange. |
208 | // After that the set is flushed to the segment vector and deleted. |
209 | using SegmentSet = std::set<Segment>; |
210 | std::unique_ptr<SegmentSet> segmentSet; |
211 | |
212 | using iterator = Segments::iterator; |
213 | using const_iterator = Segments::const_iterator; |
214 | |
215 | iterator begin() { return segments.begin(); } |
216 | iterator end() { return segments.end(); } |
217 | |
218 | const_iterator begin() const { return segments.begin(); } |
219 | const_iterator end() const { return segments.end(); } |
220 | |
221 | using vni_iterator = VNInfoList::iterator; |
222 | using const_vni_iterator = VNInfoList::const_iterator; |
223 | |
224 | vni_iterator vni_begin() { return valnos.begin(); } |
225 | vni_iterator vni_end() { return valnos.end(); } |
226 | |
227 | const_vni_iterator vni_begin() const { return valnos.begin(); } |
228 | const_vni_iterator vni_end() const { return valnos.end(); } |
229 | |
230 | /// Constructs a new LiveRange object. |
231 | LiveRange(bool UseSegmentSet = false) |
232 | : segmentSet(UseSegmentSet ? std::make_unique<SegmentSet>() |
233 | : nullptr) {} |
234 | |
235 | /// Constructs a new LiveRange object by copying segments and valnos from |
236 | /// another LiveRange. |
237 | LiveRange(const LiveRange &Other, BumpPtrAllocator &Allocator) { |
238 | assert(Other.segmentSet == nullptr &&((Other.segmentSet == nullptr && "Copying of LiveRanges with active SegmentSets is not supported" ) ? static_cast<void> (0) : __assert_fail ("Other.segmentSet == nullptr && \"Copying of LiveRanges with active SegmentSets is not supported\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 239, __PRETTY_FUNCTION__)) |
239 | "Copying of LiveRanges with active SegmentSets is not supported")((Other.segmentSet == nullptr && "Copying of LiveRanges with active SegmentSets is not supported" ) ? static_cast<void> (0) : __assert_fail ("Other.segmentSet == nullptr && \"Copying of LiveRanges with active SegmentSets is not supported\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 239, __PRETTY_FUNCTION__)); |
240 | assign(Other, Allocator); |
241 | } |
242 | |
243 | /// Copies values numbers and live segments from \p Other into this range. |
244 | void assign(const LiveRange &Other, BumpPtrAllocator &Allocator) { |
245 | if (this == &Other) |
246 | return; |
247 | |
248 | assert(Other.segmentSet == nullptr &&((Other.segmentSet == nullptr && "Copying of LiveRanges with active SegmentSets is not supported" ) ? static_cast<void> (0) : __assert_fail ("Other.segmentSet == nullptr && \"Copying of LiveRanges with active SegmentSets is not supported\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 249, __PRETTY_FUNCTION__)) |
249 | "Copying of LiveRanges with active SegmentSets is not supported")((Other.segmentSet == nullptr && "Copying of LiveRanges with active SegmentSets is not supported" ) ? static_cast<void> (0) : __assert_fail ("Other.segmentSet == nullptr && \"Copying of LiveRanges with active SegmentSets is not supported\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 249, __PRETTY_FUNCTION__)); |
250 | // Duplicate valnos. |
251 | for (const VNInfo *VNI : Other.valnos) |
252 | createValueCopy(VNI, Allocator); |
253 | // Now we can copy segments and remap their valnos. |
254 | for (const Segment &S : Other.segments) |
255 | segments.push_back(Segment(S.start, S.end, valnos[S.valno->id])); |
256 | } |
257 | |
258 | /// advanceTo - Advance the specified iterator to point to the Segment |
259 | /// containing the specified position, or end() if the position is past the |
260 | /// end of the range. If no Segment contains this position, but the |
261 | /// position is in a hole, this method returns an iterator pointing to the |
262 | /// Segment immediately after the hole. |
263 | iterator advanceTo(iterator I, SlotIndex Pos) { |
264 | assert(I != end())((I != end()) ? static_cast<void> (0) : __assert_fail ( "I != end()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 264, __PRETTY_FUNCTION__)); |
265 | if (Pos >= endIndex()) |
266 | return end(); |
267 | while (I->end <= Pos) ++I; |
268 | return I; |
269 | } |
270 | |
271 | const_iterator advanceTo(const_iterator I, SlotIndex Pos) const { |
272 | assert(I != end())((I != end()) ? static_cast<void> (0) : __assert_fail ( "I != end()", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 272, __PRETTY_FUNCTION__)); |
273 | if (Pos >= endIndex()) |
274 | return end(); |
275 | while (I->end <= Pos) ++I; |
276 | return I; |
277 | } |
278 | |
279 | /// find - Return an iterator pointing to the first segment that ends after |
280 | /// Pos, or end(). This is the same as advanceTo(begin(), Pos), but faster |
281 | /// when searching large ranges. |
282 | /// |
283 | /// If Pos is contained in a Segment, that segment is returned. |
284 | /// If Pos is in a hole, the following Segment is returned. |
285 | /// If Pos is beyond endIndex, end() is returned. |
286 | iterator find(SlotIndex Pos); |
287 | |
288 | const_iterator find(SlotIndex Pos) const { |
289 | return const_cast<LiveRange*>(this)->find(Pos); |
290 | } |
291 | |
292 | void clear() { |
293 | valnos.clear(); |
294 | segments.clear(); |
295 | } |
296 | |
297 | size_t size() const { |
298 | return segments.size(); |
299 | } |
300 | |
301 | bool hasAtLeastOneValue() const { return !valnos.empty(); } |
302 | |
303 | bool containsOneValue() const { return valnos.size() == 1; } |
304 | |
305 | unsigned getNumValNums() const { return (unsigned)valnos.size(); } |
306 | |
307 | /// getValNumInfo - Returns pointer to the specified val#. |
308 | /// |
309 | inline VNInfo *getValNumInfo(unsigned ValNo) { |
310 | return valnos[ValNo]; |
311 | } |
312 | inline const VNInfo *getValNumInfo(unsigned ValNo) const { |
313 | return valnos[ValNo]; |
314 | } |
315 | |
316 | /// containsValue - Returns true if VNI belongs to this range. |
317 | bool containsValue(const VNInfo *VNI) const { |
318 | return VNI && VNI->id < getNumValNums() && VNI == getValNumInfo(VNI->id); |
319 | } |
320 | |
321 | /// getNextValue - Create a new value number and return it. MIIdx specifies |
322 | /// the instruction that defines the value number. |
323 | VNInfo *getNextValue(SlotIndex def, VNInfo::Allocator &VNInfoAllocator) { |
324 | VNInfo *VNI = |
325 | new (VNInfoAllocator) VNInfo((unsigned)valnos.size(), def); |
326 | valnos.push_back(VNI); |
327 | return VNI; |
328 | } |
329 | |
330 | /// createDeadDef - Make sure the range has a value defined at Def. |
331 | /// If one already exists, return it. Otherwise allocate a new value and |
332 | /// add liveness for a dead def. |
333 | VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator &VNIAlloc); |
334 | |
335 | /// Create a def of value @p VNI. Return @p VNI. If there already exists |
336 | /// a definition at VNI->def, the value defined there must be @p VNI. |
337 | VNInfo *createDeadDef(VNInfo *VNI); |
338 | |
339 | /// Create a copy of the given value. The new value will be identical except |
340 | /// for the Value number. |
341 | VNInfo *createValueCopy(const VNInfo *orig, |
342 | VNInfo::Allocator &VNInfoAllocator) { |
343 | VNInfo *VNI = |
344 | new (VNInfoAllocator) VNInfo((unsigned)valnos.size(), *orig); |
345 | valnos.push_back(VNI); |
346 | return VNI; |
347 | } |
348 | |
349 | /// RenumberValues - Renumber all values in order of appearance and remove |
350 | /// unused values. |
351 | void RenumberValues(); |
352 | |
353 | /// MergeValueNumberInto - This method is called when two value numbers |
354 | /// are found to be equivalent. This eliminates V1, replacing all |
355 | /// segments with the V1 value number with the V2 value number. This can |
356 | /// cause merging of V1/V2 values numbers and compaction of the value space. |
357 | VNInfo* MergeValueNumberInto(VNInfo *V1, VNInfo *V2); |
358 | |
359 | /// Merge all of the live segments of a specific val# in RHS into this live |
360 | /// range as the specified value number. The segments in RHS are allowed |
361 | /// to overlap with segments in the current range, it will replace the |
362 | /// value numbers of the overlaped live segments with the specified value |
363 | /// number. |
364 | void MergeSegmentsInAsValue(const LiveRange &RHS, VNInfo *LHSValNo); |
365 | |
366 | /// MergeValueInAsValue - Merge all of the segments of a specific val# |
367 | /// in RHS into this live range as the specified value number. |
368 | /// The segments in RHS are allowed to overlap with segments in the |
369 | /// current range, but only if the overlapping segments have the |
370 | /// specified value number. |
371 | void MergeValueInAsValue(const LiveRange &RHS, |
372 | const VNInfo *RHSValNo, VNInfo *LHSValNo); |
373 | |
374 | bool empty() const { return segments.empty(); } |
375 | |
376 | /// beginIndex - Return the lowest numbered slot covered. |
377 | SlotIndex beginIndex() const { |
378 | assert(!empty() && "Call to beginIndex() on empty range.")((!empty() && "Call to beginIndex() on empty range.") ? static_cast<void> (0) : __assert_fail ("!empty() && \"Call to beginIndex() on empty range.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 378, __PRETTY_FUNCTION__)); |
379 | return segments.front().start; |
380 | } |
381 | |
382 | /// endNumber - return the maximum point of the range of the whole, |
383 | /// exclusive. |
384 | SlotIndex endIndex() const { |
385 | assert(!empty() && "Call to endIndex() on empty range.")((!empty() && "Call to endIndex() on empty range.") ? static_cast<void> (0) : __assert_fail ("!empty() && \"Call to endIndex() on empty range.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 385, __PRETTY_FUNCTION__)); |
386 | return segments.back().end; |
387 | } |
388 | |
389 | bool expiredAt(SlotIndex index) const { |
390 | return index >= endIndex(); |
391 | } |
392 | |
393 | bool liveAt(SlotIndex index) const { |
394 | const_iterator r = find(index); |
395 | return r != end() && r->start <= index; |
396 | } |
397 | |
398 | /// Return the segment that contains the specified index, or null if there |
399 | /// is none. |
400 | const Segment *getSegmentContaining(SlotIndex Idx) const { |
401 | const_iterator I = FindSegmentContaining(Idx); |
402 | return I == end() ? nullptr : &*I; |
403 | } |
404 | |
405 | /// Return the live segment that contains the specified index, or null if |
406 | /// there is none. |
407 | Segment *getSegmentContaining(SlotIndex Idx) { |
408 | iterator I = FindSegmentContaining(Idx); |
409 | return I == end() ? nullptr : &*I; |
410 | } |
411 | |
412 | /// getVNInfoAt - Return the VNInfo that is live at Idx, or NULL. |
413 | VNInfo *getVNInfoAt(SlotIndex Idx) const { |
414 | const_iterator I = FindSegmentContaining(Idx); |
415 | return I == end() ? nullptr : I->valno; |
416 | } |
417 | |
418 | /// getVNInfoBefore - Return the VNInfo that is live up to but not |
419 | /// necessarilly including Idx, or NULL. Use this to find the reaching def |
420 | /// used by an instruction at this SlotIndex position. |
421 | VNInfo *getVNInfoBefore(SlotIndex Idx) const { |
422 | const_iterator I = FindSegmentContaining(Idx.getPrevSlot()); |
423 | return I == end() ? nullptr : I->valno; |
424 | } |
425 | |
426 | /// Return an iterator to the segment that contains the specified index, or |
427 | /// end() if there is none. |
428 | iterator FindSegmentContaining(SlotIndex Idx) { |
429 | iterator I = find(Idx); |
430 | return I != end() && I->start <= Idx ? I : end(); |
431 | } |
432 | |
433 | const_iterator FindSegmentContaining(SlotIndex Idx) const { |
434 | const_iterator I = find(Idx); |
435 | return I != end() && I->start <= Idx ? I : end(); |
436 | } |
437 | |
438 | /// overlaps - Return true if the intersection of the two live ranges is |
439 | /// not empty. |
440 | bool overlaps(const LiveRange &other) const { |
441 | if (other.empty()) |
442 | return false; |
443 | return overlapsFrom(other, other.begin()); |
444 | } |
445 | |
446 | /// overlaps - Return true if the two ranges have overlapping segments |
447 | /// that are not coalescable according to CP. |
448 | /// |
449 | /// Overlapping segments where one range is defined by a coalescable |
450 | /// copy are allowed. |
451 | bool overlaps(const LiveRange &Other, const CoalescerPair &CP, |
452 | const SlotIndexes&) const; |
453 | |
454 | /// overlaps - Return true if the live range overlaps an interval specified |
455 | /// by [Start, End). |
456 | bool overlaps(SlotIndex Start, SlotIndex End) const; |
457 | |
458 | /// overlapsFrom - Return true if the intersection of the two live ranges |
459 | /// is not empty. The specified iterator is a hint that we can begin |
460 | /// scanning the Other range starting at I. |
461 | bool overlapsFrom(const LiveRange &Other, const_iterator StartPos) const; |
462 | |
463 | /// Returns true if all segments of the @p Other live range are completely |
464 | /// covered by this live range. |
465 | /// Adjacent live ranges do not affect the covering:the liverange |
466 | /// [1,5](5,10] covers (3,7]. |
467 | bool covers(const LiveRange &Other) const; |
468 | |
469 | /// Add the specified Segment to this range, merging segments as |
470 | /// appropriate. This returns an iterator to the inserted segment (which |
471 | /// may have grown since it was inserted). |
472 | iterator addSegment(Segment S); |
473 | |
474 | /// Attempt to extend a value defined after @p StartIdx to include @p Use. |
475 | /// Both @p StartIdx and @p Use should be in the same basic block. In case |
476 | /// of subranges, an extension could be prevented by an explicit "undef" |
477 | /// caused by a <def,read-undef> on a non-overlapping lane. The list of |
478 | /// location of such "undefs" should be provided in @p Undefs. |
479 | /// The return value is a pair: the first element is VNInfo of the value |
480 | /// that was extended (possibly nullptr), the second is a boolean value |
481 | /// indicating whether an "undef" was encountered. |
482 | /// If this range is live before @p Use in the basic block that starts at |
483 | /// @p StartIdx, and there is no intervening "undef", extend it to be live |
484 | /// up to @p Use, and return the pair {value, false}. If there is no |
485 | /// segment before @p Use and there is no "undef" between @p StartIdx and |
486 | /// @p Use, return {nullptr, false}. If there is an "undef" before @p Use, |
487 | /// return {nullptr, true}. |
488 | std::pair<VNInfo*,bool> extendInBlock(ArrayRef<SlotIndex> Undefs, |
489 | SlotIndex StartIdx, SlotIndex Kill); |
490 | |
491 | /// Simplified version of the above "extendInBlock", which assumes that |
492 | /// no register lanes are undefined by <def,read-undef> operands. |
493 | /// If this range is live before @p Use in the basic block that starts |
494 | /// at @p StartIdx, extend it to be live up to @p Use, and return the |
495 | /// value. If there is no segment before @p Use, return nullptr. |
496 | VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Kill); |
497 | |
498 | /// join - Join two live ranges (this, and other) together. This applies |
499 | /// mappings to the value numbers in the LHS/RHS ranges as specified. If |
500 | /// the ranges are not joinable, this aborts. |
501 | void join(LiveRange &Other, |
502 | const int *ValNoAssignments, |
503 | const int *RHSValNoAssignments, |
504 | SmallVectorImpl<VNInfo *> &NewVNInfo); |
505 | |
506 | /// True iff this segment is a single segment that lies between the |
507 | /// specified boundaries, exclusively. Vregs live across a backedge are not |
508 | /// considered local. The boundaries are expected to lie within an extended |
509 | /// basic block, so vregs that are not live out should contain no holes. |
510 | bool isLocal(SlotIndex Start, SlotIndex End) const { |
511 | return beginIndex() > Start.getBaseIndex() && |
512 | endIndex() < End.getBoundaryIndex(); |
513 | } |
514 | |
515 | /// Remove the specified segment from this range. Note that the segment |
516 | /// must be a single Segment in its entirety. |
517 | void removeSegment(SlotIndex Start, SlotIndex End, |
518 | bool RemoveDeadValNo = false); |
519 | |
520 | void removeSegment(Segment S, bool RemoveDeadValNo = false) { |
521 | removeSegment(S.start, S.end, RemoveDeadValNo); |
522 | } |
523 | |
524 | /// Remove segment pointed to by iterator @p I from this range. This does |
525 | /// not remove dead value numbers. |
526 | iterator removeSegment(iterator I) { |
527 | return segments.erase(I); |
528 | } |
529 | |
530 | /// Query Liveness at Idx. |
531 | /// The sub-instruction slot of Idx doesn't matter, only the instruction |
532 | /// it refers to is considered. |
533 | LiveQueryResult Query(SlotIndex Idx) const { |
534 | // Find the segment that enters the instruction. |
535 | const_iterator I = find(Idx.getBaseIndex()); |
536 | const_iterator E = end(); |
537 | if (I == E) |
538 | return LiveQueryResult(nullptr, nullptr, SlotIndex(), false); |
539 | |
540 | // Is this an instruction live-in segment? |
541 | // If Idx is the start index of a basic block, include live-in segments |
542 | // that start at Idx.getBaseIndex(). |
543 | VNInfo *EarlyVal = nullptr; |
544 | VNInfo *LateVal = nullptr; |
545 | SlotIndex EndPoint; |
546 | bool Kill = false; |
547 | if (I->start <= Idx.getBaseIndex()) { |
548 | EarlyVal = I->valno; |
549 | EndPoint = I->end; |
550 | // Move to the potentially live-out segment. |
551 | if (SlotIndex::isSameInstr(Idx, I->end)) { |
552 | Kill = true; |
553 | if (++I == E) |
554 | return LiveQueryResult(EarlyVal, LateVal, EndPoint, Kill); |
555 | } |
556 | // Special case: A PHIDef value can have its def in the middle of a |
557 | // segment if the value happens to be live out of the layout |
558 | // predecessor. |
559 | // Such a value is not live-in. |
560 | if (EarlyVal->def == Idx.getBaseIndex()) |
561 | EarlyVal = nullptr; |
562 | } |
563 | // I now points to the segment that may be live-through, or defined by |
564 | // this instr. Ignore segments starting after the current instr. |
565 | if (!SlotIndex::isEarlierInstr(Idx, I->start)) { |
566 | LateVal = I->valno; |
567 | EndPoint = I->end; |
568 | } |
569 | return LiveQueryResult(EarlyVal, LateVal, EndPoint, Kill); |
570 | } |
571 | |
572 | /// removeValNo - Remove all the segments defined by the specified value#. |
573 | /// Also remove the value# from value# list. |
574 | void removeValNo(VNInfo *ValNo); |
575 | |
576 | /// Returns true if the live range is zero length, i.e. no live segments |
577 | /// span instructions. It doesn't pay to spill such a range. |
578 | bool isZeroLength(SlotIndexes *Indexes) const { |
579 | for (const Segment &S : segments) |
580 | if (Indexes->getNextNonNullIndex(S.start).getBaseIndex() < |
581 | S.end.getBaseIndex()) |
582 | return false; |
583 | return true; |
584 | } |
585 | |
586 | // Returns true if any segment in the live range contains any of the |
587 | // provided slot indexes. Slots which occur in holes between |
588 | // segments will not cause the function to return true. |
589 | bool isLiveAtIndexes(ArrayRef<SlotIndex> Slots) const; |
590 | |
591 | bool operator<(const LiveRange& other) const { |
592 | const SlotIndex &thisIndex = beginIndex(); |
593 | const SlotIndex &otherIndex = other.beginIndex(); |
594 | return thisIndex < otherIndex; |
595 | } |
596 | |
597 | /// Returns true if there is an explicit "undef" between @p Begin |
598 | /// @p End. |
599 | bool isUndefIn(ArrayRef<SlotIndex> Undefs, SlotIndex Begin, |
600 | SlotIndex End) const { |
601 | return llvm::any_of(Undefs, [Begin, End](SlotIndex Idx) -> bool { |
602 | return Begin <= Idx && Idx < End; |
603 | }); |
604 | } |
605 | |
606 | /// Flush segment set into the regular segment vector. |
607 | /// The method is to be called after the live range |
608 | /// has been created, if use of the segment set was |
609 | /// activated in the constructor of the live range. |
610 | void flushSegmentSet(); |
611 | |
612 | /// Stores indexes from the input index sequence R at which this LiveRange |
613 | /// is live to the output O iterator. |
614 | /// R is a range of _ascending sorted_ _random_ access iterators |
615 | /// to the input indexes. Indexes stored at O are ascending sorted so it |
616 | /// can be used directly in the subsequent search (for example for |
617 | /// subranges). Returns true if found at least one index. |
618 | template <typename Range, typename OutputIt> |
619 | bool findIndexesLiveAt(Range &&R, OutputIt O) const { |
620 | assert(llvm::is_sorted(R))((llvm::is_sorted(R)) ? static_cast<void> (0) : __assert_fail ("llvm::is_sorted(R)", "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/LiveInterval.h" , 620, __PRETTY_FUNCTION__)); |
621 | auto Idx = R.begin(), EndIdx = R.end(); |
622 | auto Seg = segments.begin(), EndSeg = segments.end(); |
623 | bool Found = false; |
624 | while (Idx != EndIdx && Seg != EndSeg) { |
625 | // if the Seg is lower find first segment that is above Idx using binary |
626 | // search |
627 | if (Seg->end <= *Idx) { |
628 | Seg = std::upper_bound( |
629 | ++Seg, EndSeg, *Idx, |
630 | [=](std::remove_reference_t<decltype(*Idx)> V, |
631 | const std::remove_reference_t<decltype(*Seg)> &S) { |
632 | return V < S.end; |
633 | }); |
634 | if (Seg == EndSeg) |
635 | break; |
636 | } |
637 | auto NotLessStart = std::lower_bound(Idx, EndIdx, Seg->start); |
638 | if (NotLessStart == EndIdx) |
639 | break; |
640 | auto NotLessEnd = std::lower_bound(NotLessStart, EndIdx, Seg->end); |
641 | if (NotLessEnd != NotLessStart) { |
642 | Found = true; |
643 | O = std::copy(NotLessStart, NotLessEnd, O); |
644 | } |
645 | Idx = NotLessEnd; |
646 | ++Seg; |
647 | } |
648 | return Found; |
649 | } |
650 | |
651 | void print(raw_ostream &OS) const; |
652 | void dump() const; |
653 | |
654 | /// Walk the range and assert if any invariants fail to hold. |
655 | /// |
656 | /// Note that this is a no-op when asserts are disabled. |
657 | #ifdef NDEBUG |
658 | void verify() const {} |
659 | #else |
660 | void verify() const; |
661 | #endif |
662 | |
663 | protected: |
664 | /// Append a segment to the list of segments. |
665 | void append(const LiveRange::Segment S); |
666 | |
667 | private: |
668 | friend class LiveRangeUpdater; |
669 | void addSegmentToSet(Segment S); |
670 | void markValNoForDeletion(VNInfo *V); |
671 | }; |
672 | |
673 | inline raw_ostream &operator<<(raw_ostream &OS, const LiveRange &LR) { |
674 | LR.print(OS); |
675 | return OS; |
676 | } |
677 | |
678 | /// LiveInterval - This class represents the liveness of a register, |
679 | /// or stack slot. |
680 | class LiveInterval : public LiveRange { |
681 | public: |
682 | using super = LiveRange; |
683 | |
684 | /// A live range for subregisters. The LaneMask specifies which parts of the |
685 | /// super register are covered by the interval. |
686 | /// (@sa TargetRegisterInfo::getSubRegIndexLaneMask()). |
687 | class SubRange : public LiveRange { |
688 | public: |
689 | SubRange *Next = nullptr; |
690 | LaneBitmask LaneMask; |
691 | |
692 | /// Constructs a new SubRange object. |
693 | SubRange(LaneBitmask LaneMask) : LaneMask(LaneMask) {} |
694 | |
695 | /// Constructs a new SubRange object by copying liveness from @p Other. |
696 | SubRange(LaneBitmask LaneMask, const LiveRange &Other, |
697 | BumpPtrAllocator &Allocator) |
698 | : LiveRange(Other, Allocator), LaneMask(LaneMask) {} |
699 | |
700 | void print(raw_ostream &OS) const; |
701 | void dump() const; |
702 | }; |
703 | |
704 | private: |
705 | SubRange *SubRanges = nullptr; ///< Single linked list of subregister live |
706 | /// ranges. |
707 | const Register Reg; // the register or stack slot of this interval. |
708 | float Weight = 0.0; // weight of this interval |
709 | |
710 | public: |
711 | Register reg() const { return Reg; } |
712 | float weight() const { return Weight; } |
713 | void incrementWeight(float Inc) { Weight += Inc; } |
714 | void setWeight(float Value) { Weight = Value; } |
715 | |
716 | LiveInterval(unsigned Reg, float Weight) : Reg(Reg), Weight(Weight) {} |
717 | |
718 | ~LiveInterval() { |
719 | clearSubRanges(); |
720 | } |
721 | |
722 | template<typename T> |
723 | class SingleLinkedListIterator { |
724 | T *P; |
725 | |
726 | public: |
727 | SingleLinkedListIterator<T>(T *P) : P(P) {} |
728 | |
729 | SingleLinkedListIterator<T> &operator++() { |
730 | P = P->Next; |
731 | return *this; |
732 | } |
733 | SingleLinkedListIterator<T> operator++(int) { |
734 | SingleLinkedListIterator res = *this; |
735 | ++*this; |
736 | return res; |
737 | } |
738 | bool operator!=(const SingleLinkedListIterator<T> &Other) const { |
739 | return P != Other.operator->(); |
740 | } |
741 | bool operator==(const SingleLinkedListIterator<T> &Other) const { |
742 | return P == Other.operator->(); |
743 | } |
744 | T &operator*() const { |
745 | return *P; |
746 | } |
747 | T *operator->() const { |
748 | return P; |
749 | } |
750 | }; |
751 | |
752 | using subrange_iterator = SingleLinkedListIterator<SubRange>; |
753 | using const_subrange_iterator = SingleLinkedListIterator<const SubRange>; |
754 | |
755 | subrange_iterator subrange_begin() { |
756 | return subrange_iterator(SubRanges); |
757 | } |
758 | subrange_iterator subrange_end() { |
759 | return subrange_iterator(nullptr); |
760 | } |
761 | |
762 | const_subrange_iterator subrange_begin() const { |
763 | return const_subrange_iterator(SubRanges); |
764 | } |
765 | const_subrange_iterator subrange_end() const { |
766 | return const_subrange_iterator(nullptr); |
767 | } |
768 | |
769 | iterator_range<subrange_iterator> subranges() { |
770 | return make_range(subrange_begin(), subrange_end()); |
771 | } |
772 | |
773 | iterator_range<const_subrange_iterator> subranges() const { |
774 | return make_range(subrange_begin(), subrange_end()); |
775 | } |
776 | |
777 | /// Creates a new empty subregister live range. The range is added at the |
778 | /// beginning of the subrange list; subrange iterators stay valid. |
779 | SubRange *createSubRange(BumpPtrAllocator &Allocator, |
780 | LaneBitmask LaneMask) { |
781 | SubRange *Range = new (Allocator) SubRange(LaneMask); |
782 | appendSubRange(Range); |
783 | return Range; |
784 | } |
785 | |
786 | /// Like createSubRange() but the new range is filled with a copy of the |
787 | /// liveness information in @p CopyFrom. |
788 | SubRange *createSubRangeFrom(BumpPtrAllocator &Allocator, |
789 | LaneBitmask LaneMask, |
790 | const LiveRange &CopyFrom) { |
791 | SubRange *Range = new (Allocator) SubRange(LaneMask, CopyFrom, Allocator); |
792 | appendSubRange(Range); |
793 | return Range; |
794 | } |
795 | |
796 | /// Returns true if subregister liveness information is available. |
797 | bool hasSubRanges() const { |
798 | return SubRanges != nullptr; |
799 | } |
800 | |
801 | /// Removes all subregister liveness information. |
802 | void clearSubRanges(); |
803 | |
804 | /// Removes all subranges without any segments (subranges without segments |
805 | /// are not considered valid and should only exist temporarily). |
806 | void removeEmptySubRanges(); |
807 | |
808 | /// getSize - Returns the sum of sizes of all the LiveRange's. |
809 | /// |
810 | unsigned getSize() const; |
811 | |
812 | /// isSpillable - Can this interval be spilled? |
813 | bool isSpillable() const { return Weight != huge_valf; } |
814 | |
815 | /// markNotSpillable - Mark interval as not spillable |
816 | void markNotSpillable() { Weight = huge_valf; } |
817 | |
818 | /// For a given lane mask @p LaneMask, compute indexes at which the |
819 | /// lane is marked undefined by subregister <def,read-undef> definitions. |
820 | void computeSubRangeUndefs(SmallVectorImpl<SlotIndex> &Undefs, |
821 | LaneBitmask LaneMask, |
822 | const MachineRegisterInfo &MRI, |
823 | const SlotIndexes &Indexes) const; |
824 | |
825 | /// Refines the subranges to support \p LaneMask. This may only be called |
826 | /// for LI.hasSubrange()==true. Subregister ranges are split or created |
827 | /// until \p LaneMask can be matched exactly. \p Mod is executed on the |
828 | /// matching subranges. |
829 | /// |
830 | /// Example: |
831 | /// Given an interval with subranges with lanemasks L0F00, L00F0 and |
832 | /// L000F, refining for mask L0018. Will split the L00F0 lane into |
833 | /// L00E0 and L0010 and the L000F lane into L0007 and L0008. The Mod |
834 | /// function will be applied to the L0010 and L0008 subranges. |
835 | /// |
836 | /// \p Indexes and \p TRI are required to clean up the VNIs that |
837 | /// don't define the related lane masks after they get shrunk. E.g., |
838 | /// when L000F gets split into L0007 and L0008 maybe only a subset |
839 | /// of the VNIs that defined L000F defines L0007. |
840 | /// |
841 | /// The clean up of the VNIs need to look at the actual instructions |
842 | /// to decide what is or is not live at a definition point. If the |
843 | /// update of the subranges occurs while the IR does not reflect these |
844 | /// changes, \p ComposeSubRegIdx can be used to specify how the |
845 | /// definition are going to be rewritten. |
846 | /// E.g., let say we want to merge: |
847 | /// V1.sub1:<2 x s32> = COPY V2.sub3:<4 x s32> |
848 | /// We do that by choosing a class where sub1:<2 x s32> and sub3:<4 x s32> |
849 | /// overlap, i.e., by choosing a class where we can find "offset + 1 == 3". |
850 | /// Put differently we align V2's sub3 with V1's sub1: |
851 | /// V2: sub0 sub1 sub2 sub3 |
852 | /// V1: <offset> sub0 sub1 |
853 | /// |
854 | /// This offset will look like a composed subregidx in the the class: |
855 | /// V1.(composed sub2 with sub1):<4 x s32> = COPY V2.sub3:<4 x s32> |
856 | /// => V1.(composed sub2 with sub1):<4 x s32> = COPY V2.sub3:<4 x s32> |
857 | /// |
858 | /// Now if we didn't rewrite the uses and def of V1, all the checks for V1 |
859 | /// need to account for this offset. |
860 | /// This happens during coalescing where we update the live-ranges while |
861 | /// still having the old IR around because updating the IR on-the-fly |
862 | /// would actually clobber some information on how the live-ranges that |
863 | /// are being updated look like. |
864 | void refineSubRanges(BumpPtrAllocator &Allocator, LaneBitmask LaneMask, |
865 | std::function<void(LiveInterval::SubRange &)> Apply, |
866 | const SlotIndexes &Indexes, |
867 | const TargetRegisterInfo &TRI, |
868 | unsigned ComposeSubRegIdx = 0); |
869 | |
870 | bool operator<(const LiveInterval& other) const { |
871 | const SlotIndex &thisIndex = beginIndex(); |
872 | const SlotIndex &otherIndex = other.beginIndex(); |
873 | return std::tie(thisIndex, Reg) < std::tie(otherIndex, other.Reg); |
874 | } |
875 | |
876 | void print(raw_ostream &OS) const; |
877 | void dump() const; |
878 | |
879 | /// Walks the interval and assert if any invariants fail to hold. |
880 | /// |
881 | /// Note that this is a no-op when asserts are disabled. |
882 | #ifdef NDEBUG |
883 | void verify(const MachineRegisterInfo *MRI = nullptr) const {} |
884 | #else |
885 | void verify(const MachineRegisterInfo *MRI = nullptr) const; |
886 | #endif |
887 | |
888 | private: |
889 | /// Appends @p Range to SubRanges list. |
890 | void appendSubRange(SubRange *Range) { |
891 | Range->Next = SubRanges; |
892 | SubRanges = Range; |
893 | } |
894 | |
895 | /// Free memory held by SubRange. |
896 | void freeSubRange(SubRange *S); |
897 | }; |
898 | |
899 | inline raw_ostream &operator<<(raw_ostream &OS, |
900 | const LiveInterval::SubRange &SR) { |
901 | SR.print(OS); |
902 | return OS; |
903 | } |
904 | |
905 | inline raw_ostream &operator<<(raw_ostream &OS, const LiveInterval &LI) { |
906 | LI.print(OS); |
907 | return OS; |
908 | } |
909 | |
910 | raw_ostream &operator<<(raw_ostream &OS, const LiveRange::Segment &S); |
911 | |
912 | inline bool operator<(SlotIndex V, const LiveRange::Segment &S) { |
913 | return V < S.start; |
914 | } |
915 | |
916 | inline bool operator<(const LiveRange::Segment &S, SlotIndex V) { |
917 | return S.start < V; |
918 | } |
919 | |
920 | /// Helper class for performant LiveRange bulk updates. |
921 | /// |
922 | /// Calling LiveRange::addSegment() repeatedly can be expensive on large |
923 | /// live ranges because segments after the insertion point may need to be |
924 | /// shifted. The LiveRangeUpdater class can defer the shifting when adding |
925 | /// many segments in order. |
926 | /// |
927 | /// The LiveRange will be in an invalid state until flush() is called. |
928 | class LiveRangeUpdater { |
929 | LiveRange *LR; |
930 | SlotIndex LastStart; |
931 | LiveRange::iterator WriteI; |
932 | LiveRange::iterator ReadI; |
933 | SmallVector<LiveRange::Segment, 16> Spills; |
934 | void mergeSpills(); |
935 | |
936 | public: |
937 | /// Create a LiveRangeUpdater for adding segments to LR. |
938 | /// LR will temporarily be in an invalid state until flush() is called. |
939 | LiveRangeUpdater(LiveRange *lr = nullptr) : LR(lr) {} |
940 | |
941 | ~LiveRangeUpdater() { flush(); } |
942 | |
943 | /// Add a segment to LR and coalesce when possible, just like |
944 | /// LR.addSegment(). Segments should be added in increasing start order for |
945 | /// best performance. |
946 | void add(LiveRange::Segment); |
947 | |
948 | void add(SlotIndex Start, SlotIndex End, VNInfo *VNI) { |
949 | add(LiveRange::Segment(Start, End, VNI)); |
950 | } |
951 | |
952 | /// Return true if the LR is currently in an invalid state, and flush() |
953 | /// needs to be called. |
954 | bool isDirty() const { return LastStart.isValid(); } |
955 | |
956 | /// Flush the updater state to LR so it is valid and contains all added |
957 | /// segments. |
958 | void flush(); |
959 | |
960 | /// Select a different destination live range. |
961 | void setDest(LiveRange *lr) { |
962 | if (LR != lr && isDirty()) |
963 | flush(); |
964 | LR = lr; |
965 | } |
966 | |
967 | /// Get the current destination live range. |
968 | LiveRange *getDest() const { return LR; } |
969 | |
970 | void dump() const; |
971 | void print(raw_ostream&) const; |
972 | }; |
973 | |
974 | inline raw_ostream &operator<<(raw_ostream &OS, const LiveRangeUpdater &X) { |
975 | X.print(OS); |
976 | return OS; |
977 | } |
978 | |
979 | /// ConnectedVNInfoEqClasses - Helper class that can divide VNInfos in a |
980 | /// LiveInterval into equivalence clases of connected components. A |
981 | /// LiveInterval that has multiple connected components can be broken into |
982 | /// multiple LiveIntervals. |
983 | /// |
984 | /// Given a LiveInterval that may have multiple connected components, run: |
985 | /// |
986 | /// unsigned numComps = ConEQ.Classify(LI); |
987 | /// if (numComps > 1) { |
988 | /// // allocate numComps-1 new LiveIntervals into LIS[1..] |
989 | /// ConEQ.Distribute(LIS); |
990 | /// } |
991 | |
992 | class ConnectedVNInfoEqClasses { |
993 | LiveIntervals &LIS; |
994 | IntEqClasses EqClass; |
995 | |
996 | public: |
997 | explicit ConnectedVNInfoEqClasses(LiveIntervals &lis) : LIS(lis) {} |
998 | |
999 | /// Classify the values in \p LR into connected components. |
1000 | /// Returns the number of connected components. |
1001 | unsigned Classify(const LiveRange &LR); |
1002 | |
1003 | /// getEqClass - Classify creates equivalence classes numbered 0..N. Return |
1004 | /// the equivalence class assigned the VNI. |
1005 | unsigned getEqClass(const VNInfo *VNI) const { return EqClass[VNI->id]; } |
1006 | |
1007 | /// Distribute values in \p LI into a separate LiveIntervals |
1008 | /// for each connected component. LIV must have an empty LiveInterval for |
1009 | /// each additional connected component. The first connected component is |
1010 | /// left in \p LI. |
1011 | void Distribute(LiveInterval &LI, LiveInterval *LIV[], |
1012 | MachineRegisterInfo &MRI); |
1013 | }; |
1014 | |
1015 | } // end namespace llvm |
1016 | |
1017 | #endif // LLVM_CODEGEN_LIVEINTERVAL_H |
1 | //===- llvm/CodeGen/SlotIndexes.h - Slot indexes representation -*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements SlotIndex and related classes. The purpose of SlotIndex |
10 | // is to describe a position at which a register can become live, or cease to |
11 | // be live. |
12 | // |
13 | // SlotIndex is mostly a proxy for entries of the SlotIndexList, a class which |
14 | // is held is LiveIntervals and provides the real numbering. This allows |
15 | // LiveIntervals to perform largely transparent renumbering. |
16 | //===----------------------------------------------------------------------===// |
17 | |
18 | #ifndef LLVM_CODEGEN_SLOTINDEXES_H |
19 | #define LLVM_CODEGEN_SLOTINDEXES_H |
20 | |
21 | #include "llvm/ADT/DenseMap.h" |
22 | #include "llvm/ADT/IntervalMap.h" |
23 | #include "llvm/ADT/PointerIntPair.h" |
24 | #include "llvm/ADT/SmallVector.h" |
25 | #include "llvm/ADT/ilist.h" |
26 | #include "llvm/CodeGen/MachineBasicBlock.h" |
27 | #include "llvm/CodeGen/MachineFunction.h" |
28 | #include "llvm/CodeGen/MachineFunctionPass.h" |
29 | #include "llvm/CodeGen/MachineInstr.h" |
30 | #include "llvm/CodeGen/MachineInstrBundle.h" |
31 | #include "llvm/Pass.h" |
32 | #include "llvm/Support/Allocator.h" |
33 | #include <algorithm> |
34 | #include <cassert> |
35 | #include <iterator> |
36 | #include <utility> |
37 | |
38 | namespace llvm { |
39 | |
40 | class raw_ostream; |
41 | |
42 | /// This class represents an entry in the slot index list held in the |
43 | /// SlotIndexes pass. It should not be used directly. See the |
44 | /// SlotIndex & SlotIndexes classes for the public interface to this |
45 | /// information. |
46 | class IndexListEntry : public ilist_node<IndexListEntry> { |
47 | MachineInstr *mi; |
48 | unsigned index; |
49 | |
50 | public: |
51 | IndexListEntry(MachineInstr *mi, unsigned index) : mi(mi), index(index) {} |
52 | |
53 | MachineInstr* getInstr() const { return mi; } |
54 | void setInstr(MachineInstr *mi) { |
55 | this->mi = mi; |
56 | } |
57 | |
58 | unsigned getIndex() const { return index; } |
59 | void setIndex(unsigned index) { |
60 | this->index = index; |
61 | } |
62 | |
63 | #ifdef EXPENSIVE_CHECKS |
64 | // When EXPENSIVE_CHECKS is defined, "erased" index list entries will |
65 | // actually be moved to a "graveyard" list, and have their pointers |
66 | // poisoned, so that dangling SlotIndex access can be reliably detected. |
67 | void setPoison() { |
68 | intptr_t tmp = reinterpret_cast<intptr_t>(mi); |
69 | assert(((tmp & 0x1) == 0x0) && "Pointer already poisoned?")((((tmp & 0x1) == 0x0) && "Pointer already poisoned?" ) ? static_cast<void> (0) : __assert_fail ("((tmp & 0x1) == 0x0) && \"Pointer already poisoned?\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 69, __PRETTY_FUNCTION__)); |
70 | tmp |= 0x1; |
71 | mi = reinterpret_cast<MachineInstr*>(tmp); |
72 | } |
73 | |
74 | bool isPoisoned() const { return (reinterpret_cast<intptr_t>(mi) & 0x1) == 0x1; } |
75 | #endif // EXPENSIVE_CHECKS |
76 | }; |
77 | |
78 | template <> |
79 | struct ilist_alloc_traits<IndexListEntry> |
80 | : public ilist_noalloc_traits<IndexListEntry> {}; |
81 | |
82 | /// SlotIndex - An opaque wrapper around machine indexes. |
83 | class SlotIndex { |
84 | friend class SlotIndexes; |
85 | |
86 | enum Slot { |
87 | /// Basic block boundary. Used for live ranges entering and leaving a |
88 | /// block without being live in the layout neighbor. Also used as the |
89 | /// def slot of PHI-defs. |
90 | Slot_Block, |
91 | |
92 | /// Early-clobber register use/def slot. A live range defined at |
93 | /// Slot_EarlyClobber interferes with normal live ranges killed at |
94 | /// Slot_Register. Also used as the kill slot for live ranges tied to an |
95 | /// early-clobber def. |
96 | Slot_EarlyClobber, |
97 | |
98 | /// Normal register use/def slot. Normal instructions kill and define |
99 | /// register live ranges at this slot. |
100 | Slot_Register, |
101 | |
102 | /// Dead def kill point. Kill slot for a live range that is defined by |
103 | /// the same instruction (Slot_Register or Slot_EarlyClobber), but isn't |
104 | /// used anywhere. |
105 | Slot_Dead, |
106 | |
107 | Slot_Count |
108 | }; |
109 | |
110 | PointerIntPair<IndexListEntry*, 2, unsigned> lie; |
111 | |
112 | SlotIndex(IndexListEntry *entry, unsigned slot) |
113 | : lie(entry, slot) {} |
114 | |
115 | IndexListEntry* listEntry() const { |
116 | assert(isValid() && "Attempt to compare reserved index.")((isValid() && "Attempt to compare reserved index.") ? static_cast<void> (0) : __assert_fail ("isValid() && \"Attempt to compare reserved index.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 116, __PRETTY_FUNCTION__)); |
117 | #ifdef EXPENSIVE_CHECKS |
118 | assert(!lie.getPointer()->isPoisoned() &&((!lie.getPointer()->isPoisoned() && "Attempt to access deleted list-entry." ) ? static_cast<void> (0) : __assert_fail ("!lie.getPointer()->isPoisoned() && \"Attempt to access deleted list-entry.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 119, __PRETTY_FUNCTION__)) |
119 | "Attempt to access deleted list-entry.")((!lie.getPointer()->isPoisoned() && "Attempt to access deleted list-entry." ) ? static_cast<void> (0) : __assert_fail ("!lie.getPointer()->isPoisoned() && \"Attempt to access deleted list-entry.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 119, __PRETTY_FUNCTION__)); |
120 | #endif // EXPENSIVE_CHECKS |
121 | return lie.getPointer(); |
122 | } |
123 | |
124 | unsigned getIndex() const { |
125 | return listEntry()->getIndex() | getSlot(); |
126 | } |
127 | |
128 | /// Returns the slot for this SlotIndex. |
129 | Slot getSlot() const { |
130 | return static_cast<Slot>(lie.getInt()); |
131 | } |
132 | |
133 | public: |
134 | enum { |
135 | /// The default distance between instructions as returned by distance(). |
136 | /// This may vary as instructions are inserted and removed. |
137 | InstrDist = 4 * Slot_Count |
138 | }; |
139 | |
140 | /// Construct an invalid index. |
141 | SlotIndex() = default; |
142 | |
143 | // Construct a new slot index from the given one, and set the slot. |
144 | SlotIndex(const SlotIndex &li, Slot s) : lie(li.listEntry(), unsigned(s)) { |
145 | assert(lie.getPointer() != nullptr &&((lie.getPointer() != nullptr && "Attempt to construct index with 0 pointer." ) ? static_cast<void> (0) : __assert_fail ("lie.getPointer() != nullptr && \"Attempt to construct index with 0 pointer.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 146, __PRETTY_FUNCTION__)) |
146 | "Attempt to construct index with 0 pointer.")((lie.getPointer() != nullptr && "Attempt to construct index with 0 pointer." ) ? static_cast<void> (0) : __assert_fail ("lie.getPointer() != nullptr && \"Attempt to construct index with 0 pointer.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 146, __PRETTY_FUNCTION__)); |
147 | } |
148 | |
149 | /// Returns true if this is a valid index. Invalid indices do |
150 | /// not point into an index table, and cannot be compared. |
151 | bool isValid() const { |
152 | return lie.getPointer(); |
153 | } |
154 | |
155 | /// Return true for a valid index. |
156 | explicit operator bool() const { return isValid(); } |
157 | |
158 | /// Print this index to the given raw_ostream. |
159 | void print(raw_ostream &os) const; |
160 | |
161 | /// Dump this index to stderr. |
162 | void dump() const; |
163 | |
164 | /// Compare two SlotIndex objects for equality. |
165 | bool operator==(SlotIndex other) const { |
166 | return lie == other.lie; |
167 | } |
168 | /// Compare two SlotIndex objects for inequality. |
169 | bool operator!=(SlotIndex other) const { |
170 | return lie != other.lie; |
171 | } |
172 | |
173 | /// Compare two SlotIndex objects. Return true if the first index |
174 | /// is strictly lower than the second. |
175 | bool operator<(SlotIndex other) const { |
176 | return getIndex() < other.getIndex(); |
177 | } |
178 | /// Compare two SlotIndex objects. Return true if the first index |
179 | /// is lower than, or equal to, the second. |
180 | bool operator<=(SlotIndex other) const { |
181 | return getIndex() <= other.getIndex(); |
182 | } |
183 | |
184 | /// Compare two SlotIndex objects. Return true if the first index |
185 | /// is greater than the second. |
186 | bool operator>(SlotIndex other) const { |
187 | return getIndex() > other.getIndex(); |
188 | } |
189 | |
190 | /// Compare two SlotIndex objects. Return true if the first index |
191 | /// is greater than, or equal to, the second. |
192 | bool operator>=(SlotIndex other) const { |
193 | return getIndex() >= other.getIndex(); |
194 | } |
195 | |
196 | /// isSameInstr - Return true if A and B refer to the same instruction. |
197 | static bool isSameInstr(SlotIndex A, SlotIndex B) { |
198 | return A.lie.getPointer() == B.lie.getPointer(); |
199 | } |
200 | |
201 | /// isEarlierInstr - Return true if A refers to an instruction earlier than |
202 | /// B. This is equivalent to A < B && !isSameInstr(A, B). |
203 | static bool isEarlierInstr(SlotIndex A, SlotIndex B) { |
204 | return A.listEntry()->getIndex() < B.listEntry()->getIndex(); |
205 | } |
206 | |
207 | /// Return true if A refers to the same instruction as B or an earlier one. |
208 | /// This is equivalent to !isEarlierInstr(B, A). |
209 | static bool isEarlierEqualInstr(SlotIndex A, SlotIndex B) { |
210 | return !isEarlierInstr(B, A); |
211 | } |
212 | |
213 | /// Return the distance from this index to the given one. |
214 | int distance(SlotIndex other) const { |
215 | return other.getIndex() - getIndex(); |
216 | } |
217 | |
218 | /// Return the scaled distance from this index to the given one, where all |
219 | /// slots on the same instruction have zero distance. |
220 | int getInstrDistance(SlotIndex other) const { |
221 | return (other.listEntry()->getIndex() - listEntry()->getIndex()) |
222 | / Slot_Count; |
223 | } |
224 | |
225 | /// isBlock - Returns true if this is a block boundary slot. |
226 | bool isBlock() const { return getSlot() == Slot_Block; } |
227 | |
228 | /// isEarlyClobber - Returns true if this is an early-clobber slot. |
229 | bool isEarlyClobber() const { return getSlot() == Slot_EarlyClobber; } |
230 | |
231 | /// isRegister - Returns true if this is a normal register use/def slot. |
232 | /// Note that early-clobber slots may also be used for uses and defs. |
233 | bool isRegister() const { return getSlot() == Slot_Register; } |
234 | |
235 | /// isDead - Returns true if this is a dead def kill slot. |
236 | bool isDead() const { return getSlot() == Slot_Dead; } |
237 | |
238 | /// Returns the base index for associated with this index. The base index |
239 | /// is the one associated with the Slot_Block slot for the instruction |
240 | /// pointed to by this index. |
241 | SlotIndex getBaseIndex() const { |
242 | return SlotIndex(listEntry(), Slot_Block); |
243 | } |
244 | |
245 | /// Returns the boundary index for associated with this index. The boundary |
246 | /// index is the one associated with the Slot_Block slot for the instruction |
247 | /// pointed to by this index. |
248 | SlotIndex getBoundaryIndex() const { |
249 | return SlotIndex(listEntry(), Slot_Dead); |
250 | } |
251 | |
252 | /// Returns the register use/def slot in the current instruction for a |
253 | /// normal or early-clobber def. |
254 | SlotIndex getRegSlot(bool EC = false) const { |
255 | return SlotIndex(listEntry(), EC ? Slot_EarlyClobber : Slot_Register); |
256 | } |
257 | |
258 | /// Returns the dead def kill slot for the current instruction. |
259 | SlotIndex getDeadSlot() const { |
260 | return SlotIndex(listEntry(), Slot_Dead); |
261 | } |
262 | |
263 | /// Returns the next slot in the index list. This could be either the |
264 | /// next slot for the instruction pointed to by this index or, if this |
265 | /// index is a STORE, the first slot for the next instruction. |
266 | /// WARNING: This method is considerably more expensive than the methods |
267 | /// that return specific slots (getUseIndex(), etc). If you can - please |
268 | /// use one of those methods. |
269 | SlotIndex getNextSlot() const { |
270 | Slot s = getSlot(); |
271 | if (s == Slot_Dead) { |
272 | return SlotIndex(&*++listEntry()->getIterator(), Slot_Block); |
273 | } |
274 | return SlotIndex(listEntry(), s + 1); |
275 | } |
276 | |
277 | /// Returns the next index. This is the index corresponding to the this |
278 | /// index's slot, but for the next instruction. |
279 | SlotIndex getNextIndex() const { |
280 | return SlotIndex(&*++listEntry()->getIterator(), getSlot()); |
281 | } |
282 | |
283 | /// Returns the previous slot in the index list. This could be either the |
284 | /// previous slot for the instruction pointed to by this index or, if this |
285 | /// index is a Slot_Block, the last slot for the previous instruction. |
286 | /// WARNING: This method is considerably more expensive than the methods |
287 | /// that return specific slots (getUseIndex(), etc). If you can - please |
288 | /// use one of those methods. |
289 | SlotIndex getPrevSlot() const { |
290 | Slot s = getSlot(); |
291 | if (s == Slot_Block) { |
292 | return SlotIndex(&*--listEntry()->getIterator(), Slot_Dead); |
293 | } |
294 | return SlotIndex(listEntry(), s - 1); |
295 | } |
296 | |
297 | /// Returns the previous index. This is the index corresponding to this |
298 | /// index's slot, but for the previous instruction. |
299 | SlotIndex getPrevIndex() const { |
300 | return SlotIndex(&*--listEntry()->getIterator(), getSlot()); |
301 | } |
302 | }; |
303 | |
304 | inline raw_ostream& operator<<(raw_ostream &os, SlotIndex li) { |
305 | li.print(os); |
306 | return os; |
307 | } |
308 | |
309 | using IdxMBBPair = std::pair<SlotIndex, MachineBasicBlock *>; |
310 | |
311 | /// SlotIndexes pass. |
312 | /// |
313 | /// This pass assigns indexes to each instruction. |
314 | class SlotIndexes : public MachineFunctionPass { |
315 | private: |
316 | // IndexListEntry allocator. |
317 | BumpPtrAllocator ileAllocator; |
318 | |
319 | using IndexList = ilist<IndexListEntry>; |
320 | IndexList indexList; |
321 | |
322 | MachineFunction *mf; |
323 | |
324 | using Mi2IndexMap = DenseMap<const MachineInstr *, SlotIndex>; |
325 | Mi2IndexMap mi2iMap; |
326 | |
327 | /// MBBRanges - Map MBB number to (start, stop) indexes. |
328 | SmallVector<std::pair<SlotIndex, SlotIndex>, 8> MBBRanges; |
329 | |
330 | /// Idx2MBBMap - Sorted list of pairs of index of first instruction |
331 | /// and MBB id. |
332 | SmallVector<IdxMBBPair, 8> idx2MBBMap; |
333 | |
334 | IndexListEntry* createEntry(MachineInstr *mi, unsigned index) { |
335 | IndexListEntry *entry = |
336 | static_cast<IndexListEntry *>(ileAllocator.Allocate( |
337 | sizeof(IndexListEntry), alignof(IndexListEntry))); |
338 | |
339 | new (entry) IndexListEntry(mi, index); |
340 | |
341 | return entry; |
342 | } |
343 | |
344 | /// Renumber locally after inserting curItr. |
345 | void renumberIndexes(IndexList::iterator curItr); |
346 | |
347 | public: |
348 | static char ID; |
349 | |
350 | SlotIndexes(); |
351 | |
352 | ~SlotIndexes() override; |
353 | |
354 | void getAnalysisUsage(AnalysisUsage &au) const override; |
355 | void releaseMemory() override; |
356 | |
357 | bool runOnMachineFunction(MachineFunction &fn) override; |
358 | |
359 | /// Dump the indexes. |
360 | void dump() const; |
361 | |
362 | /// Repair indexes after adding and removing instructions. |
363 | void repairIndexesInRange(MachineBasicBlock *MBB, |
364 | MachineBasicBlock::iterator Begin, |
365 | MachineBasicBlock::iterator End); |
366 | |
367 | /// Returns the zero index for this analysis. |
368 | SlotIndex getZeroIndex() { |
369 | assert(indexList.front().getIndex() == 0 && "First index is not 0?")((indexList.front().getIndex() == 0 && "First index is not 0?" ) ? static_cast<void> (0) : __assert_fail ("indexList.front().getIndex() == 0 && \"First index is not 0?\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 369, __PRETTY_FUNCTION__)); |
370 | return SlotIndex(&indexList.front(), 0); |
371 | } |
372 | |
373 | /// Returns the base index of the last slot in this analysis. |
374 | SlotIndex getLastIndex() { |
375 | return SlotIndex(&indexList.back(), 0); |
376 | } |
377 | |
378 | /// Returns true if the given machine instr is mapped to an index, |
379 | /// otherwise returns false. |
380 | bool hasIndex(const MachineInstr &instr) const { |
381 | return mi2iMap.count(&instr); |
382 | } |
383 | |
384 | /// Returns the base index for the given instruction. |
385 | SlotIndex getInstructionIndex(const MachineInstr &MI, |
386 | bool IgnoreBundle = false) const { |
387 | // Instructions inside a bundle have the same number as the bundle itself. |
388 | auto BundleStart = getBundleStart(MI.getIterator()); |
389 | auto BundleEnd = getBundleEnd(MI.getIterator()); |
390 | // Use the first non-debug instruction in the bundle to get SlotIndex. |
391 | const MachineInstr &BundleNonDebug = |
392 | IgnoreBundle ? MI |
393 | : *skipDebugInstructionsForward(BundleStart, BundleEnd); |
394 | assert(!BundleNonDebug.isDebugInstr() &&((!BundleNonDebug.isDebugInstr() && "Could not use a debug instruction to query mi2iMap." ) ? static_cast<void> (0) : __assert_fail ("!BundleNonDebug.isDebugInstr() && \"Could not use a debug instruction to query mi2iMap.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 395, __PRETTY_FUNCTION__)) |
395 | "Could not use a debug instruction to query mi2iMap.")((!BundleNonDebug.isDebugInstr() && "Could not use a debug instruction to query mi2iMap." ) ? static_cast<void> (0) : __assert_fail ("!BundleNonDebug.isDebugInstr() && \"Could not use a debug instruction to query mi2iMap.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 395, __PRETTY_FUNCTION__)); |
396 | Mi2IndexMap::const_iterator itr = mi2iMap.find(&BundleNonDebug); |
397 | assert(itr != mi2iMap.end() && "Instruction not found in maps.")((itr != mi2iMap.end() && "Instruction not found in maps." ) ? static_cast<void> (0) : __assert_fail ("itr != mi2iMap.end() && \"Instruction not found in maps.\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 397, __PRETTY_FUNCTION__)); |
398 | return itr->second; |
399 | } |
400 | |
401 | /// Returns the instruction for the given index, or null if the given |
402 | /// index has no instruction associated with it. |
403 | MachineInstr* getInstructionFromIndex(SlotIndex index) const { |
404 | return index.isValid() ? index.listEntry()->getInstr() : nullptr; |
405 | } |
406 | |
407 | /// Returns the next non-null index, if one exists. |
408 | /// Otherwise returns getLastIndex(). |
409 | SlotIndex getNextNonNullIndex(SlotIndex Index) { |
410 | IndexList::iterator I = Index.listEntry()->getIterator(); |
411 | IndexList::iterator E = indexList.end(); |
412 | while (++I != E) |
413 | if (I->getInstr()) |
414 | return SlotIndex(&*I, Index.getSlot()); |
415 | // We reached the end of the function. |
416 | return getLastIndex(); |
417 | } |
418 | |
419 | /// getIndexBefore - Returns the index of the last indexed instruction |
420 | /// before MI, or the start index of its basic block. |
421 | /// MI is not required to have an index. |
422 | SlotIndex getIndexBefore(const MachineInstr &MI) const { |
423 | const MachineBasicBlock *MBB = MI.getParent(); |
424 | assert(MBB && "MI must be inserted in a basic block")((MBB && "MI must be inserted in a basic block") ? static_cast <void> (0) : __assert_fail ("MBB && \"MI must be inserted in a basic block\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 424, __PRETTY_FUNCTION__)); |
425 | MachineBasicBlock::const_iterator I = MI, B = MBB->begin(); |
426 | while (true) { |
427 | if (I == B) |
428 | return getMBBStartIdx(MBB); |
429 | --I; |
430 | Mi2IndexMap::const_iterator MapItr = mi2iMap.find(&*I); |
431 | if (MapItr != mi2iMap.end()) |
432 | return MapItr->second; |
433 | } |
434 | } |
435 | |
436 | /// getIndexAfter - Returns the index of the first indexed instruction |
437 | /// after MI, or the end index of its basic block. |
438 | /// MI is not required to have an index. |
439 | SlotIndex getIndexAfter(const MachineInstr &MI) const { |
440 | const MachineBasicBlock *MBB = MI.getParent(); |
441 | assert(MBB && "MI must be inserted in a basic block")((MBB && "MI must be inserted in a basic block") ? static_cast <void> (0) : __assert_fail ("MBB && \"MI must be inserted in a basic block\"" , "/build/llvm-toolchain-snapshot-12~++20210124100612+2afaf072f5c1/llvm/include/llvm/CodeGen/SlotIndexes.h" , 441, __PRETTY_FUNCTION__)); |
442 | MachineBasicBlock::const_iterator I = MI, E = MBB->end(); |
443 | while (true) { |
444 | ++I; |
445 | if (I == E) |
446 | return getMBBEndIdx(MBB); |
447 | Mi2IndexMap::const_iterator MapItr = mi2iMap.find(&*I); |
448 | if (MapItr != mi2iMap.end()) |
449 | return MapItr->second; |