| File: | llvm/lib/CodeGen/InlineSpiller.cpp |
| Warning: | line 319, column 62 The left operand of '==' is a garbage value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- InlineSpiller.cpp - Insert spills and restores inline --------------===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // The inline spiller modifies the machine function directly instead of | |||
| 10 | // inserting spills and restores in VirtRegMap. | |||
| 11 | // | |||
| 12 | //===----------------------------------------------------------------------===// | |||
| 13 | ||||
| 14 | #include "SplitKit.h" | |||
| 15 | #include "llvm/ADT/ArrayRef.h" | |||
| 16 | #include "llvm/ADT/DenseMap.h" | |||
| 17 | #include "llvm/ADT/MapVector.h" | |||
| 18 | #include "llvm/ADT/None.h" | |||
| 19 | #include "llvm/ADT/STLExtras.h" | |||
| 20 | #include "llvm/ADT/SetVector.h" | |||
| 21 | #include "llvm/ADT/SmallPtrSet.h" | |||
| 22 | #include "llvm/ADT/SmallVector.h" | |||
| 23 | #include "llvm/ADT/Statistic.h" | |||
| 24 | #include "llvm/Analysis/AliasAnalysis.h" | |||
| 25 | #include "llvm/CodeGen/LiveInterval.h" | |||
| 26 | #include "llvm/CodeGen/LiveIntervalCalc.h" | |||
| 27 | #include "llvm/CodeGen/LiveIntervals.h" | |||
| 28 | #include "llvm/CodeGen/LiveRangeEdit.h" | |||
| 29 | #include "llvm/CodeGen/LiveStacks.h" | |||
| 30 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
| 31 | #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" | |||
| 32 | #include "llvm/CodeGen/MachineDominators.h" | |||
| 33 | #include "llvm/CodeGen/MachineFunction.h" | |||
| 34 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
| 35 | #include "llvm/CodeGen/MachineInstr.h" | |||
| 36 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
| 37 | #include "llvm/CodeGen/MachineInstrBundle.h" | |||
| 38 | #include "llvm/CodeGen/MachineLoopInfo.h" | |||
| 39 | #include "llvm/CodeGen/MachineOperand.h" | |||
| 40 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
| 41 | #include "llvm/CodeGen/SlotIndexes.h" | |||
| 42 | #include "llvm/CodeGen/Spiller.h" | |||
| 43 | #include "llvm/CodeGen/StackMaps.h" | |||
| 44 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
| 45 | #include "llvm/CodeGen/TargetOpcodes.h" | |||
| 46 | #include "llvm/CodeGen/TargetRegisterInfo.h" | |||
| 47 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
| 48 | #include "llvm/CodeGen/VirtRegMap.h" | |||
| 49 | #include "llvm/Config/llvm-config.h" | |||
| 50 | #include "llvm/Support/BlockFrequency.h" | |||
| 51 | #include "llvm/Support/BranchProbability.h" | |||
| 52 | #include "llvm/Support/CommandLine.h" | |||
| 53 | #include "llvm/Support/Compiler.h" | |||
| 54 | #include "llvm/Support/Debug.h" | |||
| 55 | #include "llvm/Support/ErrorHandling.h" | |||
| 56 | #include "llvm/Support/raw_ostream.h" | |||
| 57 | #include <cassert> | |||
| 58 | #include <iterator> | |||
| 59 | #include <tuple> | |||
| 60 | #include <utility> | |||
| 61 | #include <vector> | |||
| 62 | ||||
| 63 | using namespace llvm; | |||
| 64 | ||||
| 65 | #define DEBUG_TYPE"regalloc" "regalloc" | |||
| 66 | ||||
| 67 | STATISTIC(NumSpilledRanges, "Number of spilled live ranges")static llvm::Statistic NumSpilledRanges = {"regalloc", "NumSpilledRanges" , "Number of spilled live ranges"}; | |||
| 68 | STATISTIC(NumSnippets, "Number of spilled snippets")static llvm::Statistic NumSnippets = {"regalloc", "NumSnippets" , "Number of spilled snippets"}; | |||
| 69 | STATISTIC(NumSpills, "Number of spills inserted")static llvm::Statistic NumSpills = {"regalloc", "NumSpills", "Number of spills inserted" }; | |||
| 70 | STATISTIC(NumSpillsRemoved, "Number of spills removed")static llvm::Statistic NumSpillsRemoved = {"regalloc", "NumSpillsRemoved" , "Number of spills removed"}; | |||
| 71 | STATISTIC(NumReloads, "Number of reloads inserted")static llvm::Statistic NumReloads = {"regalloc", "NumReloads" , "Number of reloads inserted"}; | |||
| 72 | STATISTIC(NumReloadsRemoved, "Number of reloads removed")static llvm::Statistic NumReloadsRemoved = {"regalloc", "NumReloadsRemoved" , "Number of reloads removed"}; | |||
| 73 | STATISTIC(NumFolded, "Number of folded stack accesses")static llvm::Statistic NumFolded = {"regalloc", "NumFolded", "Number of folded stack accesses" }; | |||
| 74 | STATISTIC(NumFoldedLoads, "Number of folded loads")static llvm::Statistic NumFoldedLoads = {"regalloc", "NumFoldedLoads" , "Number of folded loads"}; | |||
| 75 | STATISTIC(NumRemats, "Number of rematerialized defs for spilling")static llvm::Statistic NumRemats = {"regalloc", "NumRemats", "Number of rematerialized defs for spilling" }; | |||
| 76 | ||||
| 77 | static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden, | |||
| 78 | cl::desc("Disable inline spill hoisting")); | |||
| 79 | static cl::opt<bool> | |||
| 80 | RestrictStatepointRemat("restrict-statepoint-remat", | |||
| 81 | cl::init(false), cl::Hidden, | |||
| 82 | cl::desc("Restrict remat for statepoint operands")); | |||
| 83 | ||||
| 84 | namespace { | |||
| 85 | ||||
| 86 | class HoistSpillHelper : private LiveRangeEdit::Delegate { | |||
| 87 | MachineFunction &MF; | |||
| 88 | LiveIntervals &LIS; | |||
| 89 | LiveStacks &LSS; | |||
| 90 | AliasAnalysis *AA; | |||
| 91 | MachineDominatorTree &MDT; | |||
| 92 | MachineLoopInfo &Loops; | |||
| 93 | VirtRegMap &VRM; | |||
| 94 | MachineRegisterInfo &MRI; | |||
| 95 | const TargetInstrInfo &TII; | |||
| 96 | const TargetRegisterInfo &TRI; | |||
| 97 | const MachineBlockFrequencyInfo &MBFI; | |||
| 98 | ||||
| 99 | InsertPointAnalysis IPA; | |||
| 100 | ||||
| 101 | // Map from StackSlot to the LiveInterval of the original register. | |||
| 102 | // Note the LiveInterval of the original register may have been deleted | |||
| 103 | // after it is spilled. We keep a copy here to track the range where | |||
| 104 | // spills can be moved. | |||
| 105 | DenseMap<int, std::unique_ptr<LiveInterval>> StackSlotToOrigLI; | |||
| 106 | ||||
| 107 | // Map from pair of (StackSlot and Original VNI) to a set of spills which | |||
| 108 | // have the same stackslot and have equal values defined by Original VNI. | |||
| 109 | // These spills are mergeable and are hoist candiates. | |||
| 110 | using MergeableSpillsMap = | |||
| 111 | MapVector<std::pair<int, VNInfo *>, SmallPtrSet<MachineInstr *, 16>>; | |||
| 112 | MergeableSpillsMap MergeableSpills; | |||
| 113 | ||||
| 114 | /// This is the map from original register to a set containing all its | |||
| 115 | /// siblings. To hoist a spill to another BB, we need to find out a live | |||
| 116 | /// sibling there and use it as the source of the new spill. | |||
| 117 | DenseMap<Register, SmallSetVector<Register, 16>> Virt2SiblingsMap; | |||
| 118 | ||||
| 119 | bool isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
| 120 | MachineBasicBlock &BB, Register &LiveReg); | |||
| 121 | ||||
| 122 | void rmRedundantSpills( | |||
| 123 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
| 124 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
| 125 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill); | |||
| 126 | ||||
| 127 | void getVisitOrders( | |||
| 128 | MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills, | |||
| 129 | SmallVectorImpl<MachineDomTreeNode *> &Orders, | |||
| 130 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
| 131 | DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep, | |||
| 132 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill); | |||
| 133 | ||||
| 134 | void runHoistSpills(LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
| 135 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
| 136 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
| 137 | DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns); | |||
| 138 | ||||
| 139 | public: | |||
| 140 | HoistSpillHelper(MachineFunctionPass &pass, MachineFunction &mf, | |||
| 141 | VirtRegMap &vrm) | |||
| 142 | : MF(mf), LIS(pass.getAnalysis<LiveIntervals>()), | |||
| 143 | LSS(pass.getAnalysis<LiveStacks>()), | |||
| 144 | AA(&pass.getAnalysis<AAResultsWrapperPass>().getAAResults()), | |||
| 145 | MDT(pass.getAnalysis<MachineDominatorTree>()), | |||
| 146 | Loops(pass.getAnalysis<MachineLoopInfo>()), VRM(vrm), | |||
| 147 | MRI(mf.getRegInfo()), TII(*mf.getSubtarget().getInstrInfo()), | |||
| 148 | TRI(*mf.getSubtarget().getRegisterInfo()), | |||
| 149 | MBFI(pass.getAnalysis<MachineBlockFrequencyInfo>()), | |||
| 150 | IPA(LIS, mf.getNumBlockIDs()) {} | |||
| 151 | ||||
| 152 | void addToMergeableSpills(MachineInstr &Spill, int StackSlot, | |||
| 153 | unsigned Original); | |||
| 154 | bool rmFromMergeableSpills(MachineInstr &Spill, int StackSlot); | |||
| 155 | void hoistAllSpills(); | |||
| 156 | void LRE_DidCloneVirtReg(Register, Register) override; | |||
| 157 | }; | |||
| 158 | ||||
| 159 | class InlineSpiller : public Spiller { | |||
| 160 | MachineFunction &MF; | |||
| 161 | LiveIntervals &LIS; | |||
| 162 | LiveStacks &LSS; | |||
| 163 | AliasAnalysis *AA; | |||
| 164 | MachineDominatorTree &MDT; | |||
| 165 | MachineLoopInfo &Loops; | |||
| 166 | VirtRegMap &VRM; | |||
| 167 | MachineRegisterInfo &MRI; | |||
| 168 | const TargetInstrInfo &TII; | |||
| 169 | const TargetRegisterInfo &TRI; | |||
| 170 | const MachineBlockFrequencyInfo &MBFI; | |||
| 171 | ||||
| 172 | // Variables that are valid during spill(), but used by multiple methods. | |||
| 173 | LiveRangeEdit *Edit; | |||
| 174 | LiveInterval *StackInt; | |||
| 175 | int StackSlot; | |||
| 176 | Register Original; | |||
| 177 | ||||
| 178 | // All registers to spill to StackSlot, including the main register. | |||
| 179 | SmallVector<Register, 8> RegsToSpill; | |||
| 180 | ||||
| 181 | // All COPY instructions to/from snippets. | |||
| 182 | // They are ignored since both operands refer to the same stack slot. | |||
| 183 | SmallPtrSet<MachineInstr*, 8> SnippetCopies; | |||
| 184 | ||||
| 185 | // Values that failed to remat at some point. | |||
| 186 | SmallPtrSet<VNInfo*, 8> UsedValues; | |||
| 187 | ||||
| 188 | // Dead defs generated during spilling. | |||
| 189 | SmallVector<MachineInstr*, 8> DeadDefs; | |||
| 190 | ||||
| 191 | // Object records spills information and does the hoisting. | |||
| 192 | HoistSpillHelper HSpiller; | |||
| 193 | ||||
| 194 | // Live range weight calculator. | |||
| 195 | VirtRegAuxInfo &VRAI; | |||
| 196 | ||||
| 197 | ~InlineSpiller() override = default; | |||
| 198 | ||||
| 199 | public: | |||
| 200 | InlineSpiller(MachineFunctionPass &Pass, MachineFunction &MF, VirtRegMap &VRM, | |||
| 201 | VirtRegAuxInfo &VRAI) | |||
| 202 | : MF(MF), LIS(Pass.getAnalysis<LiveIntervals>()), | |||
| 203 | LSS(Pass.getAnalysis<LiveStacks>()), | |||
| 204 | AA(&Pass.getAnalysis<AAResultsWrapperPass>().getAAResults()), | |||
| 205 | MDT(Pass.getAnalysis<MachineDominatorTree>()), | |||
| 206 | Loops(Pass.getAnalysis<MachineLoopInfo>()), VRM(VRM), | |||
| 207 | MRI(MF.getRegInfo()), TII(*MF.getSubtarget().getInstrInfo()), | |||
| 208 | TRI(*MF.getSubtarget().getRegisterInfo()), | |||
| 209 | MBFI(Pass.getAnalysis<MachineBlockFrequencyInfo>()), | |||
| 210 | HSpiller(Pass, MF, VRM), VRAI(VRAI) {} | |||
| 211 | ||||
| 212 | void spill(LiveRangeEdit &) override; | |||
| 213 | void postOptimization() override; | |||
| 214 | ||||
| 215 | private: | |||
| 216 | bool isSnippet(const LiveInterval &SnipLI); | |||
| 217 | void collectRegsToSpill(); | |||
| 218 | ||||
| 219 | bool isRegToSpill(Register Reg) { return is_contained(RegsToSpill, Reg); } | |||
| 220 | ||||
| 221 | bool isSibling(Register Reg); | |||
| 222 | bool hoistSpillInsideBB(LiveInterval &SpillLI, MachineInstr &CopyMI); | |||
| 223 | void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI); | |||
| 224 | ||||
| 225 | void markValueUsed(LiveInterval*, VNInfo*); | |||
| 226 | bool canGuaranteeAssignmentAfterRemat(Register VReg, MachineInstr &MI); | |||
| 227 | bool reMaterializeFor(LiveInterval &, MachineInstr &MI); | |||
| 228 | void reMaterializeAll(); | |||
| 229 | ||||
| 230 | bool coalesceStackAccess(MachineInstr *MI, Register Reg); | |||
| 231 | bool foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>>, | |||
| 232 | MachineInstr *LoadMI = nullptr); | |||
| 233 | void insertReload(Register VReg, SlotIndex, MachineBasicBlock::iterator MI); | |||
| 234 | void insertSpill(Register VReg, bool isKill, MachineBasicBlock::iterator MI); | |||
| 235 | ||||
| 236 | void spillAroundUses(Register Reg); | |||
| 237 | void spillAll(); | |||
| 238 | }; | |||
| 239 | ||||
| 240 | } // end anonymous namespace | |||
| 241 | ||||
| 242 | Spiller::~Spiller() = default; | |||
| 243 | ||||
| 244 | void Spiller::anchor() {} | |||
| 245 | ||||
| 246 | Spiller *llvm::createInlineSpiller(MachineFunctionPass &Pass, | |||
| 247 | MachineFunction &MF, VirtRegMap &VRM, | |||
| 248 | VirtRegAuxInfo &VRAI) { | |||
| 249 | return new InlineSpiller(Pass, MF, VRM, VRAI); | |||
| 250 | } | |||
| 251 | ||||
| 252 | //===----------------------------------------------------------------------===// | |||
| 253 | // Snippets | |||
| 254 | //===----------------------------------------------------------------------===// | |||
| 255 | ||||
| 256 | // When spilling a virtual register, we also spill any snippets it is connected | |||
| 257 | // to. The snippets are small live ranges that only have a single real use, | |||
| 258 | // leftovers from live range splitting. Spilling them enables memory operand | |||
| 259 | // folding or tightens the live range around the single use. | |||
| 260 | // | |||
| 261 | // This minimizes register pressure and maximizes the store-to-load distance for | |||
| 262 | // spill slots which can be important in tight loops. | |||
| 263 | ||||
| 264 | /// isFullCopyOf - If MI is a COPY to or from Reg, return the other register, | |||
| 265 | /// otherwise return 0. | |||
| 266 | static Register isFullCopyOf(const MachineInstr &MI, Register Reg) { | |||
| 267 | if (!MI.isFullCopy()) | |||
| 268 | return Register(); | |||
| 269 | if (MI.getOperand(0).getReg() == Reg) | |||
| 270 | return MI.getOperand(1).getReg(); | |||
| 271 | if (MI.getOperand(1).getReg() == Reg) | |||
| 272 | return MI.getOperand(0).getReg(); | |||
| 273 | return Register(); | |||
| 274 | } | |||
| 275 | ||||
| 276 | static void getVDefInterval(const MachineInstr &MI, LiveIntervals &LIS) { | |||
| 277 | for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { | |||
| 278 | const MachineOperand &MO = MI.getOperand(I); | |||
| 279 | if (MO.isReg() && MO.isDef() && Register::isVirtualRegister(MO.getReg())) | |||
| 280 | LIS.getInterval(MO.getReg()); | |||
| 281 | } | |||
| 282 | } | |||
| 283 | ||||
| 284 | /// isSnippet - Identify if a live interval is a snippet that should be spilled. | |||
| 285 | /// It is assumed that SnipLI is a virtual register with the same original as | |||
| 286 | /// Edit->getReg(). | |||
| 287 | bool InlineSpiller::isSnippet(const LiveInterval &SnipLI) { | |||
| 288 | Register Reg = Edit->getReg(); | |||
| 289 | ||||
| 290 | // A snippet is a tiny live range with only a single instruction using it | |||
| 291 | // besides copies to/from Reg or spills/fills. We accept: | |||
| 292 | // | |||
| 293 | // %snip = COPY %Reg / FILL fi# | |||
| 294 | // %snip = USE %snip | |||
| 295 | // %Reg = COPY %snip / SPILL %snip, fi# | |||
| 296 | // | |||
| 297 | if (SnipLI.getNumValNums() > 2 || !LIS.intervalIsInOneMBB(SnipLI)) | |||
| ||||
| 298 | return false; | |||
| 299 | ||||
| 300 | MachineInstr *UseMI = nullptr; | |||
| 301 | ||||
| 302 | // Check that all uses satisfy our criteria. | |||
| 303 | for (MachineRegisterInfo::reg_instr_nodbg_iterator | |||
| 304 | RI = MRI.reg_instr_nodbg_begin(SnipLI.reg()), | |||
| 305 | E = MRI.reg_instr_nodbg_end(); | |||
| 306 | RI != E;) { | |||
| 307 | MachineInstr &MI = *RI++; | |||
| 308 | ||||
| 309 | // Allow copies to/from Reg. | |||
| 310 | if (isFullCopyOf(MI, Reg)) | |||
| 311 | continue; | |||
| 312 | ||||
| 313 | // Allow stack slot loads. | |||
| 314 | int FI; | |||
| 315 | if (SnipLI.reg() == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) | |||
| 316 | continue; | |||
| 317 | ||||
| 318 | // Allow stack slot stores. | |||
| 319 | if (SnipLI.reg() == TII.isStoreToStackSlot(MI, FI) && FI == StackSlot) | |||
| ||||
| 320 | continue; | |||
| 321 | ||||
| 322 | // Allow a single additional instruction. | |||
| 323 | if (UseMI && &MI != UseMI) | |||
| 324 | return false; | |||
| 325 | UseMI = &MI; | |||
| 326 | } | |||
| 327 | return true; | |||
| 328 | } | |||
| 329 | ||||
| 330 | /// collectRegsToSpill - Collect live range snippets that only have a single | |||
| 331 | /// real use. | |||
| 332 | void InlineSpiller::collectRegsToSpill() { | |||
| 333 | Register Reg = Edit->getReg(); | |||
| 334 | ||||
| 335 | // Main register always spills. | |||
| 336 | RegsToSpill.assign(1, Reg); | |||
| 337 | SnippetCopies.clear(); | |||
| 338 | ||||
| 339 | // Snippets all have the same original, so there can't be any for an original | |||
| 340 | // register. | |||
| 341 | if (Original == Reg) | |||
| 342 | return; | |||
| 343 | ||||
| 344 | for (MachineRegisterInfo::reg_instr_iterator | |||
| 345 | RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end(); RI != E; ) { | |||
| 346 | MachineInstr &MI = *RI++; | |||
| 347 | Register SnipReg = isFullCopyOf(MI, Reg); | |||
| 348 | if (!isSibling(SnipReg)) | |||
| 349 | continue; | |||
| 350 | LiveInterval &SnipLI = LIS.getInterval(SnipReg); | |||
| 351 | if (!isSnippet(SnipLI)) | |||
| 352 | continue; | |||
| 353 | SnippetCopies.insert(&MI); | |||
| 354 | if (isRegToSpill(SnipReg)) | |||
| 355 | continue; | |||
| 356 | RegsToSpill.push_back(SnipReg); | |||
| 357 | LLVM_DEBUG(dbgs() << "\talso spill snippet " << SnipLI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\talso spill snippet " << SnipLI << '\n'; } } while (false); | |||
| 358 | ++NumSnippets; | |||
| 359 | } | |||
| 360 | } | |||
| 361 | ||||
| 362 | bool InlineSpiller::isSibling(Register Reg) { | |||
| 363 | return Reg.isVirtual() && VRM.getOriginal(Reg) == Original; | |||
| 364 | } | |||
| 365 | ||||
| 366 | /// It is beneficial to spill to earlier place in the same BB in case | |||
| 367 | /// as follows: | |||
| 368 | /// There is an alternative def earlier in the same MBB. | |||
| 369 | /// Hoist the spill as far as possible in SpillMBB. This can ease | |||
| 370 | /// register pressure: | |||
| 371 | /// | |||
| 372 | /// x = def | |||
| 373 | /// y = use x | |||
| 374 | /// s = copy x | |||
| 375 | /// | |||
| 376 | /// Hoisting the spill of s to immediately after the def removes the | |||
| 377 | /// interference between x and y: | |||
| 378 | /// | |||
| 379 | /// x = def | |||
| 380 | /// spill x | |||
| 381 | /// y = use killed x | |||
| 382 | /// | |||
| 383 | /// This hoist only helps when the copy kills its source. | |||
| 384 | /// | |||
| 385 | bool InlineSpiller::hoistSpillInsideBB(LiveInterval &SpillLI, | |||
| 386 | MachineInstr &CopyMI) { | |||
| 387 | SlotIndex Idx = LIS.getInstructionIndex(CopyMI); | |||
| 388 | #ifndef NDEBUG | |||
| 389 | VNInfo *VNI = SpillLI.getVNInfoAt(Idx.getRegSlot()); | |||
| 390 | assert(VNI && VNI->def == Idx.getRegSlot() && "Not defined by copy")((VNI && VNI->def == Idx.getRegSlot() && "Not defined by copy" ) ? static_cast<void> (0) : __assert_fail ("VNI && VNI->def == Idx.getRegSlot() && \"Not defined by copy\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 390, __PRETTY_FUNCTION__)); | |||
| 391 | #endif | |||
| 392 | ||||
| 393 | Register SrcReg = CopyMI.getOperand(1).getReg(); | |||
| 394 | LiveInterval &SrcLI = LIS.getInterval(SrcReg); | |||
| 395 | VNInfo *SrcVNI = SrcLI.getVNInfoAt(Idx); | |||
| 396 | LiveQueryResult SrcQ = SrcLI.Query(Idx); | |||
| 397 | MachineBasicBlock *DefMBB = LIS.getMBBFromIndex(SrcVNI->def); | |||
| 398 | if (DefMBB != CopyMI.getParent() || !SrcQ.isKill()) | |||
| 399 | return false; | |||
| 400 | ||||
| 401 | // Conservatively extend the stack slot range to the range of the original | |||
| 402 | // value. We may be able to do better with stack slot coloring by being more | |||
| 403 | // careful here. | |||
| 404 | assert(StackInt && "No stack slot assigned yet.")((StackInt && "No stack slot assigned yet.") ? static_cast <void> (0) : __assert_fail ("StackInt && \"No stack slot assigned yet.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 404, __PRETTY_FUNCTION__)); | |||
| 405 | LiveInterval &OrigLI = LIS.getInterval(Original); | |||
| 406 | VNInfo *OrigVNI = OrigLI.getVNInfoAt(Idx); | |||
| 407 | StackInt->MergeValueInAsValue(OrigLI, OrigVNI, StackInt->getValNumInfo(0)); | |||
| 408 | LLVM_DEBUG(dbgs() << "\tmerged orig valno " << OrigVNI->id << ": "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tmerged orig valno " << OrigVNI->id << ": " << *StackInt << '\n' ; } } while (false) | |||
| 409 | << *StackInt << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tmerged orig valno " << OrigVNI->id << ": " << *StackInt << '\n' ; } } while (false); | |||
| 410 | ||||
| 411 | // We are going to spill SrcVNI immediately after its def, so clear out | |||
| 412 | // any later spills of the same value. | |||
| 413 | eliminateRedundantSpills(SrcLI, SrcVNI); | |||
| 414 | ||||
| 415 | MachineBasicBlock *MBB = LIS.getMBBFromIndex(SrcVNI->def); | |||
| 416 | MachineBasicBlock::iterator MII; | |||
| 417 | if (SrcVNI->isPHIDef()) | |||
| 418 | MII = MBB->SkipPHIsLabelsAndDebug(MBB->begin()); | |||
| 419 | else { | |||
| 420 | MachineInstr *DefMI = LIS.getInstructionFromIndex(SrcVNI->def); | |||
| 421 | assert(DefMI && "Defining instruction disappeared")((DefMI && "Defining instruction disappeared") ? static_cast <void> (0) : __assert_fail ("DefMI && \"Defining instruction disappeared\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 421, __PRETTY_FUNCTION__)); | |||
| 422 | MII = DefMI; | |||
| 423 | ++MII; | |||
| 424 | } | |||
| 425 | MachineInstrSpan MIS(MII, MBB); | |||
| 426 | // Insert spill without kill flag immediately after def. | |||
| 427 | TII.storeRegToStackSlot(*MBB, MII, SrcReg, false, StackSlot, | |||
| 428 | MRI.getRegClass(SrcReg), &TRI); | |||
| 429 | LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MII); | |||
| 430 | for (const MachineInstr &MI : make_range(MIS.begin(), MII)) | |||
| 431 | getVDefInterval(MI, LIS); | |||
| 432 | --MII; // Point to store instruction. | |||
| 433 | LLVM_DEBUG(dbgs() << "\thoisted: " << SrcVNI->def << '\t' << *MII)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\thoisted: " << SrcVNI ->def << '\t' << *MII; } } while (false); | |||
| 434 | ||||
| 435 | // If there is only 1 store instruction is required for spill, add it | |||
| 436 | // to mergeable list. In X86 AMX, 2 intructions are required to store. | |||
| 437 | // We disable the merge for this case. | |||
| 438 | if (MIS.begin() == MII) | |||
| 439 | HSpiller.addToMergeableSpills(*MII, StackSlot, Original); | |||
| 440 | ++NumSpills; | |||
| 441 | return true; | |||
| 442 | } | |||
| 443 | ||||
| 444 | /// eliminateRedundantSpills - SLI:VNI is known to be on the stack. Remove any | |||
| 445 | /// redundant spills of this value in SLI.reg and sibling copies. | |||
| 446 | void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { | |||
| 447 | assert(VNI && "Missing value")((VNI && "Missing value") ? static_cast<void> ( 0) : __assert_fail ("VNI && \"Missing value\"", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 447, __PRETTY_FUNCTION__)); | |||
| 448 | SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList; | |||
| 449 | WorkList.push_back(std::make_pair(&SLI, VNI)); | |||
| 450 | assert(StackInt && "No stack slot assigned yet.")((StackInt && "No stack slot assigned yet.") ? static_cast <void> (0) : __assert_fail ("StackInt && \"No stack slot assigned yet.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 450, __PRETTY_FUNCTION__)); | |||
| 451 | ||||
| 452 | do { | |||
| 453 | LiveInterval *LI; | |||
| 454 | std::tie(LI, VNI) = WorkList.pop_back_val(); | |||
| 455 | Register Reg = LI->reg(); | |||
| 456 | LLVM_DEBUG(dbgs() << "Checking redundant spills for " << VNI->id << '@'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Checking redundant spills for " << VNI->id << '@' << VNI->def << " in " << *LI << '\n'; } } while (false) | |||
| 457 | << VNI->def << " in " << *LI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Checking redundant spills for " << VNI->id << '@' << VNI->def << " in " << *LI << '\n'; } } while (false); | |||
| 458 | ||||
| 459 | // Regs to spill are taken care of. | |||
| 460 | if (isRegToSpill(Reg)) | |||
| 461 | continue; | |||
| 462 | ||||
| 463 | // Add all of VNI's live range to StackInt. | |||
| 464 | StackInt->MergeValueInAsValue(*LI, VNI, StackInt->getValNumInfo(0)); | |||
| 465 | LLVM_DEBUG(dbgs() << "Merged to stack int: " << *StackInt << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Merged to stack int: " << *StackInt << '\n'; } } while (false); | |||
| 466 | ||||
| 467 | // Find all spills and copies of VNI. | |||
| 468 | for (MachineRegisterInfo::use_instr_nodbg_iterator | |||
| 469 | UI = MRI.use_instr_nodbg_begin(Reg), E = MRI.use_instr_nodbg_end(); | |||
| 470 | UI != E; ) { | |||
| 471 | MachineInstr &MI = *UI++; | |||
| 472 | if (!MI.isCopy() && !MI.mayStore()) | |||
| 473 | continue; | |||
| 474 | SlotIndex Idx = LIS.getInstructionIndex(MI); | |||
| 475 | if (LI->getVNInfoAt(Idx) != VNI) | |||
| 476 | continue; | |||
| 477 | ||||
| 478 | // Follow sibling copies down the dominator tree. | |||
| 479 | if (Register DstReg = isFullCopyOf(MI, Reg)) { | |||
| 480 | if (isSibling(DstReg)) { | |||
| 481 | LiveInterval &DstLI = LIS.getInterval(DstReg); | |||
| 482 | VNInfo *DstVNI = DstLI.getVNInfoAt(Idx.getRegSlot()); | |||
| 483 | assert(DstVNI && "Missing defined value")((DstVNI && "Missing defined value") ? static_cast< void> (0) : __assert_fail ("DstVNI && \"Missing defined value\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 483, __PRETTY_FUNCTION__)); | |||
| 484 | assert(DstVNI->def == Idx.getRegSlot() && "Wrong copy def slot")((DstVNI->def == Idx.getRegSlot() && "Wrong copy def slot" ) ? static_cast<void> (0) : __assert_fail ("DstVNI->def == Idx.getRegSlot() && \"Wrong copy def slot\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 484, __PRETTY_FUNCTION__)); | |||
| 485 | WorkList.push_back(std::make_pair(&DstLI, DstVNI)); | |||
| 486 | } | |||
| 487 | continue; | |||
| 488 | } | |||
| 489 | ||||
| 490 | // Erase spills. | |||
| 491 | int FI; | |||
| 492 | if (Reg == TII.isStoreToStackSlot(MI, FI) && FI == StackSlot) { | |||
| 493 | LLVM_DEBUG(dbgs() << "Redundant spill " << Idx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Redundant spill " << Idx << '\t' << MI; } } while (false); | |||
| 494 | // eliminateDeadDefs won't normally remove stores, so switch opcode. | |||
| 495 | MI.setDesc(TII.get(TargetOpcode::KILL)); | |||
| 496 | DeadDefs.push_back(&MI); | |||
| 497 | ++NumSpillsRemoved; | |||
| 498 | if (HSpiller.rmFromMergeableSpills(MI, StackSlot)) | |||
| 499 | --NumSpills; | |||
| 500 | } | |||
| 501 | } | |||
| 502 | } while (!WorkList.empty()); | |||
| 503 | } | |||
| 504 | ||||
| 505 | //===----------------------------------------------------------------------===// | |||
| 506 | // Rematerialization | |||
| 507 | //===----------------------------------------------------------------------===// | |||
| 508 | ||||
| 509 | /// markValueUsed - Remember that VNI failed to rematerialize, so its defining | |||
| 510 | /// instruction cannot be eliminated. See through snippet copies | |||
| 511 | void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) { | |||
| 512 | SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList; | |||
| 513 | WorkList.push_back(std::make_pair(LI, VNI)); | |||
| 514 | do { | |||
| 515 | std::tie(LI, VNI) = WorkList.pop_back_val(); | |||
| 516 | if (!UsedValues.insert(VNI).second) | |||
| 517 | continue; | |||
| 518 | ||||
| 519 | if (VNI->isPHIDef()) { | |||
| 520 | MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); | |||
| 521 | for (MachineBasicBlock *P : MBB->predecessors()) { | |||
| 522 | VNInfo *PVNI = LI->getVNInfoBefore(LIS.getMBBEndIdx(P)); | |||
| 523 | if (PVNI) | |||
| 524 | WorkList.push_back(std::make_pair(LI, PVNI)); | |||
| 525 | } | |||
| 526 | continue; | |||
| 527 | } | |||
| 528 | ||||
| 529 | // Follow snippet copies. | |||
| 530 | MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def); | |||
| 531 | if (!SnippetCopies.count(MI)) | |||
| 532 | continue; | |||
| 533 | LiveInterval &SnipLI = LIS.getInterval(MI->getOperand(1).getReg()); | |||
| 534 | assert(isRegToSpill(SnipLI.reg()) && "Unexpected register in copy")((isRegToSpill(SnipLI.reg()) && "Unexpected register in copy" ) ? static_cast<void> (0) : __assert_fail ("isRegToSpill(SnipLI.reg()) && \"Unexpected register in copy\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 534, __PRETTY_FUNCTION__)); | |||
| 535 | VNInfo *SnipVNI = SnipLI.getVNInfoAt(VNI->def.getRegSlot(true)); | |||
| 536 | assert(SnipVNI && "Snippet undefined before copy")((SnipVNI && "Snippet undefined before copy") ? static_cast <void> (0) : __assert_fail ("SnipVNI && \"Snippet undefined before copy\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 536, __PRETTY_FUNCTION__)); | |||
| 537 | WorkList.push_back(std::make_pair(&SnipLI, SnipVNI)); | |||
| 538 | } while (!WorkList.empty()); | |||
| 539 | } | |||
| 540 | ||||
| 541 | bool InlineSpiller::canGuaranteeAssignmentAfterRemat(Register VReg, | |||
| 542 | MachineInstr &MI) { | |||
| 543 | if (!RestrictStatepointRemat) | |||
| 544 | return true; | |||
| 545 | // Here's a quick explanation of the problem we're trying to handle here: | |||
| 546 | // * There are some pseudo instructions with more vreg uses than there are | |||
| 547 | // physical registers on the machine. | |||
| 548 | // * This is normally handled by spilling the vreg, and folding the reload | |||
| 549 | // into the user instruction. (Thus decreasing the number of used vregs | |||
| 550 | // until the remainder can be assigned to physregs.) | |||
| 551 | // * However, since we may try to spill vregs in any order, we can end up | |||
| 552 | // trying to spill each operand to the instruction, and then rematting it | |||
| 553 | // instead. When that happens, the new live intervals (for the remats) are | |||
| 554 | // expected to be trivially assignable (i.e. RS_Done). However, since we | |||
| 555 | // may have more remats than physregs, we're guaranteed to fail to assign | |||
| 556 | // one. | |||
| 557 | // At the moment, we only handle this for STATEPOINTs since they're the only | |||
| 558 | // pseudo op where we've seen this. If we start seeing other instructions | |||
| 559 | // with the same problem, we need to revisit this. | |||
| 560 | if (MI.getOpcode() != TargetOpcode::STATEPOINT) | |||
| 561 | return true; | |||
| 562 | // For STATEPOINTs we allow re-materialization for fixed arguments only hoping | |||
| 563 | // that number of physical registers is enough to cover all fixed arguments. | |||
| 564 | // If it is not true we need to revisit it. | |||
| 565 | for (unsigned Idx = StatepointOpers(&MI).getVarIdx(), | |||
| 566 | EndIdx = MI.getNumOperands(); | |||
| 567 | Idx < EndIdx; ++Idx) { | |||
| 568 | MachineOperand &MO = MI.getOperand(Idx); | |||
| 569 | if (MO.isReg() && MO.getReg() == VReg) | |||
| 570 | return false; | |||
| 571 | } | |||
| 572 | return true; | |||
| 573 | } | |||
| 574 | ||||
| 575 | /// reMaterializeFor - Attempt to rematerialize before MI instead of reloading. | |||
| 576 | bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) { | |||
| 577 | // Analyze instruction | |||
| 578 | SmallVector<std::pair<MachineInstr *, unsigned>, 8> Ops; | |||
| 579 | VirtRegInfo RI = AnalyzeVirtRegInBundle(MI, VirtReg.reg(), &Ops); | |||
| 580 | ||||
| 581 | if (!RI.Reads) | |||
| 582 | return false; | |||
| 583 | ||||
| 584 | SlotIndex UseIdx = LIS.getInstructionIndex(MI).getRegSlot(true); | |||
| 585 | VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex()); | |||
| 586 | ||||
| 587 | if (!ParentVNI) { | |||
| 588 | LLVM_DEBUG(dbgs() << "\tadding <undef> flags: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tadding <undef> flags: " ; } } while (false); | |||
| 589 | for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { | |||
| 590 | MachineOperand &MO = MI.getOperand(i); | |||
| 591 | if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg()) | |||
| 592 | MO.setIsUndef(); | |||
| 593 | } | |||
| 594 | LLVM_DEBUG(dbgs() << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << UseIdx << '\t' << MI; } } while (false); | |||
| 595 | return true; | |||
| 596 | } | |||
| 597 | ||||
| 598 | if (SnippetCopies.count(&MI)) | |||
| 599 | return false; | |||
| 600 | ||||
| 601 | LiveInterval &OrigLI = LIS.getInterval(Original); | |||
| 602 | VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx); | |||
| 603 | LiveRangeEdit::Remat RM(ParentVNI); | |||
| 604 | RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def); | |||
| 605 | ||||
| 606 | if (!Edit->canRematerializeAt(RM, OrigVNI, UseIdx, false)) { | |||
| 607 | markValueUsed(&VirtReg, ParentVNI); | |||
| 608 | LLVM_DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI; } } while (false); | |||
| 609 | return false; | |||
| 610 | } | |||
| 611 | ||||
| 612 | // If the instruction also writes VirtReg.reg, it had better not require the | |||
| 613 | // same register for uses and defs. | |||
| 614 | if (RI.Tied) { | |||
| 615 | markValueUsed(&VirtReg, ParentVNI); | |||
| 616 | LLVM_DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << MI; } } while (false); | |||
| 617 | return false; | |||
| 618 | } | |||
| 619 | ||||
| 620 | // Before rematerializing into a register for a single instruction, try to | |||
| 621 | // fold a load into the instruction. That avoids allocating a new register. | |||
| 622 | if (RM.OrigMI->canFoldAsLoad() && | |||
| 623 | foldMemoryOperand(Ops, RM.OrigMI)) { | |||
| 624 | Edit->markRematerialized(RM.ParentVNI); | |||
| 625 | ++NumFoldedLoads; | |||
| 626 | return true; | |||
| 627 | } | |||
| 628 | ||||
| 629 | // If we can't guarantee that we'll be able to actually assign the new vreg, | |||
| 630 | // we can't remat. | |||
| 631 | if (!canGuaranteeAssignmentAfterRemat(VirtReg.reg(), MI)) { | |||
| 632 | markValueUsed(&VirtReg, ParentVNI); | |||
| 633 | LLVM_DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI; } } while (false); | |||
| 634 | return false; | |||
| 635 | } | |||
| 636 | ||||
| 637 | // Allocate a new register for the remat. | |||
| 638 | Register NewVReg = Edit->createFrom(Original); | |||
| 639 | ||||
| 640 | // Finally we can rematerialize OrigMI before MI. | |||
| 641 | SlotIndex DefIdx = | |||
| 642 | Edit->rematerializeAt(*MI.getParent(), MI, NewVReg, RM, TRI); | |||
| 643 | ||||
| 644 | // We take the DebugLoc from MI, since OrigMI may be attributed to a | |||
| 645 | // different source location. | |||
| 646 | auto *NewMI = LIS.getInstructionFromIndex(DefIdx); | |||
| 647 | NewMI->setDebugLoc(MI.getDebugLoc()); | |||
| 648 | ||||
| 649 | (void)DefIdx; | |||
| 650 | LLVM_DEBUG(dbgs() << "\tremat: " << DefIdx << '\t'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremat: " << DefIdx << '\t' << *LIS.getInstructionFromIndex(DefIdx); } } while (false) | |||
| 651 | << *LIS.getInstructionFromIndex(DefIdx))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\tremat: " << DefIdx << '\t' << *LIS.getInstructionFromIndex(DefIdx); } } while (false); | |||
| 652 | ||||
| 653 | // Replace operands | |||
| 654 | for (const auto &OpPair : Ops) { | |||
| 655 | MachineOperand &MO = OpPair.first->getOperand(OpPair.second); | |||
| 656 | if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg()) { | |||
| 657 | MO.setReg(NewVReg); | |||
| 658 | MO.setIsKill(); | |||
| 659 | } | |||
| 660 | } | |||
| 661 | LLVM_DEBUG(dbgs() << "\t " << UseIdx << '\t' << MI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\t " << UseIdx << '\t' << MI << '\n'; } } while (false); | |||
| 662 | ||||
| 663 | ++NumRemats; | |||
| 664 | return true; | |||
| 665 | } | |||
| 666 | ||||
| 667 | /// reMaterializeAll - Try to rematerialize as many uses as possible, | |||
| 668 | /// and trim the live ranges after. | |||
| 669 | void InlineSpiller::reMaterializeAll() { | |||
| 670 | if (!Edit->anyRematerializable(AA)) | |||
| 671 | return; | |||
| 672 | ||||
| 673 | UsedValues.clear(); | |||
| 674 | ||||
| 675 | // Try to remat before all uses of snippets. | |||
| 676 | bool anyRemat = false; | |||
| 677 | for (Register Reg : RegsToSpill) { | |||
| 678 | LiveInterval &LI = LIS.getInterval(Reg); | |||
| 679 | for (MachineRegisterInfo::reg_bundle_iterator | |||
| 680 | RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end(); | |||
| 681 | RegI != E; ) { | |||
| 682 | MachineInstr &MI = *RegI++; | |||
| 683 | ||||
| 684 | // Debug values are not allowed to affect codegen. | |||
| 685 | if (MI.isDebugValue()) | |||
| 686 | continue; | |||
| 687 | ||||
| 688 | assert(!MI.isDebugInstr() && "Did not expect to find a use in debug "((!MI.isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? static_cast<void> (0) : __assert_fail ("!MI.isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 689, __PRETTY_FUNCTION__)) | |||
| 689 | "instruction that isn't a DBG_VALUE")((!MI.isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? static_cast<void> (0) : __assert_fail ("!MI.isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 689, __PRETTY_FUNCTION__)); | |||
| 690 | ||||
| 691 | anyRemat |= reMaterializeFor(LI, MI); | |||
| 692 | } | |||
| 693 | } | |||
| 694 | if (!anyRemat) | |||
| 695 | return; | |||
| 696 | ||||
| 697 | // Remove any values that were completely rematted. | |||
| 698 | for (Register Reg : RegsToSpill) { | |||
| 699 | LiveInterval &LI = LIS.getInterval(Reg); | |||
| 700 | for (LiveInterval::vni_iterator I = LI.vni_begin(), E = LI.vni_end(); | |||
| 701 | I != E; ++I) { | |||
| 702 | VNInfo *VNI = *I; | |||
| 703 | if (VNI->isUnused() || VNI->isPHIDef() || UsedValues.count(VNI)) | |||
| 704 | continue; | |||
| 705 | MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def); | |||
| 706 | MI->addRegisterDead(Reg, &TRI); | |||
| 707 | if (!MI->allDefsAreDead()) | |||
| 708 | continue; | |||
| 709 | LLVM_DEBUG(dbgs() << "All defs dead: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "All defs dead: " << *MI ; } } while (false); | |||
| 710 | DeadDefs.push_back(MI); | |||
| 711 | } | |||
| 712 | } | |||
| 713 | ||||
| 714 | // Eliminate dead code after remat. Note that some snippet copies may be | |||
| 715 | // deleted here. | |||
| 716 | if (DeadDefs.empty()) | |||
| 717 | return; | |||
| 718 | LLVM_DEBUG(dbgs() << "Remat created " << DeadDefs.size() << " dead defs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Remat created " << DeadDefs .size() << " dead defs.\n"; } } while (false); | |||
| 719 | Edit->eliminateDeadDefs(DeadDefs, RegsToSpill, AA); | |||
| 720 | ||||
| 721 | // LiveRangeEdit::eliminateDeadDef is used to remove dead define instructions | |||
| 722 | // after rematerialization. To remove a VNI for a vreg from its LiveInterval, | |||
| 723 | // LiveIntervals::removeVRegDefAt is used. However, after non-PHI VNIs are all | |||
| 724 | // removed, PHI VNI are still left in the LiveInterval. | |||
| 725 | // So to get rid of unused reg, we need to check whether it has non-dbg | |||
| 726 | // reference instead of whether it has non-empty interval. | |||
| 727 | unsigned ResultPos = 0; | |||
| 728 | for (Register Reg : RegsToSpill) { | |||
| 729 | if (MRI.reg_nodbg_empty(Reg)) { | |||
| 730 | Edit->eraseVirtReg(Reg); | |||
| 731 | continue; | |||
| 732 | } | |||
| 733 | ||||
| 734 | assert(LIS.hasInterval(Reg) &&((LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty () || !MRI.reg_nodbg_empty(Reg)) && "Empty and not used live-range?!" ) ? static_cast<void> (0) : __assert_fail ("LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && \"Empty and not used live-range?!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 736, __PRETTY_FUNCTION__)) | |||
| 735 | (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) &&((LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty () || !MRI.reg_nodbg_empty(Reg)) && "Empty and not used live-range?!" ) ? static_cast<void> (0) : __assert_fail ("LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && \"Empty and not used live-range?!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 736, __PRETTY_FUNCTION__)) | |||
| 736 | "Empty and not used live-range?!")((LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty () || !MRI.reg_nodbg_empty(Reg)) && "Empty and not used live-range?!" ) ? static_cast<void> (0) : __assert_fail ("LIS.hasInterval(Reg) && (!LIS.getInterval(Reg).empty() || !MRI.reg_nodbg_empty(Reg)) && \"Empty and not used live-range?!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 736, __PRETTY_FUNCTION__)); | |||
| 737 | ||||
| 738 | RegsToSpill[ResultPos++] = Reg; | |||
| 739 | } | |||
| 740 | RegsToSpill.erase(RegsToSpill.begin() + ResultPos, RegsToSpill.end()); | |||
| 741 | LLVM_DEBUG(dbgs() << RegsToSpill.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << RegsToSpill.size() << " registers to spill after remat.\n" ; } } while (false) | |||
| 742 | << " registers to spill after remat.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << RegsToSpill.size() << " registers to spill after remat.\n" ; } } while (false); | |||
| 743 | } | |||
| 744 | ||||
| 745 | //===----------------------------------------------------------------------===// | |||
| 746 | // Spilling | |||
| 747 | //===----------------------------------------------------------------------===// | |||
| 748 | ||||
| 749 | /// If MI is a load or store of StackSlot, it can be removed. | |||
| 750 | bool InlineSpiller::coalesceStackAccess(MachineInstr *MI, Register Reg) { | |||
| 751 | int FI = 0; | |||
| 752 | Register InstrReg = TII.isLoadFromStackSlot(*MI, FI); | |||
| 753 | bool IsLoad = InstrReg; | |||
| 754 | if (!IsLoad) | |||
| 755 | InstrReg = TII.isStoreToStackSlot(*MI, FI); | |||
| 756 | ||||
| 757 | // We have a stack access. Is it the right register and slot? | |||
| 758 | if (InstrReg != Reg || FI != StackSlot) | |||
| 759 | return false; | |||
| 760 | ||||
| 761 | if (!IsLoad) | |||
| 762 | HSpiller.rmFromMergeableSpills(*MI, StackSlot); | |||
| 763 | ||||
| 764 | LLVM_DEBUG(dbgs() << "Coalescing stack access: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Coalescing stack access: " << *MI; } } while (false); | |||
| 765 | LIS.RemoveMachineInstrFromMaps(*MI); | |||
| 766 | MI->eraseFromParent(); | |||
| 767 | ||||
| 768 | if (IsLoad) { | |||
| 769 | ++NumReloadsRemoved; | |||
| 770 | --NumReloads; | |||
| 771 | } else { | |||
| 772 | ++NumSpillsRemoved; | |||
| 773 | --NumSpills; | |||
| 774 | } | |||
| 775 | ||||
| 776 | return true; | |||
| 777 | } | |||
| 778 | ||||
| 779 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
| 780 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | |||
| 781 | // Dump the range of instructions from B to E with their slot indexes. | |||
| 782 | static void dumpMachineInstrRangeWithSlotIndex(MachineBasicBlock::iterator B, | |||
| 783 | MachineBasicBlock::iterator E, | |||
| 784 | LiveIntervals const &LIS, | |||
| 785 | const char *const header, | |||
| 786 | Register VReg = Register()) { | |||
| 787 | char NextLine = '\n'; | |||
| 788 | char SlotIndent = '\t'; | |||
| 789 | ||||
| 790 | if (std::next(B) == E) { | |||
| 791 | NextLine = ' '; | |||
| 792 | SlotIndent = ' '; | |||
| 793 | } | |||
| 794 | ||||
| 795 | dbgs() << '\t' << header << ": " << NextLine; | |||
| 796 | ||||
| 797 | for (MachineBasicBlock::iterator I = B; I != E; ++I) { | |||
| 798 | SlotIndex Idx = LIS.getInstructionIndex(*I).getRegSlot(); | |||
| 799 | ||||
| 800 | // If a register was passed in and this instruction has it as a | |||
| 801 | // destination that is marked as an early clobber, print the | |||
| 802 | // early-clobber slot index. | |||
| 803 | if (VReg) { | |||
| 804 | MachineOperand *MO = I->findRegisterDefOperand(VReg); | |||
| 805 | if (MO && MO->isEarlyClobber()) | |||
| 806 | Idx = Idx.getRegSlot(true); | |||
| 807 | } | |||
| 808 | ||||
| 809 | dbgs() << SlotIndent << Idx << '\t' << *I; | |||
| 810 | } | |||
| 811 | } | |||
| 812 | #endif | |||
| 813 | ||||
| 814 | /// foldMemoryOperand - Try folding stack slot references in Ops into their | |||
| 815 | /// instructions. | |||
| 816 | /// | |||
| 817 | /// @param Ops Operand indices from AnalyzeVirtRegInBundle(). | |||
| 818 | /// @param LoadMI Load instruction to use instead of stack slot when non-null. | |||
| 819 | /// @return True on success. | |||
| 820 | bool InlineSpiller:: | |||
| 821 | foldMemoryOperand(ArrayRef<std::pair<MachineInstr *, unsigned>> Ops, | |||
| 822 | MachineInstr *LoadMI) { | |||
| 823 | if (Ops.empty()) | |||
| 824 | return false; | |||
| 825 | // Don't attempt folding in bundles. | |||
| 826 | MachineInstr *MI = Ops.front().first; | |||
| 827 | if (Ops.back().first != MI || MI->isBundled()) | |||
| 828 | return false; | |||
| 829 | ||||
| 830 | bool WasCopy = MI->isCopy(); | |||
| 831 | Register ImpReg; | |||
| 832 | ||||
| 833 | // TII::foldMemoryOperand will do what we need here for statepoint | |||
| 834 | // (fold load into use and remove corresponding def). We will replace | |||
| 835 | // uses of removed def with loads (spillAroundUses). | |||
| 836 | // For that to work we need to untie def and use to pass it through | |||
| 837 | // foldMemoryOperand and signal foldPatchpoint that it is allowed to | |||
| 838 | // fold them. | |||
| 839 | bool UntieRegs = MI->getOpcode() == TargetOpcode::STATEPOINT; | |||
| 840 | ||||
| 841 | // Spill subregs if the target allows it. | |||
| 842 | // We always want to spill subregs for stackmap/patchpoint pseudos. | |||
| 843 | bool SpillSubRegs = TII.isSubregFoldable() || | |||
| 844 | MI->getOpcode() == TargetOpcode::STATEPOINT || | |||
| 845 | MI->getOpcode() == TargetOpcode::PATCHPOINT || | |||
| 846 | MI->getOpcode() == TargetOpcode::STACKMAP; | |||
| 847 | ||||
| 848 | // TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied | |||
| 849 | // operands. | |||
| 850 | SmallVector<unsigned, 8> FoldOps; | |||
| 851 | for (const auto &OpPair : Ops) { | |||
| 852 | unsigned Idx = OpPair.second; | |||
| 853 | assert(MI == OpPair.first && "Instruction conflict during operand folding")((MI == OpPair.first && "Instruction conflict during operand folding" ) ? static_cast<void> (0) : __assert_fail ("MI == OpPair.first && \"Instruction conflict during operand folding\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 853, __PRETTY_FUNCTION__)); | |||
| 854 | MachineOperand &MO = MI->getOperand(Idx); | |||
| 855 | if (MO.isImplicit()) { | |||
| 856 | ImpReg = MO.getReg(); | |||
| 857 | continue; | |||
| 858 | } | |||
| 859 | ||||
| 860 | if (!SpillSubRegs && MO.getSubReg()) | |||
| 861 | return false; | |||
| 862 | // We cannot fold a load instruction into a def. | |||
| 863 | if (LoadMI && MO.isDef()) | |||
| 864 | return false; | |||
| 865 | // Tied use operands should not be passed to foldMemoryOperand. | |||
| 866 | if (UntieRegs || !MI->isRegTiedToDefOperand(Idx)) | |||
| 867 | FoldOps.push_back(Idx); | |||
| 868 | } | |||
| 869 | ||||
| 870 | // If we only have implicit uses, we won't be able to fold that. | |||
| 871 | // Moreover, TargetInstrInfo::foldMemoryOperand will assert if we try! | |||
| 872 | if (FoldOps.empty()) | |||
| 873 | return false; | |||
| 874 | ||||
| 875 | MachineInstrSpan MIS(MI, MI->getParent()); | |||
| 876 | ||||
| 877 | SmallVector<std::pair<unsigned, unsigned> > TiedOps; | |||
| 878 | if (UntieRegs) | |||
| 879 | for (unsigned Idx : FoldOps) { | |||
| 880 | MachineOperand &MO = MI->getOperand(Idx); | |||
| 881 | if (!MO.isTied()) | |||
| 882 | continue; | |||
| 883 | unsigned Tied = MI->findTiedOperandIdx(Idx); | |||
| 884 | if (MO.isUse()) | |||
| 885 | TiedOps.emplace_back(Tied, Idx); | |||
| 886 | else { | |||
| 887 | assert(MO.isDef() && "Tied to not use and def?")((MO.isDef() && "Tied to not use and def?") ? static_cast <void> (0) : __assert_fail ("MO.isDef() && \"Tied to not use and def?\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 887, __PRETTY_FUNCTION__)); | |||
| 888 | TiedOps.emplace_back(Idx, Tied); | |||
| 889 | } | |||
| 890 | MI->untieRegOperand(Idx); | |||
| 891 | } | |||
| 892 | ||||
| 893 | MachineInstr *FoldMI = | |||
| 894 | LoadMI ? TII.foldMemoryOperand(*MI, FoldOps, *LoadMI, &LIS) | |||
| 895 | : TII.foldMemoryOperand(*MI, FoldOps, StackSlot, &LIS, &VRM); | |||
| 896 | if (!FoldMI) { | |||
| 897 | // Re-tie operands. | |||
| 898 | for (auto Tied : TiedOps) | |||
| 899 | MI->tieOperands(Tied.first, Tied.second); | |||
| 900 | return false; | |||
| 901 | } | |||
| 902 | ||||
| 903 | // Remove LIS for any dead defs in the original MI not in FoldMI. | |||
| 904 | for (MIBundleOperands MO(*MI); MO.isValid(); ++MO) { | |||
| 905 | if (!MO->isReg()) | |||
| 906 | continue; | |||
| 907 | Register Reg = MO->getReg(); | |||
| 908 | if (!Reg || Register::isVirtualRegister(Reg) || MRI.isReserved(Reg)) { | |||
| 909 | continue; | |||
| 910 | } | |||
| 911 | // Skip non-Defs, including undef uses and internal reads. | |||
| 912 | if (MO->isUse()) | |||
| 913 | continue; | |||
| 914 | PhysRegInfo RI = AnalyzePhysRegInBundle(*FoldMI, Reg, &TRI); | |||
| 915 | if (RI.FullyDefined) | |||
| 916 | continue; | |||
| 917 | // FoldMI does not define this physreg. Remove the LI segment. | |||
| 918 | assert(MO->isDead() && "Cannot fold physreg def")((MO->isDead() && "Cannot fold physreg def") ? static_cast <void> (0) : __assert_fail ("MO->isDead() && \"Cannot fold physreg def\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 918, __PRETTY_FUNCTION__)); | |||
| 919 | SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot(); | |||
| 920 | LIS.removePhysRegDefAt(Reg.asMCReg(), Idx); | |||
| 921 | } | |||
| 922 | ||||
| 923 | int FI; | |||
| 924 | if (TII.isStoreToStackSlot(*MI, FI) && | |||
| 925 | HSpiller.rmFromMergeableSpills(*MI, FI)) | |||
| 926 | --NumSpills; | |||
| 927 | LIS.ReplaceMachineInstrInMaps(*MI, *FoldMI); | |||
| 928 | // Update the call site info. | |||
| 929 | if (MI->isCandidateForCallSiteEntry()) | |||
| 930 | MI->getMF()->moveCallSiteInfo(MI, FoldMI); | |||
| 931 | MI->eraseFromParent(); | |||
| 932 | ||||
| 933 | // Insert any new instructions other than FoldMI into the LIS maps. | |||
| 934 | assert(!MIS.empty() && "Unexpected empty span of instructions!")((!MIS.empty() && "Unexpected empty span of instructions!" ) ? static_cast<void> (0) : __assert_fail ("!MIS.empty() && \"Unexpected empty span of instructions!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 934, __PRETTY_FUNCTION__)); | |||
| 935 | for (MachineInstr &MI : MIS) | |||
| 936 | if (&MI != FoldMI) | |||
| 937 | LIS.InsertMachineInstrInMaps(MI); | |||
| 938 | ||||
| 939 | // TII.foldMemoryOperand may have left some implicit operands on the | |||
| 940 | // instruction. Strip them. | |||
| 941 | if (ImpReg) | |||
| 942 | for (unsigned i = FoldMI->getNumOperands(); i; --i) { | |||
| 943 | MachineOperand &MO = FoldMI->getOperand(i - 1); | |||
| 944 | if (!MO.isReg() || !MO.isImplicit()) | |||
| 945 | break; | |||
| 946 | if (MO.getReg() == ImpReg) | |||
| 947 | FoldMI->RemoveOperand(i - 1); | |||
| 948 | } | |||
| 949 | ||||
| 950 | LLVM_DEBUG(dumpMachineInstrRangeWithSlotIndex(MIS.begin(), MIS.end(), LIS,do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MIS.end(), LIS, "folded"); } } while (false) | |||
| 951 | "folded"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MIS.end(), LIS, "folded"); } } while (false); | |||
| 952 | ||||
| 953 | if (!WasCopy) | |||
| 954 | ++NumFolded; | |||
| 955 | else if (Ops.front().second == 0) { | |||
| 956 | ++NumSpills; | |||
| 957 | // If there is only 1 store instruction is required for spill, add it | |||
| 958 | // to mergeable list. In X86 AMX, 2 intructions are required to store. | |||
| 959 | // We disable the merge for this case. | |||
| 960 | if (std::distance(MIS.begin(), MIS.end()) <= 1) | |||
| 961 | HSpiller.addToMergeableSpills(*FoldMI, StackSlot, Original); | |||
| 962 | } else | |||
| 963 | ++NumReloads; | |||
| 964 | return true; | |||
| 965 | } | |||
| 966 | ||||
| 967 | void InlineSpiller::insertReload(Register NewVReg, | |||
| 968 | SlotIndex Idx, | |||
| 969 | MachineBasicBlock::iterator MI) { | |||
| 970 | MachineBasicBlock &MBB = *MI->getParent(); | |||
| 971 | ||||
| 972 | MachineInstrSpan MIS(MI, &MBB); | |||
| 973 | TII.loadRegFromStackSlot(MBB, MI, NewVReg, StackSlot, | |||
| 974 | MRI.getRegClass(NewVReg), &TRI); | |||
| 975 | ||||
| 976 | LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MI); | |||
| 977 | ||||
| 978 | LLVM_DEBUG(dumpMachineInstrRangeWithSlotIndex(MIS.begin(), MI, LIS, "reload",do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MI, LIS, "reload", NewVReg); } } while (false) | |||
| 979 | NewVReg))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(MIS.begin( ), MI, LIS, "reload", NewVReg); } } while (false); | |||
| 980 | ++NumReloads; | |||
| 981 | } | |||
| 982 | ||||
| 983 | /// Check if \p Def fully defines a VReg with an undefined value. | |||
| 984 | /// If that's the case, that means the value of VReg is actually | |||
| 985 | /// not relevant. | |||
| 986 | static bool isRealSpill(const MachineInstr &Def) { | |||
| 987 | if (!Def.isImplicitDef()) | |||
| 988 | return true; | |||
| 989 | assert(Def.getNumOperands() == 1 &&((Def.getNumOperands() == 1 && "Implicit def with more than one definition" ) ? static_cast<void> (0) : __assert_fail ("Def.getNumOperands() == 1 && \"Implicit def with more than one definition\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 990, __PRETTY_FUNCTION__)) | |||
| 990 | "Implicit def with more than one definition")((Def.getNumOperands() == 1 && "Implicit def with more than one definition" ) ? static_cast<void> (0) : __assert_fail ("Def.getNumOperands() == 1 && \"Implicit def with more than one definition\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 990, __PRETTY_FUNCTION__)); | |||
| 991 | // We can say that the VReg defined by Def is undef, only if it is | |||
| 992 | // fully defined by Def. Otherwise, some of the lanes may not be | |||
| 993 | // undef and the value of the VReg matters. | |||
| 994 | return Def.getOperand(0).getSubReg(); | |||
| 995 | } | |||
| 996 | ||||
| 997 | /// insertSpill - Insert a spill of NewVReg after MI. | |||
| 998 | void InlineSpiller::insertSpill(Register NewVReg, bool isKill, | |||
| 999 | MachineBasicBlock::iterator MI) { | |||
| 1000 | // Spill are not terminators, so inserting spills after terminators will | |||
| 1001 | // violate invariants in MachineVerifier. | |||
| 1002 | assert(!MI->isTerminator() && "Inserting a spill after a terminator")((!MI->isTerminator() && "Inserting a spill after a terminator" ) ? static_cast<void> (0) : __assert_fail ("!MI->isTerminator() && \"Inserting a spill after a terminator\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1002, __PRETTY_FUNCTION__)); | |||
| 1003 | MachineBasicBlock &MBB = *MI->getParent(); | |||
| 1004 | ||||
| 1005 | MachineInstrSpan MIS(MI, &MBB); | |||
| 1006 | MachineBasicBlock::iterator SpillBefore = std::next(MI); | |||
| 1007 | bool IsRealSpill = isRealSpill(*MI); | |||
| 1008 | ||||
| 1009 | if (IsRealSpill) | |||
| 1010 | TII.storeRegToStackSlot(MBB, SpillBefore, NewVReg, isKill, StackSlot, | |||
| 1011 | MRI.getRegClass(NewVReg), &TRI); | |||
| 1012 | else | |||
| 1013 | // Don't spill undef value. | |||
| 1014 | // Anything works for undef, in particular keeping the memory | |||
| 1015 | // uninitialized is a viable option and it saves code size and | |||
| 1016 | // run time. | |||
| 1017 | BuildMI(MBB, SpillBefore, MI->getDebugLoc(), TII.get(TargetOpcode::KILL)) | |||
| 1018 | .addReg(NewVReg, getKillRegState(isKill)); | |||
| 1019 | ||||
| 1020 | MachineBasicBlock::iterator Spill = std::next(MI); | |||
| 1021 | LIS.InsertMachineInstrRangeInMaps(Spill, MIS.end()); | |||
| 1022 | for (const MachineInstr &MI : make_range(Spill, MIS.end())) | |||
| 1023 | getVDefInterval(MI, LIS); | |||
| 1024 | ||||
| 1025 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(Spill, MIS .end(), LIS, "spill"); } } while (false) | |||
| 1026 | dumpMachineInstrRangeWithSlotIndex(Spill, MIS.end(), LIS, "spill"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dumpMachineInstrRangeWithSlotIndex(Spill, MIS .end(), LIS, "spill"); } } while (false); | |||
| 1027 | ++NumSpills; | |||
| 1028 | // If there is only 1 store instruction is required for spill, add it | |||
| 1029 | // to mergeable list. In X86 AMX, 2 intructions are required to store. | |||
| 1030 | // We disable the merge for this case. | |||
| 1031 | if (IsRealSpill && std::distance(Spill, MIS.end()) <= 1) | |||
| 1032 | HSpiller.addToMergeableSpills(*Spill, StackSlot, Original); | |||
| 1033 | } | |||
| 1034 | ||||
| 1035 | /// spillAroundUses - insert spill code around each use of Reg. | |||
| 1036 | void InlineSpiller::spillAroundUses(Register Reg) { | |||
| 1037 | LLVM_DEBUG(dbgs() << "spillAroundUses " << printReg(Reg) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "spillAroundUses " << printReg (Reg) << '\n'; } } while (false); | |||
| 1038 | LiveInterval &OldLI = LIS.getInterval(Reg); | |||
| 1039 | ||||
| 1040 | // Iterate over instructions using Reg. | |||
| 1041 | for (MachineRegisterInfo::reg_bundle_iterator | |||
| 1042 | RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end(); | |||
| 1043 | RegI != E; ) { | |||
| 1044 | MachineInstr *MI = &*(RegI++); | |||
| 1045 | ||||
| 1046 | // Debug values are not allowed to affect codegen. | |||
| 1047 | if (MI->isDebugValue()) { | |||
| 1048 | // Modify DBG_VALUE now that the value is in a spill slot. | |||
| 1049 | MachineBasicBlock *MBB = MI->getParent(); | |||
| 1050 | LLVM_DEBUG(dbgs() << "Modifying debug info due to spill:\t" << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Modifying debug info due to spill:\t" << *MI; } } while (false); | |||
| 1051 | buildDbgValueForSpill(*MBB, MI, *MI, StackSlot, Reg); | |||
| 1052 | MBB->erase(MI); | |||
| 1053 | continue; | |||
| 1054 | } | |||
| 1055 | ||||
| 1056 | assert(!MI->isDebugInstr() && "Did not expect to find a use in debug "((!MI->isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? static_cast<void> (0) : __assert_fail ("!MI->isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1057, __PRETTY_FUNCTION__)) | |||
| 1057 | "instruction that isn't a DBG_VALUE")((!MI->isDebugInstr() && "Did not expect to find a use in debug " "instruction that isn't a DBG_VALUE") ? static_cast<void> (0) : __assert_fail ("!MI->isDebugInstr() && \"Did not expect to find a use in debug \" \"instruction that isn't a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1057, __PRETTY_FUNCTION__)); | |||
| 1058 | ||||
| 1059 | // Ignore copies to/from snippets. We'll delete them. | |||
| 1060 | if (SnippetCopies.count(MI)) | |||
| 1061 | continue; | |||
| 1062 | ||||
| 1063 | // Stack slot accesses may coalesce away. | |||
| 1064 | if (coalesceStackAccess(MI, Reg)) | |||
| 1065 | continue; | |||
| 1066 | ||||
| 1067 | // Analyze instruction. | |||
| 1068 | SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops; | |||
| 1069 | VirtRegInfo RI = AnalyzeVirtRegInBundle(*MI, Reg, &Ops); | |||
| 1070 | ||||
| 1071 | // Find the slot index where this instruction reads and writes OldLI. | |||
| 1072 | // This is usually the def slot, except for tied early clobbers. | |||
| 1073 | SlotIndex Idx = LIS.getInstructionIndex(*MI).getRegSlot(); | |||
| 1074 | if (VNInfo *VNI = OldLI.getVNInfoAt(Idx.getRegSlot(true))) | |||
| 1075 | if (SlotIndex::isSameInstr(Idx, VNI->def)) | |||
| 1076 | Idx = VNI->def; | |||
| 1077 | ||||
| 1078 | // Check for a sibling copy. | |||
| 1079 | Register SibReg = isFullCopyOf(*MI, Reg); | |||
| 1080 | if (SibReg && isSibling(SibReg)) { | |||
| 1081 | // This may actually be a copy between snippets. | |||
| 1082 | if (isRegToSpill(SibReg)) { | |||
| 1083 | LLVM_DEBUG(dbgs() << "Found new snippet copy: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Found new snippet copy: " << *MI; } } while (false); | |||
| 1084 | SnippetCopies.insert(MI); | |||
| 1085 | continue; | |||
| 1086 | } | |||
| 1087 | if (RI.Writes) { | |||
| 1088 | if (hoistSpillInsideBB(OldLI, *MI)) { | |||
| 1089 | // This COPY is now dead, the value is already in the stack slot. | |||
| 1090 | MI->getOperand(0).setIsDead(); | |||
| 1091 | DeadDefs.push_back(MI); | |||
| 1092 | continue; | |||
| 1093 | } | |||
| 1094 | } else { | |||
| 1095 | // This is a reload for a sib-reg copy. Drop spills downstream. | |||
| 1096 | LiveInterval &SibLI = LIS.getInterval(SibReg); | |||
| 1097 | eliminateRedundantSpills(SibLI, SibLI.getVNInfoAt(Idx)); | |||
| 1098 | // The COPY will fold to a reload below. | |||
| 1099 | } | |||
| 1100 | } | |||
| 1101 | ||||
| 1102 | // Attempt to fold memory ops. | |||
| 1103 | if (foldMemoryOperand(Ops)) | |||
| 1104 | continue; | |||
| 1105 | ||||
| 1106 | // Create a new virtual register for spill/fill. | |||
| 1107 | // FIXME: Infer regclass from instruction alone. | |||
| 1108 | Register NewVReg = Edit->createFrom(Reg); | |||
| 1109 | ||||
| 1110 | if (RI.Reads) | |||
| 1111 | insertReload(NewVReg, Idx, MI); | |||
| 1112 | ||||
| 1113 | // Rewrite instruction operands. | |||
| 1114 | bool hasLiveDef = false; | |||
| 1115 | for (const auto &OpPair : Ops) { | |||
| 1116 | MachineOperand &MO = OpPair.first->getOperand(OpPair.second); | |||
| 1117 | MO.setReg(NewVReg); | |||
| 1118 | if (MO.isUse()) { | |||
| 1119 | if (!OpPair.first->isRegTiedToDefOperand(OpPair.second)) | |||
| 1120 | MO.setIsKill(); | |||
| 1121 | } else { | |||
| 1122 | if (!MO.isDead()) | |||
| 1123 | hasLiveDef = true; | |||
| 1124 | } | |||
| 1125 | } | |||
| 1126 | LLVM_DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\trewrite: " << Idx << '\t' << *MI << '\n'; } } while (false); | |||
| 1127 | ||||
| 1128 | // FIXME: Use a second vreg if instruction has no tied ops. | |||
| 1129 | if (RI.Writes) | |||
| 1130 | if (hasLiveDef) | |||
| 1131 | insertSpill(NewVReg, true, MI); | |||
| 1132 | } | |||
| 1133 | } | |||
| 1134 | ||||
| 1135 | /// spillAll - Spill all registers remaining after rematerialization. | |||
| 1136 | void InlineSpiller::spillAll() { | |||
| 1137 | // Update LiveStacks now that we are committed to spilling. | |||
| 1138 | if (StackSlot == VirtRegMap::NO_STACK_SLOT) { | |||
| 1139 | StackSlot = VRM.assignVirt2StackSlot(Original); | |||
| 1140 | StackInt = &LSS.getOrCreateInterval(StackSlot, MRI.getRegClass(Original)); | |||
| 1141 | StackInt->getNextValue(SlotIndex(), LSS.getVNInfoAllocator()); | |||
| 1142 | } else | |||
| 1143 | StackInt = &LSS.getInterval(StackSlot); | |||
| 1144 | ||||
| 1145 | if (Original != Edit->getReg()) | |||
| 1146 | VRM.assignVirt2StackSlot(Edit->getReg(), StackSlot); | |||
| 1147 | ||||
| 1148 | assert(StackInt->getNumValNums() == 1 && "Bad stack interval values")((StackInt->getNumValNums() == 1 && "Bad stack interval values" ) ? static_cast<void> (0) : __assert_fail ("StackInt->getNumValNums() == 1 && \"Bad stack interval values\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1148, __PRETTY_FUNCTION__)); | |||
| 1149 | for (Register Reg : RegsToSpill) | |||
| 1150 | StackInt->MergeSegmentsInAsValue(LIS.getInterval(Reg), | |||
| 1151 | StackInt->getValNumInfo(0)); | |||
| 1152 | LLVM_DEBUG(dbgs() << "Merged spilled regs: " << *StackInt << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Merged spilled regs: " << *StackInt << '\n'; } } while (false); | |||
| 1153 | ||||
| 1154 | // Spill around uses of all RegsToSpill. | |||
| 1155 | for (Register Reg : RegsToSpill) | |||
| 1156 | spillAroundUses(Reg); | |||
| 1157 | ||||
| 1158 | // Hoisted spills may cause dead code. | |||
| 1159 | if (!DeadDefs.empty()) { | |||
| 1160 | LLVM_DEBUG(dbgs() << "Eliminating " << DeadDefs.size() << " dead defs\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Eliminating " << DeadDefs .size() << " dead defs\n"; } } while (false); | |||
| 1161 | Edit->eliminateDeadDefs(DeadDefs, RegsToSpill, AA); | |||
| 1162 | } | |||
| 1163 | ||||
| 1164 | // Finally delete the SnippetCopies. | |||
| 1165 | for (Register Reg : RegsToSpill) { | |||
| 1166 | for (MachineRegisterInfo::reg_instr_iterator | |||
| 1167 | RI = MRI.reg_instr_begin(Reg), E = MRI.reg_instr_end(); | |||
| 1168 | RI != E; ) { | |||
| 1169 | MachineInstr &MI = *(RI++); | |||
| 1170 | assert(SnippetCopies.count(&MI) && "Remaining use wasn't a snippet copy")((SnippetCopies.count(&MI) && "Remaining use wasn't a snippet copy" ) ? static_cast<void> (0) : __assert_fail ("SnippetCopies.count(&MI) && \"Remaining use wasn't a snippet copy\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1170, __PRETTY_FUNCTION__)); | |||
| 1171 | // FIXME: Do this with a LiveRangeEdit callback. | |||
| 1172 | LIS.RemoveMachineInstrFromMaps(MI); | |||
| 1173 | MI.eraseFromParent(); | |||
| 1174 | } | |||
| 1175 | } | |||
| 1176 | ||||
| 1177 | // Delete all spilled registers. | |||
| 1178 | for (Register Reg : RegsToSpill) | |||
| 1179 | Edit->eraseVirtReg(Reg); | |||
| 1180 | } | |||
| 1181 | ||||
| 1182 | void InlineSpiller::spill(LiveRangeEdit &edit) { | |||
| 1183 | ++NumSpilledRanges; | |||
| 1184 | Edit = &edit; | |||
| 1185 | assert(!Register::isStackSlot(edit.getReg()) &&((!Register::isStackSlot(edit.getReg()) && "Trying to spill a stack slot." ) ? static_cast<void> (0) : __assert_fail ("!Register::isStackSlot(edit.getReg()) && \"Trying to spill a stack slot.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1186, __PRETTY_FUNCTION__)) | |||
| 1186 | "Trying to spill a stack slot.")((!Register::isStackSlot(edit.getReg()) && "Trying to spill a stack slot." ) ? static_cast<void> (0) : __assert_fail ("!Register::isStackSlot(edit.getReg()) && \"Trying to spill a stack slot.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1186, __PRETTY_FUNCTION__)); | |||
| 1187 | // Share a stack slot among all descendants of Original. | |||
| 1188 | Original = VRM.getOriginal(edit.getReg()); | |||
| 1189 | StackSlot = VRM.getStackSlot(Original); | |||
| 1190 | StackInt = nullptr; | |||
| 1191 | ||||
| 1192 | LLVM_DEBUG(dbgs() << "Inline spilling "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false) | |||
| 1193 | << TRI.getRegClassName(MRI.getRegClass(edit.getReg()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false) | |||
| 1194 | << ':' << edit.getParent() << "\nFrom original "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false) | |||
| 1195 | << printReg(Original) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Inline spilling " << TRI .getRegClassName(MRI.getRegClass(edit.getReg())) << ':' << edit.getParent() << "\nFrom original " << printReg(Original) << '\n'; } } while (false); | |||
| 1196 | assert(edit.getParent().isSpillable() &&((edit.getParent().isSpillable() && "Attempting to spill already spilled value." ) ? static_cast<void> (0) : __assert_fail ("edit.getParent().isSpillable() && \"Attempting to spill already spilled value.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1197, __PRETTY_FUNCTION__)) | |||
| 1197 | "Attempting to spill already spilled value.")((edit.getParent().isSpillable() && "Attempting to spill already spilled value." ) ? static_cast<void> (0) : __assert_fail ("edit.getParent().isSpillable() && \"Attempting to spill already spilled value.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1197, __PRETTY_FUNCTION__)); | |||
| 1198 | assert(DeadDefs.empty() && "Previous spill didn't remove dead defs")((DeadDefs.empty() && "Previous spill didn't remove dead defs" ) ? static_cast<void> (0) : __assert_fail ("DeadDefs.empty() && \"Previous spill didn't remove dead defs\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1198, __PRETTY_FUNCTION__)); | |||
| 1199 | ||||
| 1200 | collectRegsToSpill(); | |||
| 1201 | reMaterializeAll(); | |||
| 1202 | ||||
| 1203 | // Remat may handle everything. | |||
| 1204 | if (!RegsToSpill.empty()) | |||
| 1205 | spillAll(); | |||
| 1206 | ||||
| 1207 | Edit->calculateRegClassAndHint(MF, VRAI); | |||
| 1208 | } | |||
| 1209 | ||||
| 1210 | /// Optimizations after all the reg selections and spills are done. | |||
| 1211 | void InlineSpiller::postOptimization() { HSpiller.hoistAllSpills(); } | |||
| 1212 | ||||
| 1213 | /// When a spill is inserted, add the spill to MergeableSpills map. | |||
| 1214 | void HoistSpillHelper::addToMergeableSpills(MachineInstr &Spill, int StackSlot, | |||
| 1215 | unsigned Original) { | |||
| 1216 | BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator(); | |||
| 1217 | LiveInterval &OrigLI = LIS.getInterval(Original); | |||
| 1218 | // save a copy of LiveInterval in StackSlotToOrigLI because the original | |||
| 1219 | // LiveInterval may be cleared after all its references are spilled. | |||
| 1220 | if (StackSlotToOrigLI.find(StackSlot) == StackSlotToOrigLI.end()) { | |||
| 1221 | auto LI = std::make_unique<LiveInterval>(OrigLI.reg(), OrigLI.weight()); | |||
| 1222 | LI->assign(OrigLI, Allocator); | |||
| 1223 | StackSlotToOrigLI[StackSlot] = std::move(LI); | |||
| 1224 | } | |||
| 1225 | SlotIndex Idx = LIS.getInstructionIndex(Spill); | |||
| 1226 | VNInfo *OrigVNI = StackSlotToOrigLI[StackSlot]->getVNInfoAt(Idx.getRegSlot()); | |||
| 1227 | std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI); | |||
| 1228 | MergeableSpills[MIdx].insert(&Spill); | |||
| 1229 | } | |||
| 1230 | ||||
| 1231 | /// When a spill is removed, remove the spill from MergeableSpills map. | |||
| 1232 | /// Return true if the spill is removed successfully. | |||
| 1233 | bool HoistSpillHelper::rmFromMergeableSpills(MachineInstr &Spill, | |||
| 1234 | int StackSlot) { | |||
| 1235 | auto It = StackSlotToOrigLI.find(StackSlot); | |||
| 1236 | if (It == StackSlotToOrigLI.end()) | |||
| 1237 | return false; | |||
| 1238 | SlotIndex Idx = LIS.getInstructionIndex(Spill); | |||
| 1239 | VNInfo *OrigVNI = It->second->getVNInfoAt(Idx.getRegSlot()); | |||
| 1240 | std::pair<int, VNInfo *> MIdx = std::make_pair(StackSlot, OrigVNI); | |||
| 1241 | return MergeableSpills[MIdx].erase(&Spill); | |||
| 1242 | } | |||
| 1243 | ||||
| 1244 | /// Check BB to see if it is a possible target BB to place a hoisted spill, | |||
| 1245 | /// i.e., there should be a living sibling of OrigReg at the insert point. | |||
| 1246 | bool HoistSpillHelper::isSpillCandBB(LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
| 1247 | MachineBasicBlock &BB, Register &LiveReg) { | |||
| 1248 | SlotIndex Idx; | |||
| 1249 | Register OrigReg = OrigLI.reg(); | |||
| 1250 | MachineBasicBlock::iterator MI = IPA.getLastInsertPointIter(OrigLI, BB); | |||
| 1251 | if (MI != BB.end()) | |||
| 1252 | Idx = LIS.getInstructionIndex(*MI); | |||
| 1253 | else | |||
| 1254 | Idx = LIS.getMBBEndIdx(&BB).getPrevSlot(); | |||
| 1255 | SmallSetVector<Register, 16> &Siblings = Virt2SiblingsMap[OrigReg]; | |||
| 1256 | assert(OrigLI.getVNInfoAt(Idx) == &OrigVNI && "Unexpected VNI")((OrigLI.getVNInfoAt(Idx) == &OrigVNI && "Unexpected VNI" ) ? static_cast<void> (0) : __assert_fail ("OrigLI.getVNInfoAt(Idx) == &OrigVNI && \"Unexpected VNI\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1256, __PRETTY_FUNCTION__)); | |||
| 1257 | ||||
| 1258 | for (const Register &SibReg : Siblings) { | |||
| 1259 | LiveInterval &LI = LIS.getInterval(SibReg); | |||
| 1260 | VNInfo *VNI = LI.getVNInfoAt(Idx); | |||
| 1261 | if (VNI) { | |||
| 1262 | LiveReg = SibReg; | |||
| 1263 | return true; | |||
| 1264 | } | |||
| 1265 | } | |||
| 1266 | return false; | |||
| 1267 | } | |||
| 1268 | ||||
| 1269 | /// Remove redundant spills in the same BB. Save those redundant spills in | |||
| 1270 | /// SpillsToRm, and save the spill to keep and its BB in SpillBBToSpill map. | |||
| 1271 | void HoistSpillHelper::rmRedundantSpills( | |||
| 1272 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
| 1273 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
| 1274 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) { | |||
| 1275 | // For each spill saw, check SpillBBToSpill[] and see if its BB already has | |||
| 1276 | // another spill inside. If a BB contains more than one spill, only keep the | |||
| 1277 | // earlier spill with smaller SlotIndex. | |||
| 1278 | for (const auto CurrentSpill : Spills) { | |||
| 1279 | MachineBasicBlock *Block = CurrentSpill->getParent(); | |||
| 1280 | MachineDomTreeNode *Node = MDT.getBase().getNode(Block); | |||
| 1281 | MachineInstr *PrevSpill = SpillBBToSpill[Node]; | |||
| 1282 | if (PrevSpill) { | |||
| 1283 | SlotIndex PIdx = LIS.getInstructionIndex(*PrevSpill); | |||
| 1284 | SlotIndex CIdx = LIS.getInstructionIndex(*CurrentSpill); | |||
| 1285 | MachineInstr *SpillToRm = (CIdx > PIdx) ? CurrentSpill : PrevSpill; | |||
| 1286 | MachineInstr *SpillToKeep = (CIdx > PIdx) ? PrevSpill : CurrentSpill; | |||
| 1287 | SpillsToRm.push_back(SpillToRm); | |||
| 1288 | SpillBBToSpill[MDT.getBase().getNode(Block)] = SpillToKeep; | |||
| 1289 | } else { | |||
| 1290 | SpillBBToSpill[MDT.getBase().getNode(Block)] = CurrentSpill; | |||
| 1291 | } | |||
| 1292 | } | |||
| 1293 | for (const auto SpillToRm : SpillsToRm) | |||
| 1294 | Spills.erase(SpillToRm); | |||
| 1295 | } | |||
| 1296 | ||||
| 1297 | /// Starting from \p Root find a top-down traversal order of the dominator | |||
| 1298 | /// tree to visit all basic blocks containing the elements of \p Spills. | |||
| 1299 | /// Redundant spills will be found and put into \p SpillsToRm at the same | |||
| 1300 | /// time. \p SpillBBToSpill will be populated as part of the process and | |||
| 1301 | /// maps a basic block to the first store occurring in the basic block. | |||
| 1302 | /// \post SpillsToRm.union(Spills\@post) == Spills\@pre | |||
| 1303 | void HoistSpillHelper::getVisitOrders( | |||
| 1304 | MachineBasicBlock *Root, SmallPtrSet<MachineInstr *, 16> &Spills, | |||
| 1305 | SmallVectorImpl<MachineDomTreeNode *> &Orders, | |||
| 1306 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
| 1307 | DenseMap<MachineDomTreeNode *, unsigned> &SpillsToKeep, | |||
| 1308 | DenseMap<MachineDomTreeNode *, MachineInstr *> &SpillBBToSpill) { | |||
| 1309 | // The set contains all the possible BB nodes to which we may hoist | |||
| 1310 | // original spills. | |||
| 1311 | SmallPtrSet<MachineDomTreeNode *, 8> WorkSet; | |||
| 1312 | // Save the BB nodes on the path from the first BB node containing | |||
| 1313 | // non-redundant spill to the Root node. | |||
| 1314 | SmallPtrSet<MachineDomTreeNode *, 8> NodesOnPath; | |||
| 1315 | // All the spills to be hoisted must originate from a single def instruction | |||
| 1316 | // to the OrigReg. It means the def instruction should dominate all the spills | |||
| 1317 | // to be hoisted. We choose the BB where the def instruction is located as | |||
| 1318 | // the Root. | |||
| 1319 | MachineDomTreeNode *RootIDomNode = MDT[Root]->getIDom(); | |||
| 1320 | // For every node on the dominator tree with spill, walk up on the dominator | |||
| 1321 | // tree towards the Root node until it is reached. If there is other node | |||
| 1322 | // containing spill in the middle of the path, the previous spill saw will | |||
| 1323 | // be redundant and the node containing it will be removed. All the nodes on | |||
| 1324 | // the path starting from the first node with non-redundant spill to the Root | |||
| 1325 | // node will be added to the WorkSet, which will contain all the possible | |||
| 1326 | // locations where spills may be hoisted to after the loop below is done. | |||
| 1327 | for (const auto Spill : Spills) { | |||
| 1328 | MachineBasicBlock *Block = Spill->getParent(); | |||
| 1329 | MachineDomTreeNode *Node = MDT[Block]; | |||
| 1330 | MachineInstr *SpillToRm = nullptr; | |||
| 1331 | while (Node != RootIDomNode) { | |||
| 1332 | // If Node dominates Block, and it already contains a spill, the spill in | |||
| 1333 | // Block will be redundant. | |||
| 1334 | if (Node != MDT[Block] && SpillBBToSpill[Node]) { | |||
| 1335 | SpillToRm = SpillBBToSpill[MDT[Block]]; | |||
| 1336 | break; | |||
| 1337 | /// If we see the Node already in WorkSet, the path from the Node to | |||
| 1338 | /// the Root node must already be traversed by another spill. | |||
| 1339 | /// Then no need to repeat. | |||
| 1340 | } else if (WorkSet.count(Node)) { | |||
| 1341 | break; | |||
| 1342 | } else { | |||
| 1343 | NodesOnPath.insert(Node); | |||
| 1344 | } | |||
| 1345 | Node = Node->getIDom(); | |||
| 1346 | } | |||
| 1347 | if (SpillToRm) { | |||
| 1348 | SpillsToRm.push_back(SpillToRm); | |||
| 1349 | } else { | |||
| 1350 | // Add a BB containing the original spills to SpillsToKeep -- i.e., | |||
| 1351 | // set the initial status before hoisting start. The value of BBs | |||
| 1352 | // containing original spills is set to 0, in order to descriminate | |||
| 1353 | // with BBs containing hoisted spills which will be inserted to | |||
| 1354 | // SpillsToKeep later during hoisting. | |||
| 1355 | SpillsToKeep[MDT[Block]] = 0; | |||
| 1356 | WorkSet.insert(NodesOnPath.begin(), NodesOnPath.end()); | |||
| 1357 | } | |||
| 1358 | NodesOnPath.clear(); | |||
| 1359 | } | |||
| 1360 | ||||
| 1361 | // Sort the nodes in WorkSet in top-down order and save the nodes | |||
| 1362 | // in Orders. Orders will be used for hoisting in runHoistSpills. | |||
| 1363 | unsigned idx = 0; | |||
| 1364 | Orders.push_back(MDT.getBase().getNode(Root)); | |||
| 1365 | do { | |||
| 1366 | MachineDomTreeNode *Node = Orders[idx++]; | |||
| 1367 | for (MachineDomTreeNode *Child : Node->children()) { | |||
| 1368 | if (WorkSet.count(Child)) | |||
| 1369 | Orders.push_back(Child); | |||
| 1370 | } | |||
| 1371 | } while (idx != Orders.size()); | |||
| 1372 | assert(Orders.size() == WorkSet.size() &&((Orders.size() == WorkSet.size() && "Orders have different size with WorkSet" ) ? static_cast<void> (0) : __assert_fail ("Orders.size() == WorkSet.size() && \"Orders have different size with WorkSet\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1373, __PRETTY_FUNCTION__)) | |||
| 1373 | "Orders have different size with WorkSet")((Orders.size() == WorkSet.size() && "Orders have different size with WorkSet" ) ? static_cast<void> (0) : __assert_fail ("Orders.size() == WorkSet.size() && \"Orders have different size with WorkSet\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1373, __PRETTY_FUNCTION__)); | |||
| 1374 | ||||
| 1375 | #ifndef NDEBUG | |||
| 1376 | LLVM_DEBUG(dbgs() << "Orders size is " << Orders.size() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "Orders size is " << Orders .size() << "\n"; } } while (false); | |||
| 1377 | SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin(); | |||
| 1378 | for (; RIt != Orders.rend(); RIt++) | |||
| 1379 | LLVM_DEBUG(dbgs() << "BB" << (*RIt)->getBlock()->getNumber() << ",")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "BB" << (*RIt)->getBlock ()->getNumber() << ","; } } while (false); | |||
| 1380 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { dbgs() << "\n"; } } while (false); | |||
| 1381 | #endif | |||
| 1382 | } | |||
| 1383 | ||||
| 1384 | /// Try to hoist spills according to BB hotness. The spills to removed will | |||
| 1385 | /// be saved in \p SpillsToRm. The spills to be inserted will be saved in | |||
| 1386 | /// \p SpillsToIns. | |||
| 1387 | void HoistSpillHelper::runHoistSpills( | |||
| 1388 | LiveInterval &OrigLI, VNInfo &OrigVNI, | |||
| 1389 | SmallPtrSet<MachineInstr *, 16> &Spills, | |||
| 1390 | SmallVectorImpl<MachineInstr *> &SpillsToRm, | |||
| 1391 | DenseMap<MachineBasicBlock *, unsigned> &SpillsToIns) { | |||
| 1392 | // Visit order of dominator tree nodes. | |||
| 1393 | SmallVector<MachineDomTreeNode *, 32> Orders; | |||
| 1394 | // SpillsToKeep contains all the nodes where spills are to be inserted | |||
| 1395 | // during hoisting. If the spill to be inserted is an original spill | |||
| 1396 | // (not a hoisted one), the value of the map entry is 0. If the spill | |||
| 1397 | // is a hoisted spill, the value of the map entry is the VReg to be used | |||
| 1398 | // as the source of the spill. | |||
| 1399 | DenseMap<MachineDomTreeNode *, unsigned> SpillsToKeep; | |||
| 1400 | // Map from BB to the first spill inside of it. | |||
| 1401 | DenseMap<MachineDomTreeNode *, MachineInstr *> SpillBBToSpill; | |||
| 1402 | ||||
| 1403 | rmRedundantSpills(Spills, SpillsToRm, SpillBBToSpill); | |||
| 1404 | ||||
| 1405 | MachineBasicBlock *Root = LIS.getMBBFromIndex(OrigVNI.def); | |||
| 1406 | getVisitOrders(Root, Spills, Orders, SpillsToRm, SpillsToKeep, | |||
| 1407 | SpillBBToSpill); | |||
| 1408 | ||||
| 1409 | // SpillsInSubTreeMap keeps the map from a dom tree node to a pair of | |||
| 1410 | // nodes set and the cost of all the spills inside those nodes. | |||
| 1411 | // The nodes set are the locations where spills are to be inserted | |||
| 1412 | // in the subtree of current node. | |||
| 1413 | using NodesCostPair = | |||
| 1414 | std::pair<SmallPtrSet<MachineDomTreeNode *, 16>, BlockFrequency>; | |||
| 1415 | DenseMap<MachineDomTreeNode *, NodesCostPair> SpillsInSubTreeMap; | |||
| 1416 | ||||
| 1417 | // Iterate Orders set in reverse order, which will be a bottom-up order | |||
| 1418 | // in the dominator tree. Once we visit a dom tree node, we know its | |||
| 1419 | // children have already been visited and the spill locations in the | |||
| 1420 | // subtrees of all the children have been determined. | |||
| 1421 | SmallVector<MachineDomTreeNode *, 32>::reverse_iterator RIt = Orders.rbegin(); | |||
| 1422 | for (; RIt != Orders.rend(); RIt++) { | |||
| 1423 | MachineBasicBlock *Block = (*RIt)->getBlock(); | |||
| 1424 | ||||
| 1425 | // If Block contains an original spill, simply continue. | |||
| 1426 | if (SpillsToKeep.find(*RIt) != SpillsToKeep.end() && !SpillsToKeep[*RIt]) { | |||
| 1427 | SpillsInSubTreeMap[*RIt].first.insert(*RIt); | |||
| 1428 | // SpillsInSubTreeMap[*RIt].second contains the cost of spill. | |||
| 1429 | SpillsInSubTreeMap[*RIt].second = MBFI.getBlockFreq(Block); | |||
| 1430 | continue; | |||
| 1431 | } | |||
| 1432 | ||||
| 1433 | // Collect spills in subtree of current node (*RIt) to | |||
| 1434 | // SpillsInSubTreeMap[*RIt].first. | |||
| 1435 | for (MachineDomTreeNode *Child : (*RIt)->children()) { | |||
| 1436 | if (SpillsInSubTreeMap.find(Child) == SpillsInSubTreeMap.end()) | |||
| 1437 | continue; | |||
| 1438 | // The stmt "SpillsInSubTree = SpillsInSubTreeMap[*RIt].first" below | |||
| 1439 | // should be placed before getting the begin and end iterators of | |||
| 1440 | // SpillsInSubTreeMap[Child].first, or else the iterators may be | |||
| 1441 | // invalidated when SpillsInSubTreeMap[*RIt] is seen the first time | |||
| 1442 | // and the map grows and then the original buckets in the map are moved. | |||
| 1443 | SmallPtrSet<MachineDomTreeNode *, 16> &SpillsInSubTree = | |||
| 1444 | SpillsInSubTreeMap[*RIt].first; | |||
| 1445 | BlockFrequency &SubTreeCost = SpillsInSubTreeMap[*RIt].second; | |||
| 1446 | SubTreeCost += SpillsInSubTreeMap[Child].second; | |||
| 1447 | auto BI = SpillsInSubTreeMap[Child].first.begin(); | |||
| 1448 | auto EI = SpillsInSubTreeMap[Child].first.end(); | |||
| 1449 | SpillsInSubTree.insert(BI, EI); | |||
| 1450 | SpillsInSubTreeMap.erase(Child); | |||
| 1451 | } | |||
| 1452 | ||||
| 1453 | SmallPtrSet<MachineDomTreeNode *, 16> &SpillsInSubTree = | |||
| 1454 | SpillsInSubTreeMap[*RIt].first; | |||
| 1455 | BlockFrequency &SubTreeCost = SpillsInSubTreeMap[*RIt].second; | |||
| 1456 | // No spills in subtree, simply continue. | |||
| 1457 | if (SpillsInSubTree.empty()) | |||
| 1458 | continue; | |||
| 1459 | ||||
| 1460 | // Check whether Block is a possible candidate to insert spill. | |||
| 1461 | Register LiveReg; | |||
| 1462 | if (!isSpillCandBB(OrigLI, OrigVNI, *Block, LiveReg)) | |||
| 1463 | continue; | |||
| 1464 | ||||
| 1465 | // If there are multiple spills that could be merged, bias a little | |||
| 1466 | // to hoist the spill. | |||
| 1467 | BranchProbability MarginProb = (SpillsInSubTree.size() > 1) | |||
| 1468 | ? BranchProbability(9, 10) | |||
| 1469 | : BranchProbability(1, 1); | |||
| 1470 | if (SubTreeCost > MBFI.getBlockFreq(Block) * MarginProb) { | |||
| 1471 | // Hoist: Move spills to current Block. | |||
| 1472 | for (const auto SpillBB : SpillsInSubTree) { | |||
| 1473 | // When SpillBB is a BB contains original spill, insert the spill | |||
| 1474 | // to SpillsToRm. | |||
| 1475 | if (SpillsToKeep.find(SpillBB) != SpillsToKeep.end() && | |||
| 1476 | !SpillsToKeep[SpillBB]) { | |||
| 1477 | MachineInstr *SpillToRm = SpillBBToSpill[SpillBB]; | |||
| 1478 | SpillsToRm.push_back(SpillToRm); | |||
| 1479 | } | |||
| 1480 | // SpillBB will not contain spill anymore, remove it from SpillsToKeep. | |||
| 1481 | SpillsToKeep.erase(SpillBB); | |||
| 1482 | } | |||
| 1483 | // Current Block is the BB containing the new hoisted spill. Add it to | |||
| 1484 | // SpillsToKeep. LiveReg is the source of the new spill. | |||
| 1485 | SpillsToKeep[*RIt] = LiveReg; | |||
| 1486 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
| 1487 | dbgs() << "spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
| 1488 | for (const auto Rspill : SpillsInSubTree)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
| 1489 | dbgs() << Rspill->getBlock()->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
| 1490 | dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
| 1491 | << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false) | |||
| 1492 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "spills in BB: "; for (const auto Rspill : SpillsInSubTree) dbgs() << Rspill->getBlock ()->getNumber() << " "; dbgs() << "were promoted to BB" << (*RIt)->getBlock()->getNumber() << "\n" ; }; } } while (false); | |||
| 1493 | SpillsInSubTree.clear(); | |||
| 1494 | SpillsInSubTree.insert(*RIt); | |||
| 1495 | SubTreeCost = MBFI.getBlockFreq(Block); | |||
| 1496 | } | |||
| 1497 | } | |||
| 1498 | // For spills in SpillsToKeep with LiveReg set (i.e., not original spill), | |||
| 1499 | // save them to SpillsToIns. | |||
| 1500 | for (const auto &Ent : SpillsToKeep) { | |||
| 1501 | if (Ent.second) | |||
| 1502 | SpillsToIns[Ent.first->getBlock()] = Ent.second; | |||
| 1503 | } | |||
| 1504 | } | |||
| 1505 | ||||
| 1506 | /// For spills with equal values, remove redundant spills and hoist those left | |||
| 1507 | /// to less hot spots. | |||
| 1508 | /// | |||
| 1509 | /// Spills with equal values will be collected into the same set in | |||
| 1510 | /// MergeableSpills when spill is inserted. These equal spills are originated | |||
| 1511 | /// from the same defining instruction and are dominated by the instruction. | |||
| 1512 | /// Before hoisting all the equal spills, redundant spills inside in the same | |||
| 1513 | /// BB are first marked to be deleted. Then starting from the spills left, walk | |||
| 1514 | /// up on the dominator tree towards the Root node where the define instruction | |||
| 1515 | /// is located, mark the dominated spills to be deleted along the way and | |||
| 1516 | /// collect the BB nodes on the path from non-dominated spills to the define | |||
| 1517 | /// instruction into a WorkSet. The nodes in WorkSet are the candidate places | |||
| 1518 | /// where we are considering to hoist the spills. We iterate the WorkSet in | |||
| 1519 | /// bottom-up order, and for each node, we will decide whether to hoist spills | |||
| 1520 | /// inside its subtree to that node. In this way, we can get benefit locally | |||
| 1521 | /// even if hoisting all the equal spills to one cold place is impossible. | |||
| 1522 | void HoistSpillHelper::hoistAllSpills() { | |||
| 1523 | SmallVector<Register, 4> NewVRegs; | |||
| 1524 | LiveRangeEdit Edit(nullptr, NewVRegs, MF, LIS, &VRM, this); | |||
| 1525 | ||||
| 1526 | for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) { | |||
| 1527 | Register Reg = Register::index2VirtReg(i); | |||
| 1528 | Register Original = VRM.getPreSplitReg(Reg); | |||
| 1529 | if (!MRI.def_empty(Reg)) | |||
| 1530 | Virt2SiblingsMap[Original].insert(Reg); | |||
| 1531 | } | |||
| 1532 | ||||
| 1533 | // Each entry in MergeableSpills contains a spill set with equal values. | |||
| 1534 | for (auto &Ent : MergeableSpills) { | |||
| 1535 | int Slot = Ent.first.first; | |||
| 1536 | LiveInterval &OrigLI = *StackSlotToOrigLI[Slot]; | |||
| 1537 | VNInfo *OrigVNI = Ent.first.second; | |||
| 1538 | SmallPtrSet<MachineInstr *, 16> &EqValSpills = Ent.second; | |||
| 1539 | if (Ent.second.empty()) | |||
| 1540 | continue; | |||
| 1541 | ||||
| 1542 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
| 1543 | dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
| 1544 | << "Equal spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
| 1545 | for (const auto spill : EqValSpills)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
| 1546 | dbgs() << spill->getParent()->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
| 1547 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false) | |||
| 1548 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "\nFor Slot" << Slot << " and VN" << OrigVNI->id << ":\n" << "Equal spills in BB: " ; for (const auto spill : EqValSpills) dbgs() << spill-> getParent()->getNumber() << " "; dbgs() << "\n" ; }; } } while (false); | |||
| 1549 | ||||
| 1550 | // SpillsToRm is the spill set to be removed from EqValSpills. | |||
| 1551 | SmallVector<MachineInstr *, 16> SpillsToRm; | |||
| 1552 | // SpillsToIns is the spill set to be newly inserted after hoisting. | |||
| 1553 | DenseMap<MachineBasicBlock *, unsigned> SpillsToIns; | |||
| 1554 | ||||
| 1555 | runHoistSpills(OrigLI, *OrigVNI, EqValSpills, SpillsToRm, SpillsToIns); | |||
| 1556 | ||||
| 1557 | LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1558 | dbgs() << "Finally inserted spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1559 | for (const auto &Ispill : SpillsToIns)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1560 | dbgs() << Ispill.first->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1561 | dbgs() << "\nFinally removed spills in BB: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1562 | for (const auto Rspill : SpillsToRm)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1563 | dbgs() << Rspill->getParent()->getNumber() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1564 | dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false) | |||
| 1565 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("regalloc")) { { dbgs() << "Finally inserted spills in BB: " ; for (const auto &Ispill : SpillsToIns) dbgs() << Ispill .first->getNumber() << " "; dbgs() << "\nFinally removed spills in BB: " ; for (const auto Rspill : SpillsToRm) dbgs() << Rspill ->getParent()->getNumber() << " "; dbgs() << "\n"; }; } } while (false); | |||
| 1566 | ||||
| 1567 | // Stack live range update. | |||
| 1568 | LiveInterval &StackIntvl = LSS.getInterval(Slot); | |||
| 1569 | if (!SpillsToIns.empty() || !SpillsToRm.empty()) | |||
| 1570 | StackIntvl.MergeValueInAsValue(OrigLI, OrigVNI, | |||
| 1571 | StackIntvl.getValNumInfo(0)); | |||
| 1572 | ||||
| 1573 | // Insert hoisted spills. | |||
| 1574 | for (auto const &Insert : SpillsToIns) { | |||
| 1575 | MachineBasicBlock *BB = Insert.first; | |||
| 1576 | Register LiveReg = Insert.second; | |||
| 1577 | MachineBasicBlock::iterator MII = IPA.getLastInsertPointIter(OrigLI, *BB); | |||
| 1578 | MachineInstrSpan MIS(MII, BB); | |||
| 1579 | TII.storeRegToStackSlot(*BB, MII, LiveReg, false, Slot, | |||
| 1580 | MRI.getRegClass(LiveReg), &TRI); | |||
| 1581 | LIS.InsertMachineInstrRangeInMaps(MIS.begin(), MII); | |||
| 1582 | for (const MachineInstr &MI : make_range(MIS.begin(), MII)) | |||
| 1583 | getVDefInterval(MI, LIS); | |||
| 1584 | ++NumSpills; | |||
| 1585 | } | |||
| 1586 | ||||
| 1587 | // Remove redundant spills or change them to dead instructions. | |||
| 1588 | NumSpills -= SpillsToRm.size(); | |||
| 1589 | for (auto const RMEnt : SpillsToRm) { | |||
| 1590 | RMEnt->setDesc(TII.get(TargetOpcode::KILL)); | |||
| 1591 | for (unsigned i = RMEnt->getNumOperands(); i; --i) { | |||
| 1592 | MachineOperand &MO = RMEnt->getOperand(i - 1); | |||
| 1593 | if (MO.isReg() && MO.isImplicit() && MO.isDef() && !MO.isDead()) | |||
| 1594 | RMEnt->RemoveOperand(i - 1); | |||
| 1595 | } | |||
| 1596 | } | |||
| 1597 | Edit.eliminateDeadDefs(SpillsToRm, None, AA); | |||
| 1598 | } | |||
| 1599 | } | |||
| 1600 | ||||
| 1601 | /// For VirtReg clone, the \p New register should have the same physreg or | |||
| 1602 | /// stackslot as the \p old register. | |||
| 1603 | void HoistSpillHelper::LRE_DidCloneVirtReg(Register New, Register Old) { | |||
| 1604 | if (VRM.hasPhys(Old)) | |||
| 1605 | VRM.assignVirt2Phys(New, VRM.getPhys(Old)); | |||
| 1606 | else if (VRM.getStackSlot(Old) != VirtRegMap::NO_STACK_SLOT) | |||
| 1607 | VRM.assignVirt2StackSlot(New, VRM.getStackSlot(Old)); | |||
| 1608 | else | |||
| 1609 | llvm_unreachable("VReg should be assigned either physreg or stackslot")::llvm::llvm_unreachable_internal("VReg should be assigned either physreg or stackslot" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/CodeGen/InlineSpiller.cpp" , 1609); | |||
| 1610 | if (VRM.hasShape(Old)) | |||
| 1611 | VRM.assignVirt2Shape(New, VRM.getShape(Old)); | |||
| 1612 | } |
| 1 | //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains the declaration of the MachineInstr class, which is the |
| 10 | // basic representation for all target dependent machine instructions used by |
| 11 | // the back end. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #ifndef LLVM_CODEGEN_MACHINEINSTR_H |
| 16 | #define LLVM_CODEGEN_MACHINEINSTR_H |
| 17 | |
| 18 | #include "llvm/ADT/DenseMapInfo.h" |
| 19 | #include "llvm/ADT/PointerSumType.h" |
| 20 | #include "llvm/ADT/SmallSet.h" |
| 21 | #include "llvm/ADT/ilist.h" |
| 22 | #include "llvm/ADT/ilist_node.h" |
| 23 | #include "llvm/ADT/iterator_range.h" |
| 24 | #include "llvm/CodeGen/MachineMemOperand.h" |
| 25 | #include "llvm/CodeGen/MachineOperand.h" |
| 26 | #include "llvm/CodeGen/TargetOpcodes.h" |
| 27 | #include "llvm/IR/DebugLoc.h" |
| 28 | #include "llvm/IR/InlineAsm.h" |
| 29 | #include "llvm/IR/PseudoProbe.h" |
| 30 | #include "llvm/MC/MCInstrDesc.h" |
| 31 | #include "llvm/MC/MCSymbol.h" |
| 32 | #include "llvm/Support/ArrayRecycler.h" |
| 33 | #include "llvm/Support/TrailingObjects.h" |
| 34 | #include <algorithm> |
| 35 | #include <cassert> |
| 36 | #include <cstdint> |
| 37 | #include <utility> |
| 38 | |
| 39 | namespace llvm { |
| 40 | |
| 41 | class AAResults; |
| 42 | template <typename T> class ArrayRef; |
| 43 | class DIExpression; |
| 44 | class DILocalVariable; |
| 45 | class MachineBasicBlock; |
| 46 | class MachineFunction; |
| 47 | class MachineRegisterInfo; |
| 48 | class ModuleSlotTracker; |
| 49 | class raw_ostream; |
| 50 | template <typename T> class SmallVectorImpl; |
| 51 | class SmallBitVector; |
| 52 | class StringRef; |
| 53 | class TargetInstrInfo; |
| 54 | class TargetRegisterClass; |
| 55 | class TargetRegisterInfo; |
| 56 | |
| 57 | //===----------------------------------------------------------------------===// |
| 58 | /// Representation of each machine instruction. |
| 59 | /// |
| 60 | /// This class isn't a POD type, but it must have a trivial destructor. When a |
| 61 | /// MachineFunction is deleted, all the contained MachineInstrs are deallocated |
| 62 | /// without having their destructor called. |
| 63 | /// |
| 64 | class MachineInstr |
| 65 | : public ilist_node_with_parent<MachineInstr, MachineBasicBlock, |
| 66 | ilist_sentinel_tracking<true>> { |
| 67 | public: |
| 68 | using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator; |
| 69 | |
| 70 | /// Flags to specify different kinds of comments to output in |
| 71 | /// assembly code. These flags carry semantic information not |
| 72 | /// otherwise easily derivable from the IR text. |
| 73 | /// |
| 74 | enum CommentFlag { |
| 75 | ReloadReuse = 0x1, // higher bits are reserved for target dep comments. |
| 76 | NoSchedComment = 0x2, |
| 77 | TAsmComments = 0x4 // Target Asm comments should start from this value. |
| 78 | }; |
| 79 | |
| 80 | enum MIFlag { |
| 81 | NoFlags = 0, |
| 82 | FrameSetup = 1 << 0, // Instruction is used as a part of |
| 83 | // function frame setup code. |
| 84 | FrameDestroy = 1 << 1, // Instruction is used as a part of |
| 85 | // function frame destruction code. |
| 86 | BundledPred = 1 << 2, // Instruction has bundled predecessors. |
| 87 | BundledSucc = 1 << 3, // Instruction has bundled successors. |
| 88 | FmNoNans = 1 << 4, // Instruction does not support Fast |
| 89 | // math nan values. |
| 90 | FmNoInfs = 1 << 5, // Instruction does not support Fast |
| 91 | // math infinity values. |
| 92 | FmNsz = 1 << 6, // Instruction is not required to retain |
| 93 | // signed zero values. |
| 94 | FmArcp = 1 << 7, // Instruction supports Fast math |
| 95 | // reciprocal approximations. |
| 96 | FmContract = 1 << 8, // Instruction supports Fast math |
| 97 | // contraction operations like fma. |
| 98 | FmAfn = 1 << 9, // Instruction may map to Fast math |
| 99 | // instrinsic approximation. |
| 100 | FmReassoc = 1 << 10, // Instruction supports Fast math |
| 101 | // reassociation of operand order. |
| 102 | NoUWrap = 1 << 11, // Instruction supports binary operator |
| 103 | // no unsigned wrap. |
| 104 | NoSWrap = 1 << 12, // Instruction supports binary operator |
| 105 | // no signed wrap. |
| 106 | IsExact = 1 << 13, // Instruction supports division is |
| 107 | // known to be exact. |
| 108 | NoFPExcept = 1 << 14, // Instruction does not raise |
| 109 | // floatint-point exceptions. |
| 110 | NoMerge = 1 << 15, // Passes that drop source location info |
| 111 | // (e.g. branch folding) should skip |
| 112 | // this instruction. |
| 113 | }; |
| 114 | |
| 115 | private: |
| 116 | const MCInstrDesc *MCID; // Instruction descriptor. |
| 117 | MachineBasicBlock *Parent = nullptr; // Pointer to the owning basic block. |
| 118 | |
| 119 | // Operands are allocated by an ArrayRecycler. |
| 120 | MachineOperand *Operands = nullptr; // Pointer to the first operand. |
| 121 | unsigned NumOperands = 0; // Number of operands on instruction. |
| 122 | |
| 123 | uint16_t Flags = 0; // Various bits of additional |
| 124 | // information about machine |
| 125 | // instruction. |
| 126 | |
| 127 | uint8_t AsmPrinterFlags = 0; // Various bits of information used by |
| 128 | // the AsmPrinter to emit helpful |
| 129 | // comments. This is *not* semantic |
| 130 | // information. Do not use this for |
| 131 | // anything other than to convey comment |
| 132 | // information to AsmPrinter. |
| 133 | |
| 134 | // OperandCapacity has uint8_t size, so it should be next to AsmPrinterFlags |
| 135 | // to properly pack. |
| 136 | using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity; |
| 137 | OperandCapacity CapOperands; // Capacity of the Operands array. |
| 138 | |
| 139 | /// Internal implementation detail class that provides out-of-line storage for |
| 140 | /// extra info used by the machine instruction when this info cannot be stored |
| 141 | /// in-line within the instruction itself. |
| 142 | /// |
| 143 | /// This has to be defined eagerly due to the implementation constraints of |
| 144 | /// `PointerSumType` where it is used. |
| 145 | class ExtraInfo final |
| 146 | : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> { |
| 147 | public: |
| 148 | static ExtraInfo *create(BumpPtrAllocator &Allocator, |
| 149 | ArrayRef<MachineMemOperand *> MMOs, |
| 150 | MCSymbol *PreInstrSymbol = nullptr, |
| 151 | MCSymbol *PostInstrSymbol = nullptr, |
| 152 | MDNode *HeapAllocMarker = nullptr) { |
| 153 | bool HasPreInstrSymbol = PreInstrSymbol != nullptr; |
| 154 | bool HasPostInstrSymbol = PostInstrSymbol != nullptr; |
| 155 | bool HasHeapAllocMarker = HeapAllocMarker != nullptr; |
| 156 | auto *Result = new (Allocator.Allocate( |
| 157 | totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>( |
| 158 | MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, |
| 159 | HasHeapAllocMarker), |
| 160 | alignof(ExtraInfo))) |
| 161 | ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, |
| 162 | HasHeapAllocMarker); |
| 163 | |
| 164 | // Copy the actual data into the trailing objects. |
| 165 | std::copy(MMOs.begin(), MMOs.end(), |
| 166 | Result->getTrailingObjects<MachineMemOperand *>()); |
| 167 | |
| 168 | if (HasPreInstrSymbol) |
| 169 | Result->getTrailingObjects<MCSymbol *>()[0] = PreInstrSymbol; |
| 170 | if (HasPostInstrSymbol) |
| 171 | Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] = |
| 172 | PostInstrSymbol; |
| 173 | if (HasHeapAllocMarker) |
| 174 | Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker; |
| 175 | |
| 176 | return Result; |
| 177 | } |
| 178 | |
| 179 | ArrayRef<MachineMemOperand *> getMMOs() const { |
| 180 | return makeArrayRef(getTrailingObjects<MachineMemOperand *>(), NumMMOs); |
| 181 | } |
| 182 | |
| 183 | MCSymbol *getPreInstrSymbol() const { |
| 184 | return HasPreInstrSymbol ? getTrailingObjects<MCSymbol *>()[0] : nullptr; |
| 185 | } |
| 186 | |
| 187 | MCSymbol *getPostInstrSymbol() const { |
| 188 | return HasPostInstrSymbol |
| 189 | ? getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] |
| 190 | : nullptr; |
| 191 | } |
| 192 | |
| 193 | MDNode *getHeapAllocMarker() const { |
| 194 | return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr; |
| 195 | } |
| 196 | |
| 197 | private: |
| 198 | friend TrailingObjects; |
| 199 | |
| 200 | // Description of the extra info, used to interpret the actual optional |
| 201 | // data appended. |
| 202 | // |
| 203 | // Note that this is not terribly space optimized. This leaves a great deal |
| 204 | // of flexibility to fit more in here later. |
| 205 | const int NumMMOs; |
| 206 | const bool HasPreInstrSymbol; |
| 207 | const bool HasPostInstrSymbol; |
| 208 | const bool HasHeapAllocMarker; |
| 209 | |
| 210 | // Implement the `TrailingObjects` internal API. |
| 211 | size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const { |
| 212 | return NumMMOs; |
| 213 | } |
| 214 | size_t numTrailingObjects(OverloadToken<MCSymbol *>) const { |
| 215 | return HasPreInstrSymbol + HasPostInstrSymbol; |
| 216 | } |
| 217 | size_t numTrailingObjects(OverloadToken<MDNode *>) const { |
| 218 | return HasHeapAllocMarker; |
| 219 | } |
| 220 | |
| 221 | // Just a boring constructor to allow us to initialize the sizes. Always use |
| 222 | // the `create` routine above. |
| 223 | ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, |
| 224 | bool HasHeapAllocMarker) |
| 225 | : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), |
| 226 | HasPostInstrSymbol(HasPostInstrSymbol), |
| 227 | HasHeapAllocMarker(HasHeapAllocMarker) {} |
| 228 | }; |
| 229 | |
| 230 | /// Enumeration of the kinds of inline extra info available. It is important |
| 231 | /// that the `MachineMemOperand` inline kind has a tag value of zero to make |
| 232 | /// it accessible as an `ArrayRef`. |
| 233 | enum ExtraInfoInlineKinds { |
| 234 | EIIK_MMO = 0, |
| 235 | EIIK_PreInstrSymbol, |
| 236 | EIIK_PostInstrSymbol, |
| 237 | EIIK_OutOfLine |
| 238 | }; |
| 239 | |
| 240 | // We store extra information about the instruction here. The common case is |
| 241 | // expected to be nothing or a single pointer (typically a MMO or a symbol). |
| 242 | // We work to optimize this common case by storing it inline here rather than |
| 243 | // requiring a separate allocation, but we fall back to an allocation when |
| 244 | // multiple pointers are needed. |
| 245 | PointerSumType<ExtraInfoInlineKinds, |
| 246 | PointerSumTypeMember<EIIK_MMO, MachineMemOperand *>, |
| 247 | PointerSumTypeMember<EIIK_PreInstrSymbol, MCSymbol *>, |
| 248 | PointerSumTypeMember<EIIK_PostInstrSymbol, MCSymbol *>, |
| 249 | PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>> |
| 250 | Info; |
| 251 | |
| 252 | DebugLoc debugLoc; // Source line information. |
| 253 | |
| 254 | /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values |
| 255 | /// defined by this instruction. |
| 256 | unsigned DebugInstrNum; |
| 257 | |
| 258 | // Intrusive list support |
| 259 | friend struct ilist_traits<MachineInstr>; |
| 260 | friend struct ilist_callback_traits<MachineBasicBlock>; |
| 261 | void setParent(MachineBasicBlock *P) { Parent = P; } |
| 262 | |
| 263 | /// This constructor creates a copy of the given |
| 264 | /// MachineInstr in the given MachineFunction. |
| 265 | MachineInstr(MachineFunction &, const MachineInstr &); |
| 266 | |
| 267 | /// This constructor create a MachineInstr and add the implicit operands. |
| 268 | /// It reserves space for number of operands specified by |
| 269 | /// MCInstrDesc. An explicit DebugLoc is supplied. |
| 270 | MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl, |
| 271 | bool NoImp = false); |
| 272 | |
| 273 | // MachineInstrs are pool-allocated and owned by MachineFunction. |
| 274 | friend class MachineFunction; |
| 275 | |
| 276 | void |
| 277 | dumprImpl(const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth, |
| 278 | SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const; |
| 279 | |
| 280 | public: |
| 281 | MachineInstr(const MachineInstr &) = delete; |
| 282 | MachineInstr &operator=(const MachineInstr &) = delete; |
| 283 | // Use MachineFunction::DeleteMachineInstr() instead. |
| 284 | ~MachineInstr() = delete; |
| 285 | |
| 286 | const MachineBasicBlock* getParent() const { return Parent; } |
| 287 | MachineBasicBlock* getParent() { return Parent; } |
| 288 | |
| 289 | /// Move the instruction before \p MovePos. |
| 290 | void moveBefore(MachineInstr *MovePos); |
| 291 | |
| 292 | /// Return the function that contains the basic block that this instruction |
| 293 | /// belongs to. |
| 294 | /// |
| 295 | /// Note: this is undefined behaviour if the instruction does not have a |
| 296 | /// parent. |
| 297 | const MachineFunction *getMF() const; |
| 298 | MachineFunction *getMF() { |
| 299 | return const_cast<MachineFunction *>( |
| 300 | static_cast<const MachineInstr *>(this)->getMF()); |
| 301 | } |
| 302 | |
| 303 | /// Return the asm printer flags bitvector. |
| 304 | uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } |
| 305 | |
| 306 | /// Clear the AsmPrinter bitvector. |
| 307 | void clearAsmPrinterFlags() { AsmPrinterFlags = 0; } |
| 308 | |
| 309 | /// Return whether an AsmPrinter flag is set. |
| 310 | bool getAsmPrinterFlag(CommentFlag Flag) const { |
| 311 | return AsmPrinterFlags & Flag; |
| 312 | } |
| 313 | |
| 314 | /// Set a flag for the AsmPrinter. |
| 315 | void setAsmPrinterFlag(uint8_t Flag) { |
| 316 | AsmPrinterFlags |= Flag; |
| 317 | } |
| 318 | |
| 319 | /// Clear specific AsmPrinter flags. |
| 320 | void clearAsmPrinterFlag(CommentFlag Flag) { |
| 321 | AsmPrinterFlags &= ~Flag; |
| 322 | } |
| 323 | |
| 324 | /// Return the MI flags bitvector. |
| 325 | uint16_t getFlags() const { |
| 326 | return Flags; |
| 327 | } |
| 328 | |
| 329 | /// Return whether an MI flag is set. |
| 330 | bool getFlag(MIFlag Flag) const { |
| 331 | return Flags & Flag; |
| 332 | } |
| 333 | |
| 334 | /// Set a MI flag. |
| 335 | void setFlag(MIFlag Flag) { |
| 336 | Flags |= (uint16_t)Flag; |
| 337 | } |
| 338 | |
| 339 | void setFlags(unsigned flags) { |
| 340 | // Filter out the automatically maintained flags. |
| 341 | unsigned Mask = BundledPred | BundledSucc; |
| 342 | Flags = (Flags & Mask) | (flags & ~Mask); |
| 343 | } |
| 344 | |
| 345 | /// clearFlag - Clear a MI flag. |
| 346 | void clearFlag(MIFlag Flag) { |
| 347 | Flags &= ~((uint16_t)Flag); |
| 348 | } |
| 349 | |
| 350 | /// Return true if MI is in a bundle (but not the first MI in a bundle). |
| 351 | /// |
| 352 | /// A bundle looks like this before it's finalized: |
| 353 | /// ---------------- |
| 354 | /// | MI | |
| 355 | /// ---------------- |
| 356 | /// | |
| 357 | /// ---------------- |
| 358 | /// | MI * | |
| 359 | /// ---------------- |
| 360 | /// | |
| 361 | /// ---------------- |
| 362 | /// | MI * | |
| 363 | /// ---------------- |
| 364 | /// In this case, the first MI starts a bundle but is not inside a bundle, the |
| 365 | /// next 2 MIs are considered "inside" the bundle. |
| 366 | /// |
| 367 | /// After a bundle is finalized, it looks like this: |
| 368 | /// ---------------- |
| 369 | /// | Bundle | |
| 370 | /// ---------------- |
| 371 | /// | |
| 372 | /// ---------------- |
| 373 | /// | MI * | |
| 374 | /// ---------------- |
| 375 | /// | |
| 376 | /// ---------------- |
| 377 | /// | MI * | |
| 378 | /// ---------------- |
| 379 | /// | |
| 380 | /// ---------------- |
| 381 | /// | MI * | |
| 382 | /// ---------------- |
| 383 | /// The first instruction has the special opcode "BUNDLE". It's not "inside" |
| 384 | /// a bundle, but the next three MIs are. |
| 385 | bool isInsideBundle() const { |
| 386 | return getFlag(BundledPred); |
| 387 | } |
| 388 | |
| 389 | /// Return true if this instruction part of a bundle. This is true |
| 390 | /// if either itself or its following instruction is marked "InsideBundle". |
| 391 | bool isBundled() const { |
| 392 | return isBundledWithPred() || isBundledWithSucc(); |
| 393 | } |
| 394 | |
| 395 | /// Return true if this instruction is part of a bundle, and it is not the |
| 396 | /// first instruction in the bundle. |
| 397 | bool isBundledWithPred() const { return getFlag(BundledPred); } |
| 398 | |
| 399 | /// Return true if this instruction is part of a bundle, and it is not the |
| 400 | /// last instruction in the bundle. |
| 401 | bool isBundledWithSucc() const { return getFlag(BundledSucc); } |
| 402 | |
| 403 | /// Bundle this instruction with its predecessor. This can be an unbundled |
| 404 | /// instruction, or it can be the first instruction in a bundle. |
| 405 | void bundleWithPred(); |
| 406 | |
| 407 | /// Bundle this instruction with its successor. This can be an unbundled |
| 408 | /// instruction, or it can be the last instruction in a bundle. |
| 409 | void bundleWithSucc(); |
| 410 | |
| 411 | /// Break bundle above this instruction. |
| 412 | void unbundleFromPred(); |
| 413 | |
| 414 | /// Break bundle below this instruction. |
| 415 | void unbundleFromSucc(); |
| 416 | |
| 417 | /// Returns the debug location id of this MachineInstr. |
| 418 | const DebugLoc &getDebugLoc() const { return debugLoc; } |
| 419 | |
| 420 | /// Return the operand containing the offset to be used if this DBG_VALUE |
| 421 | /// instruction is indirect; will be an invalid register if this value is |
| 422 | /// not indirect, and an immediate with value 0 otherwise. |
| 423 | const MachineOperand &getDebugOffset() const { |
| 424 | assert(isNonListDebugValue() && "not a DBG_VALUE")((isNonListDebugValue() && "not a DBG_VALUE") ? static_cast <void> (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 424, __PRETTY_FUNCTION__)); |
| 425 | return getOperand(1); |
| 426 | } |
| 427 | MachineOperand &getDebugOffset() { |
| 428 | assert(isNonListDebugValue() && "not a DBG_VALUE")((isNonListDebugValue() && "not a DBG_VALUE") ? static_cast <void> (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 428, __PRETTY_FUNCTION__)); |
| 429 | return getOperand(1); |
| 430 | } |
| 431 | |
| 432 | /// Return the operand for the debug variable referenced by |
| 433 | /// this DBG_VALUE instruction. |
| 434 | const MachineOperand &getDebugVariableOp() const; |
| 435 | MachineOperand &getDebugVariableOp(); |
| 436 | |
| 437 | /// Return the debug variable referenced by |
| 438 | /// this DBG_VALUE instruction. |
| 439 | const DILocalVariable *getDebugVariable() const; |
| 440 | |
| 441 | /// Return the operand for the complex address expression referenced by |
| 442 | /// this DBG_VALUE instruction. |
| 443 | const MachineOperand &getDebugExpressionOp() const; |
| 444 | MachineOperand &getDebugExpressionOp(); |
| 445 | |
| 446 | /// Return the complex address expression referenced by |
| 447 | /// this DBG_VALUE instruction. |
| 448 | const DIExpression *getDebugExpression() const; |
| 449 | |
| 450 | /// Return the debug label referenced by |
| 451 | /// this DBG_LABEL instruction. |
| 452 | const DILabel *getDebugLabel() const; |
| 453 | |
| 454 | /// Fetch the instruction number of this MachineInstr. If it does not have |
| 455 | /// one already, a new and unique number will be assigned. |
| 456 | unsigned getDebugInstrNum(); |
| 457 | |
| 458 | /// Examine the instruction number of this MachineInstr. May be zero if |
| 459 | /// it hasn't been assigned a number yet. |
| 460 | unsigned peekDebugInstrNum() const { return DebugInstrNum; } |
| 461 | |
| 462 | /// Set instruction number of this MachineInstr. Avoid using unless you're |
| 463 | /// deserializing this information. |
| 464 | void setDebugInstrNum(unsigned Num) { DebugInstrNum = Num; } |
| 465 | |
| 466 | /// Emit an error referring to the source location of this instruction. |
| 467 | /// This should only be used for inline assembly that is somehow |
| 468 | /// impossible to compile. Other errors should have been handled much |
| 469 | /// earlier. |
| 470 | /// |
| 471 | /// If this method returns, the caller should try to recover from the error. |
| 472 | void emitError(StringRef Msg) const; |
| 473 | |
| 474 | /// Returns the target instruction descriptor of this MachineInstr. |
| 475 | const MCInstrDesc &getDesc() const { return *MCID; } |
| 476 | |
| 477 | /// Returns the opcode of this MachineInstr. |
| 478 | unsigned getOpcode() const { return MCID->Opcode; } |
| 479 | |
| 480 | /// Retuns the total number of operands. |
| 481 | unsigned getNumOperands() const { return NumOperands; } |
| 482 | |
| 483 | /// Returns the total number of operands which are debug locations. |
| 484 | unsigned getNumDebugOperands() const { |
| 485 | return std::distance(debug_operands().begin(), debug_operands().end()); |
| 486 | } |
| 487 | |
| 488 | const MachineOperand& getOperand(unsigned i) const { |
| 489 | assert(i < getNumOperands() && "getOperand() out of range!")((i < getNumOperands() && "getOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 489, __PRETTY_FUNCTION__)); |
| 490 | return Operands[i]; |
| 491 | } |
| 492 | MachineOperand& getOperand(unsigned i) { |
| 493 | assert(i < getNumOperands() && "getOperand() out of range!")((i < getNumOperands() && "getOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 493, __PRETTY_FUNCTION__)); |
| 494 | return Operands[i]; |
| 495 | } |
| 496 | |
| 497 | MachineOperand &getDebugOperand(unsigned Index) { |
| 498 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")((Index < getNumDebugOperands() && "getDebugOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 498, __PRETTY_FUNCTION__)); |
| 499 | return *(debug_operands().begin() + Index); |
| 500 | } |
| 501 | const MachineOperand &getDebugOperand(unsigned Index) const { |
| 502 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")((Index < getNumDebugOperands() && "getDebugOperand() out of range!" ) ? static_cast<void> (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 502, __PRETTY_FUNCTION__)); |
| 503 | return *(debug_operands().begin() + Index); |
| 504 | } |
| 505 | |
| 506 | SmallSet<Register, 4> getUsedDebugRegs() const { |
| 507 | assert(isDebugValue() && "not a DBG_VALUE*")((isDebugValue() && "not a DBG_VALUE*") ? static_cast <void> (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE*\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 507, __PRETTY_FUNCTION__)); |
| 508 | SmallSet<Register, 4> UsedRegs; |
| 509 | for (auto MO : debug_operands()) |
| 510 | if (MO.isReg() && MO.getReg()) |
| 511 | UsedRegs.insert(MO.getReg()); |
| 512 | return UsedRegs; |
| 513 | } |
| 514 | |
| 515 | /// Returns whether this debug value has at least one debug operand with the |
| 516 | /// register \p Reg. |
| 517 | bool hasDebugOperandForReg(Register Reg) const { |
| 518 | return any_of(debug_operands(), [Reg](const MachineOperand &Op) { |
| 519 | return Op.isReg() && Op.getReg() == Reg; |
| 520 | }); |
| 521 | } |
| 522 | |
| 523 | /// Returns a range of all of the operands that correspond to a debug use of |
| 524 | /// \p Reg. |
| 525 | template <typename Operand, typename Instruction> |
| 526 | static iterator_range< |
| 527 | filter_iterator<Operand *, std::function<bool(Operand &Op)>>> |
| 528 | getDebugOperandsForReg(Instruction *MI, Register Reg) { |
| 529 | std::function<bool(Operand & Op)> OpUsesReg( |
| 530 | [Reg](Operand &Op) { return Op.isReg() && Op.getReg() == Reg; }); |
| 531 | return make_filter_range(MI->debug_operands(), OpUsesReg); |
| 532 | } |
| 533 | iterator_range<filter_iterator<const MachineOperand *, |
| 534 | std::function<bool(const MachineOperand &Op)>>> |
| 535 | getDebugOperandsForReg(Register Reg) const { |
| 536 | return MachineInstr::getDebugOperandsForReg<const MachineOperand, |
| 537 | const MachineInstr>(this, Reg); |
| 538 | } |
| 539 | iterator_range<filter_iterator<MachineOperand *, |
| 540 | std::function<bool(MachineOperand &Op)>>> |
| 541 | getDebugOperandsForReg(Register Reg) { |
| 542 | return MachineInstr::getDebugOperandsForReg<MachineOperand, MachineInstr>( |
| 543 | this, Reg); |
| 544 | } |
| 545 | |
| 546 | bool isDebugOperand(const MachineOperand *Op) const { |
| 547 | return Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()); |
| 548 | } |
| 549 | |
| 550 | unsigned getDebugOperandIndex(const MachineOperand *Op) const { |
| 551 | assert(isDebugOperand(Op) && "Expected a debug operand.")((isDebugOperand(Op) && "Expected a debug operand.") ? static_cast<void> (0) : __assert_fail ("isDebugOperand(Op) && \"Expected a debug operand.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 551, __PRETTY_FUNCTION__)); |
| 552 | return std::distance(adl_begin(debug_operands()), Op); |
| 553 | } |
| 554 | |
| 555 | /// Returns the total number of definitions. |
| 556 | unsigned getNumDefs() const { |
| 557 | return getNumExplicitDefs() + MCID->getNumImplicitDefs(); |
| 558 | } |
| 559 | |
| 560 | /// Returns true if the instruction has implicit definition. |
| 561 | bool hasImplicitDef() const { |
| 562 | for (unsigned I = getNumExplicitOperands(), E = getNumOperands(); |
| 563 | I != E; ++I) { |
| 564 | const MachineOperand &MO = getOperand(I); |
| 565 | if (MO.isDef() && MO.isImplicit()) |
| 566 | return true; |
| 567 | } |
| 568 | return false; |
| 569 | } |
| 570 | |
| 571 | /// Returns the implicit operands number. |
| 572 | unsigned getNumImplicitOperands() const { |
| 573 | return getNumOperands() - getNumExplicitOperands(); |
| 574 | } |
| 575 | |
| 576 | /// Return true if operand \p OpIdx is a subregister index. |
| 577 | bool isOperandSubregIdx(unsigned OpIdx) const { |
| 578 | assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate &&((getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && "Expected MO_Immediate operand type.") ? static_cast <void> (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 579, __PRETTY_FUNCTION__)) |
| 579 | "Expected MO_Immediate operand type.")((getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && "Expected MO_Immediate operand type.") ? static_cast <void> (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 579, __PRETTY_FUNCTION__)); |
| 580 | if (isExtractSubreg() && OpIdx == 2) |
| 581 | return true; |
| 582 | if (isInsertSubreg() && OpIdx == 3) |
| 583 | return true; |
| 584 | if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0) |
| 585 | return true; |
| 586 | if (isSubregToReg() && OpIdx == 3) |
| 587 | return true; |
| 588 | return false; |
| 589 | } |
| 590 | |
| 591 | /// Returns the number of non-implicit operands. |
| 592 | unsigned getNumExplicitOperands() const; |
| 593 | |
| 594 | /// Returns the number of non-implicit definitions. |
| 595 | unsigned getNumExplicitDefs() const; |
| 596 | |
| 597 | /// iterator/begin/end - Iterate over all operands of a machine instruction. |
| 598 | using mop_iterator = MachineOperand *; |
| 599 | using const_mop_iterator = const MachineOperand *; |
| 600 | |
| 601 | mop_iterator operands_begin() { return Operands; } |
| 602 | mop_iterator operands_end() { return Operands + NumOperands; } |
| 603 | |
| 604 | const_mop_iterator operands_begin() const { return Operands; } |
| 605 | const_mop_iterator operands_end() const { return Operands + NumOperands; } |
| 606 | |
| 607 | iterator_range<mop_iterator> operands() { |
| 608 | return make_range(operands_begin(), operands_end()); |
| 609 | } |
| 610 | iterator_range<const_mop_iterator> operands() const { |
| 611 | return make_range(operands_begin(), operands_end()); |
| 612 | } |
| 613 | iterator_range<mop_iterator> explicit_operands() { |
| 614 | return make_range(operands_begin(), |
| 615 | operands_begin() + getNumExplicitOperands()); |
| 616 | } |
| 617 | iterator_range<const_mop_iterator> explicit_operands() const { |
| 618 | return make_range(operands_begin(), |
| 619 | operands_begin() + getNumExplicitOperands()); |
| 620 | } |
| 621 | iterator_range<mop_iterator> implicit_operands() { |
| 622 | return make_range(explicit_operands().end(), operands_end()); |
| 623 | } |
| 624 | iterator_range<const_mop_iterator> implicit_operands() const { |
| 625 | return make_range(explicit_operands().end(), operands_end()); |
| 626 | } |
| 627 | /// Returns a range over all operands that are used to determine the variable |
| 628 | /// location for this DBG_VALUE instruction. |
| 629 | iterator_range<mop_iterator> debug_operands() { |
| 630 | assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction." ) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 630, __PRETTY_FUNCTION__)); |
| 631 | return isDebugValueList() |
| 632 | ? make_range(operands_begin() + 2, operands_end()) |
| 633 | : make_range(operands_begin(), operands_begin() + 1); |
| 634 | } |
| 635 | /// \copydoc debug_operands() |
| 636 | iterator_range<const_mop_iterator> debug_operands() const { |
| 637 | assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction." ) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 637, __PRETTY_FUNCTION__)); |
| 638 | return isDebugValueList() |
| 639 | ? make_range(operands_begin() + 2, operands_end()) |
| 640 | : make_range(operands_begin(), operands_begin() + 1); |
| 641 | } |
| 642 | /// Returns a range over all explicit operands that are register definitions. |
| 643 | /// Implicit definition are not included! |
| 644 | iterator_range<mop_iterator> defs() { |
| 645 | return make_range(operands_begin(), |
| 646 | operands_begin() + getNumExplicitDefs()); |
| 647 | } |
| 648 | /// \copydoc defs() |
| 649 | iterator_range<const_mop_iterator> defs() const { |
| 650 | return make_range(operands_begin(), |
| 651 | operands_begin() + getNumExplicitDefs()); |
| 652 | } |
| 653 | /// Returns a range that includes all operands that are register uses. |
| 654 | /// This may include unrelated operands which are not register uses. |
| 655 | iterator_range<mop_iterator> uses() { |
| 656 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
| 657 | } |
| 658 | /// \copydoc uses() |
| 659 | iterator_range<const_mop_iterator> uses() const { |
| 660 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
| 661 | } |
| 662 | iterator_range<mop_iterator> explicit_uses() { |
| 663 | return make_range(operands_begin() + getNumExplicitDefs(), |
| 664 | operands_begin() + getNumExplicitOperands()); |
| 665 | } |
| 666 | iterator_range<const_mop_iterator> explicit_uses() const { |
| 667 | return make_range(operands_begin() + getNumExplicitDefs(), |
| 668 | operands_begin() + getNumExplicitOperands()); |
| 669 | } |
| 670 | |
| 671 | /// Returns the number of the operand iterator \p I points to. |
| 672 | unsigned getOperandNo(const_mop_iterator I) const { |
| 673 | return I - operands_begin(); |
| 674 | } |
| 675 | |
| 676 | /// Access to memory operands of the instruction. If there are none, that does |
| 677 | /// not imply anything about whether the function accesses memory. Instead, |
| 678 | /// the caller must behave conservatively. |
| 679 | ArrayRef<MachineMemOperand *> memoperands() const { |
| 680 | if (!Info) |
| 681 | return {}; |
| 682 | |
| 683 | if (Info.is<EIIK_MMO>()) |
| 684 | return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); |
| 685 | |
| 686 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 687 | return EI->getMMOs(); |
| 688 | |
| 689 | return {}; |
| 690 | } |
| 691 | |
| 692 | /// Access to memory operands of the instruction. |
| 693 | /// |
| 694 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
| 695 | /// anything about whether the function accesses memory. Instead, the caller |
| 696 | /// must behave conservatively. |
| 697 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } |
| 698 | |
| 699 | /// Access to memory operands of the instruction. |
| 700 | /// |
| 701 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
| 702 | /// anything about whether the function accesses memory. Instead, the caller |
| 703 | /// must behave conservatively. |
| 704 | mmo_iterator memoperands_end() const { return memoperands().end(); } |
| 705 | |
| 706 | /// Return true if we don't have any memory operands which described the |
| 707 | /// memory access done by this instruction. If this is true, calling code |
| 708 | /// must be conservative. |
| 709 | bool memoperands_empty() const { return memoperands().empty(); } |
| 710 | |
| 711 | /// Return true if this instruction has exactly one MachineMemOperand. |
| 712 | bool hasOneMemOperand() const { return memoperands().size() == 1; } |
| 713 | |
| 714 | /// Return the number of memory operands. |
| 715 | unsigned getNumMemOperands() const { return memoperands().size(); } |
| 716 | |
| 717 | /// Helper to extract a pre-instruction symbol if one has been added. |
| 718 | MCSymbol *getPreInstrSymbol() const { |
| 719 | if (!Info) |
| 720 | return nullptr; |
| 721 | if (MCSymbol *S = Info.get<EIIK_PreInstrSymbol>()) |
| 722 | return S; |
| 723 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 724 | return EI->getPreInstrSymbol(); |
| 725 | |
| 726 | return nullptr; |
| 727 | } |
| 728 | |
| 729 | /// Helper to extract a post-instruction symbol if one has been added. |
| 730 | MCSymbol *getPostInstrSymbol() const { |
| 731 | if (!Info) |
| 732 | return nullptr; |
| 733 | if (MCSymbol *S = Info.get<EIIK_PostInstrSymbol>()) |
| 734 | return S; |
| 735 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 736 | return EI->getPostInstrSymbol(); |
| 737 | |
| 738 | return nullptr; |
| 739 | } |
| 740 | |
| 741 | /// Helper to extract a heap alloc marker if one has been added. |
| 742 | MDNode *getHeapAllocMarker() const { |
| 743 | if (!Info) |
| 744 | return nullptr; |
| 745 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 746 | return EI->getHeapAllocMarker(); |
| 747 | |
| 748 | return nullptr; |
| 749 | } |
| 750 | |
| 751 | /// API for querying MachineInstr properties. They are the same as MCInstrDesc |
| 752 | /// queries but they are bundle aware. |
| 753 | |
| 754 | enum QueryType { |
| 755 | IgnoreBundle, // Ignore bundles |
| 756 | AnyInBundle, // Return true if any instruction in bundle has property |
| 757 | AllInBundle // Return true if all instructions in bundle have property |
| 758 | }; |
| 759 | |
| 760 | /// Return true if the instruction (or in the case of a bundle, |
| 761 | /// the instructions inside the bundle) has the specified property. |
| 762 | /// The first argument is the property being queried. |
| 763 | /// The second argument indicates whether the query should look inside |
| 764 | /// instruction bundles. |
| 765 | bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { |
| 766 | assert(MCFlag < 64 &&((MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? static_cast<void> (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 767, __PRETTY_FUNCTION__)) |
| 767 | "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.")((MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? static_cast<void> (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 767, __PRETTY_FUNCTION__)); |
| 768 | // Inline the fast path for unbundled or bundle-internal instructions. |
| 769 | if (Type == IgnoreBundle || !isBundled() || isBundledWithPred()) |
| 770 | return getDesc().getFlags() & (1ULL << MCFlag); |
| 771 | |
| 772 | // If this is the first instruction in a bundle, take the slow path. |
| 773 | return hasPropertyInBundle(1ULL << MCFlag, Type); |
| 774 | } |
| 775 | |
| 776 | /// Return true if this is an instruction that should go through the usual |
| 777 | /// legalization steps. |
| 778 | bool isPreISelOpcode(QueryType Type = IgnoreBundle) const { |
| 779 | return hasProperty(MCID::PreISelOpcode, Type); |
| 780 | } |
| 781 | |
| 782 | /// Return true if this instruction can have a variable number of operands. |
| 783 | /// In this case, the variable operands will be after the normal |
| 784 | /// operands but before the implicit definitions and uses (if any are |
| 785 | /// present). |
| 786 | bool isVariadic(QueryType Type = IgnoreBundle) const { |
| 787 | return hasProperty(MCID::Variadic, Type); |
| 788 | } |
| 789 | |
| 790 | /// Set if this instruction has an optional definition, e.g. |
| 791 | /// ARM instructions which can set condition code if 's' bit is set. |
| 792 | bool hasOptionalDef(QueryType Type = IgnoreBundle) const { |
| 793 | return hasProperty(MCID::HasOptionalDef, Type); |
| 794 | } |
| 795 | |
| 796 | /// Return true if this is a pseudo instruction that doesn't |
| 797 | /// correspond to a real machine instruction. |
| 798 | bool isPseudo(QueryType Type = IgnoreBundle) const { |
| 799 | return hasProperty(MCID::Pseudo, Type); |
| 800 | } |
| 801 | |
| 802 | bool isReturn(QueryType Type = AnyInBundle) const { |
| 803 | return hasProperty(MCID::Return, Type); |
| 804 | } |
| 805 | |
| 806 | /// Return true if this is an instruction that marks the end of an EH scope, |
| 807 | /// i.e., a catchpad or a cleanuppad instruction. |
| 808 | bool isEHScopeReturn(QueryType Type = AnyInBundle) const { |
| 809 | return hasProperty(MCID::EHScopeReturn, Type); |
| 810 | } |
| 811 | |
| 812 | bool isCall(QueryType Type = AnyInBundle) const { |
| 813 | return hasProperty(MCID::Call, Type); |
| 814 | } |
| 815 | |
| 816 | /// Return true if this is a call instruction that may have an associated |
| 817 | /// call site entry in the debug info. |
| 818 | bool isCandidateForCallSiteEntry(QueryType Type = IgnoreBundle) const; |
| 819 | /// Return true if copying, moving, or erasing this instruction requires |
| 820 | /// updating Call Site Info (see \ref copyCallSiteInfo, \ref moveCallSiteInfo, |
| 821 | /// \ref eraseCallSiteInfo). |
| 822 | bool shouldUpdateCallSiteInfo() const; |
| 823 | |
| 824 | /// Returns true if the specified instruction stops control flow |
| 825 | /// from executing the instruction immediately following it. Examples include |
| 826 | /// unconditional branches and return instructions. |
| 827 | bool isBarrier(QueryType Type = AnyInBundle) const { |
| 828 | return hasProperty(MCID::Barrier, Type); |
| 829 | } |
| 830 | |
| 831 | /// Returns true if this instruction part of the terminator for a basic block. |
| 832 | /// Typically this is things like return and branch instructions. |
| 833 | /// |
| 834 | /// Various passes use this to insert code into the bottom of a basic block, |
| 835 | /// but before control flow occurs. |
| 836 | bool isTerminator(QueryType Type = AnyInBundle) const { |
| 837 | return hasProperty(MCID::Terminator, Type); |
| 838 | } |
| 839 | |
| 840 | /// Returns true if this is a conditional, unconditional, or indirect branch. |
| 841 | /// Predicates below can be used to discriminate between |
| 842 | /// these cases, and the TargetInstrInfo::analyzeBranch method can be used to |
| 843 | /// get more information. |
| 844 | bool isBranch(QueryType Type = AnyInBundle) const { |
| 845 | return hasProperty(MCID::Branch, Type); |
| 846 | } |
| 847 | |
| 848 | /// Return true if this is an indirect branch, such as a |
| 849 | /// branch through a register. |
| 850 | bool isIndirectBranch(QueryType Type = AnyInBundle) const { |
| 851 | return hasProperty(MCID::IndirectBranch, Type); |
| 852 | } |
| 853 | |
| 854 | /// Return true if this is a branch which may fall |
| 855 | /// through to the next instruction or may transfer control flow to some other |
| 856 | /// block. The TargetInstrInfo::analyzeBranch method can be used to get more |
| 857 | /// information about this branch. |
| 858 | bool isConditionalBranch(QueryType Type = AnyInBundle) const { |
| 859 | return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type); |
| 860 | } |
| 861 | |
| 862 | /// Return true if this is a branch which always |
| 863 | /// transfers control flow to some other block. The |
| 864 | /// TargetInstrInfo::analyzeBranch method can be used to get more information |
| 865 | /// about this branch. |
| 866 | bool isUnconditionalBranch(QueryType Type = AnyInBundle) const { |
| 867 | return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type); |
| 868 | } |
| 869 | |
| 870 | /// Return true if this instruction has a predicate operand that |
| 871 | /// controls execution. It may be set to 'always', or may be set to other |
| 872 | /// values. There are various methods in TargetInstrInfo that can be used to |
| 873 | /// control and modify the predicate in this instruction. |
| 874 | bool isPredicable(QueryType Type = AllInBundle) const { |
| 875 | // If it's a bundle than all bundled instructions must be predicable for this |
| 876 | // to return true. |
| 877 | return hasProperty(MCID::Predicable, Type); |
| 878 | } |
| 879 | |
| 880 | /// Return true if this instruction is a comparison. |
| 881 | bool isCompare(QueryType Type = IgnoreBundle) const { |
| 882 | return hasProperty(MCID::Compare, Type); |
| 883 | } |
| 884 | |
| 885 | /// Return true if this instruction is a move immediate |
| 886 | /// (including conditional moves) instruction. |
| 887 | bool isMoveImmediate(QueryType Type = IgnoreBundle) const { |
| 888 | return hasProperty(MCID::MoveImm, Type); |
| 889 | } |
| 890 | |
| 891 | /// Return true if this instruction is a register move. |
| 892 | /// (including moving values from subreg to reg) |
| 893 | bool isMoveReg(QueryType Type = IgnoreBundle) const { |
| 894 | return hasProperty(MCID::MoveReg, Type); |
| 895 | } |
| 896 | |
| 897 | /// Return true if this instruction is a bitcast instruction. |
| 898 | bool isBitcast(QueryType Type = IgnoreBundle) const { |
| 899 | return hasProperty(MCID::Bitcast, Type); |
| 900 | } |
| 901 | |
| 902 | /// Return true if this instruction is a select instruction. |
| 903 | bool isSelect(QueryType Type = IgnoreBundle) const { |
| 904 | return hasProperty(MCID::Select, Type); |
| 905 | } |
| 906 | |
| 907 | /// Return true if this instruction cannot be safely duplicated. |
| 908 | /// For example, if the instruction has a unique labels attached |
| 909 | /// to it, duplicating it would cause multiple definition errors. |
| 910 | bool isNotDuplicable(QueryType Type = AnyInBundle) const { |
| 911 | return hasProperty(MCID::NotDuplicable, Type); |
| 912 | } |
| 913 | |
| 914 | /// Return true if this instruction is convergent. |
| 915 | /// Convergent instructions can not be made control-dependent on any |
| 916 | /// additional values. |
| 917 | bool isConvergent(QueryType Type = AnyInBundle) const { |
| 918 | if (isInlineAsm()) { |
| 919 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
| 920 | if (ExtraInfo & InlineAsm::Extra_IsConvergent) |
| 921 | return true; |
| 922 | } |
| 923 | return hasProperty(MCID::Convergent, Type); |
| 924 | } |
| 925 | |
| 926 | /// Returns true if the specified instruction has a delay slot |
| 927 | /// which must be filled by the code generator. |
| 928 | bool hasDelaySlot(QueryType Type = AnyInBundle) const { |
| 929 | return hasProperty(MCID::DelaySlot, Type); |
| 930 | } |
| 931 | |
| 932 | /// Return true for instructions that can be folded as |
| 933 | /// memory operands in other instructions. The most common use for this |
| 934 | /// is instructions that are simple loads from memory that don't modify |
| 935 | /// the loaded value in any way, but it can also be used for instructions |
| 936 | /// that can be expressed as constant-pool loads, such as V_SETALLONES |
| 937 | /// on x86, to allow them to be folded when it is beneficial. |
| 938 | /// This should only be set on instructions that return a value in their |
| 939 | /// only virtual register definition. |
| 940 | bool canFoldAsLoad(QueryType Type = IgnoreBundle) const { |
| 941 | return hasProperty(MCID::FoldableAsLoad, Type); |
| 942 | } |
| 943 | |
| 944 | /// Return true if this instruction behaves |
| 945 | /// the same way as the generic REG_SEQUENCE instructions. |
| 946 | /// E.g., on ARM, |
| 947 | /// dX VMOVDRR rY, rZ |
| 948 | /// is equivalent to |
| 949 | /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. |
| 950 | /// |
| 951 | /// Note that for the optimizers to be able to take advantage of |
| 952 | /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be |
| 953 | /// override accordingly. |
| 954 | bool isRegSequenceLike(QueryType Type = IgnoreBundle) const { |
| 955 | return hasProperty(MCID::RegSequence, Type); |
| 956 | } |
| 957 | |
| 958 | /// Return true if this instruction behaves |
| 959 | /// the same way as the generic EXTRACT_SUBREG instructions. |
| 960 | /// E.g., on ARM, |
| 961 | /// rX, rY VMOVRRD dZ |
| 962 | /// is equivalent to two EXTRACT_SUBREG: |
| 963 | /// rX = EXTRACT_SUBREG dZ, ssub_0 |
| 964 | /// rY = EXTRACT_SUBREG dZ, ssub_1 |
| 965 | /// |
| 966 | /// Note that for the optimizers to be able to take advantage of |
| 967 | /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be |
| 968 | /// override accordingly. |
| 969 | bool isExtractSubregLike(QueryType Type = IgnoreBundle) const { |
| 970 | return hasProperty(MCID::ExtractSubreg, Type); |
| 971 | } |
| 972 | |
| 973 | /// Return true if this instruction behaves |
| 974 | /// the same way as the generic INSERT_SUBREG instructions. |
| 975 | /// E.g., on ARM, |
| 976 | /// dX = VSETLNi32 dY, rZ, Imm |
| 977 | /// is equivalent to a INSERT_SUBREG: |
| 978 | /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) |
| 979 | /// |
| 980 | /// Note that for the optimizers to be able to take advantage of |
| 981 | /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be |
| 982 | /// override accordingly. |
| 983 | bool isInsertSubregLike(QueryType Type = IgnoreBundle) const { |
| 984 | return hasProperty(MCID::InsertSubreg, Type); |
| 985 | } |
| 986 | |
| 987 | //===--------------------------------------------------------------------===// |
| 988 | // Side Effect Analysis |
| 989 | //===--------------------------------------------------------------------===// |
| 990 | |
| 991 | /// Return true if this instruction could possibly read memory. |
| 992 | /// Instructions with this flag set are not necessarily simple load |
| 993 | /// instructions, they may load a value and modify it, for example. |
| 994 | bool mayLoad(QueryType Type = AnyInBundle) const { |
| 995 | if (isInlineAsm()) { |
| 996 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
| 997 | if (ExtraInfo & InlineAsm::Extra_MayLoad) |
| 998 | return true; |
| 999 | } |
| 1000 | return hasProperty(MCID::MayLoad, Type); |
| 1001 | } |
| 1002 | |
| 1003 | /// Return true if this instruction could possibly modify memory. |
| 1004 | /// Instructions with this flag set are not necessarily simple store |
| 1005 | /// instructions, they may store a modified value based on their operands, or |
| 1006 | /// may not actually modify anything, for example. |
| 1007 | bool mayStore(QueryType Type = AnyInBundle) const { |
| 1008 | if (isInlineAsm()) { |
| 1009 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
| 1010 | if (ExtraInfo & InlineAsm::Extra_MayStore) |
| 1011 | return true; |
| 1012 | } |
| 1013 | return hasProperty(MCID::MayStore, Type); |
| 1014 | } |
| 1015 | |
| 1016 | /// Return true if this instruction could possibly read or modify memory. |
| 1017 | bool mayLoadOrStore(QueryType Type = AnyInBundle) const { |
| 1018 | return mayLoad(Type) || mayStore(Type); |
| 1019 | } |
| 1020 | |
| 1021 | /// Return true if this instruction could possibly raise a floating-point |
| 1022 | /// exception. This is the case if the instruction is a floating-point |
| 1023 | /// instruction that can in principle raise an exception, as indicated |
| 1024 | /// by the MCID::MayRaiseFPException property, *and* at the same time, |
| 1025 | /// the instruction is used in a context where we expect floating-point |
| 1026 | /// exceptions are not disabled, as indicated by the NoFPExcept MI flag. |
| 1027 | bool mayRaiseFPException() const { |
| 1028 | return hasProperty(MCID::MayRaiseFPException) && |
| 1029 | !getFlag(MachineInstr::MIFlag::NoFPExcept); |
| 1030 | } |
| 1031 | |
| 1032 | //===--------------------------------------------------------------------===// |
| 1033 | // Flags that indicate whether an instruction can be modified by a method. |
| 1034 | //===--------------------------------------------------------------------===// |
| 1035 | |
| 1036 | /// Return true if this may be a 2- or 3-address |
| 1037 | /// instruction (of the form "X = op Y, Z, ..."), which produces the same |
| 1038 | /// result if Y and Z are exchanged. If this flag is set, then the |
| 1039 | /// TargetInstrInfo::commuteInstruction method may be used to hack on the |
| 1040 | /// instruction. |
| 1041 | /// |
| 1042 | /// Note that this flag may be set on instructions that are only commutable |
| 1043 | /// sometimes. In these cases, the call to commuteInstruction will fail. |
| 1044 | /// Also note that some instructions require non-trivial modification to |
| 1045 | /// commute them. |
| 1046 | bool isCommutable(QueryType Type = IgnoreBundle) const { |
| 1047 | return hasProperty(MCID::Commutable, Type); |
| 1048 | } |
| 1049 | |
| 1050 | /// Return true if this is a 2-address instruction |
| 1051 | /// which can be changed into a 3-address instruction if needed. Doing this |
| 1052 | /// transformation can be profitable in the register allocator, because it |
| 1053 | /// means that the instruction can use a 2-address form if possible, but |
| 1054 | /// degrade into a less efficient form if the source and dest register cannot |
| 1055 | /// be assigned to the same register. For example, this allows the x86 |
| 1056 | /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which |
| 1057 | /// is the same speed as the shift but has bigger code size. |
| 1058 | /// |
| 1059 | /// If this returns true, then the target must implement the |
| 1060 | /// TargetInstrInfo::convertToThreeAddress method for this instruction, which |
| 1061 | /// is allowed to fail if the transformation isn't valid for this specific |
| 1062 | /// instruction (e.g. shl reg, 4 on x86). |
| 1063 | /// |
| 1064 | bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const { |
| 1065 | return hasProperty(MCID::ConvertibleTo3Addr, Type); |
| 1066 | } |
| 1067 | |
| 1068 | /// Return true if this instruction requires |
| 1069 | /// custom insertion support when the DAG scheduler is inserting it into a |
| 1070 | /// machine basic block. If this is true for the instruction, it basically |
| 1071 | /// means that it is a pseudo instruction used at SelectionDAG time that is |
| 1072 | /// expanded out into magic code by the target when MachineInstrs are formed. |
| 1073 | /// |
| 1074 | /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method |
| 1075 | /// is used to insert this into the MachineBasicBlock. |
| 1076 | bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const { |
| 1077 | return hasProperty(MCID::UsesCustomInserter, Type); |
| 1078 | } |
| 1079 | |
| 1080 | /// Return true if this instruction requires *adjustment* |
| 1081 | /// after instruction selection by calling a target hook. For example, this |
| 1082 | /// can be used to fill in ARM 's' optional operand depending on whether |
| 1083 | /// the conditional flag register is used. |
| 1084 | bool hasPostISelHook(QueryType Type = IgnoreBundle) const { |
| 1085 | return hasProperty(MCID::HasPostISelHook, Type); |
| 1086 | } |
| 1087 | |
| 1088 | /// Returns true if this instruction is a candidate for remat. |
| 1089 | /// This flag is deprecated, please don't use it anymore. If this |
| 1090 | /// flag is set, the isReallyTriviallyReMaterializable() method is called to |
| 1091 | /// verify the instruction is really rematable. |
| 1092 | bool isRematerializable(QueryType Type = AllInBundle) const { |
| 1093 | // It's only possible to re-mat a bundle if all bundled instructions are |
| 1094 | // re-materializable. |
| 1095 | return hasProperty(MCID::Rematerializable, Type); |
| 1096 | } |
| 1097 | |
| 1098 | /// Returns true if this instruction has the same cost (or less) than a move |
| 1099 | /// instruction. This is useful during certain types of optimizations |
| 1100 | /// (e.g., remat during two-address conversion or machine licm) |
| 1101 | /// where we would like to remat or hoist the instruction, but not if it costs |
| 1102 | /// more than moving the instruction into the appropriate register. Note, we |
| 1103 | /// are not marking copies from and to the same register class with this flag. |
| 1104 | bool isAsCheapAsAMove(QueryType Type = AllInBundle) const { |
| 1105 | // Only returns true for a bundle if all bundled instructions are cheap. |
| 1106 | return hasProperty(MCID::CheapAsAMove, Type); |
| 1107 | } |
| 1108 | |
| 1109 | /// Returns true if this instruction source operands |
| 1110 | /// have special register allocation requirements that are not captured by the |
| 1111 | /// operand register classes. e.g. ARM::STRD's two source registers must be an |
| 1112 | /// even / odd pair, ARM::STM registers have to be in ascending order. |
| 1113 | /// Post-register allocation passes should not attempt to change allocations |
| 1114 | /// for sources of instructions with this flag. |
| 1115 | bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const { |
| 1116 | return hasProperty(MCID::ExtraSrcRegAllocReq, Type); |
| 1117 | } |
| 1118 | |
| 1119 | /// Returns true if this instruction def operands |
| 1120 | /// have special register allocation requirements that are not captured by the |
| 1121 | /// operand register classes. e.g. ARM::LDRD's two def registers must be an |
| 1122 | /// even / odd pair, ARM::LDM registers have to be in ascending order. |
| 1123 | /// Post-register allocation passes should not attempt to change allocations |
| 1124 | /// for definitions of instructions with this flag. |
| 1125 | bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const { |
| 1126 | return hasProperty(MCID::ExtraDefRegAllocReq, Type); |
| 1127 | } |
| 1128 | |
| 1129 | enum MICheckType { |
| 1130 | CheckDefs, // Check all operands for equality |
| 1131 | CheckKillDead, // Check all operands including kill / dead markers |
| 1132 | IgnoreDefs, // Ignore all definitions |
| 1133 | IgnoreVRegDefs // Ignore virtual register definitions |
| 1134 | }; |
| 1135 | |
| 1136 | /// Return true if this instruction is identical to \p Other. |
| 1137 | /// Two instructions are identical if they have the same opcode and all their |
| 1138 | /// operands are identical (with respect to MachineOperand::isIdenticalTo()). |
| 1139 | /// Note that this means liveness related flags (dead, undef, kill) do not |
| 1140 | /// affect the notion of identical. |
| 1141 | bool isIdenticalTo(const MachineInstr &Other, |
| 1142 | MICheckType Check = CheckDefs) const; |
| 1143 | |
| 1144 | /// Unlink 'this' from the containing basic block, and return it without |
| 1145 | /// deleting it. |
| 1146 | /// |
| 1147 | /// This function can not be used on bundled instructions, use |
| 1148 | /// removeFromBundle() to remove individual instructions from a bundle. |
| 1149 | MachineInstr *removeFromParent(); |
| 1150 | |
| 1151 | /// Unlink this instruction from its basic block and return it without |
| 1152 | /// deleting it. |
| 1153 | /// |
| 1154 | /// If the instruction is part of a bundle, the other instructions in the |
| 1155 | /// bundle remain bundled. |
| 1156 | MachineInstr *removeFromBundle(); |
| 1157 | |
| 1158 | /// Unlink 'this' from the containing basic block and delete it. |
| 1159 | /// |
| 1160 | /// If this instruction is the header of a bundle, the whole bundle is erased. |
| 1161 | /// This function can not be used for instructions inside a bundle, use |
| 1162 | /// eraseFromBundle() to erase individual bundled instructions. |
| 1163 | void eraseFromParent(); |
| 1164 | |
| 1165 | /// Unlink 'this' from the containing basic block and delete it. |
| 1166 | /// |
| 1167 | /// For all definitions mark their uses in DBG_VALUE nodes |
| 1168 | /// as undefined. Otherwise like eraseFromParent(). |
| 1169 | void eraseFromParentAndMarkDBGValuesForRemoval(); |
| 1170 | |
| 1171 | /// Unlink 'this' form its basic block and delete it. |
| 1172 | /// |
| 1173 | /// If the instruction is part of a bundle, the other instructions in the |
| 1174 | /// bundle remain bundled. |
| 1175 | void eraseFromBundle(); |
| 1176 | |
| 1177 | bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } |
| 1178 | bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } |
| 1179 | bool isAnnotationLabel() const { |
| 1180 | return getOpcode() == TargetOpcode::ANNOTATION_LABEL; |
| 1181 | } |
| 1182 | |
| 1183 | /// Returns true if the MachineInstr represents a label. |
| 1184 | bool isLabel() const { |
| 1185 | return isEHLabel() || isGCLabel() || isAnnotationLabel(); |
| 1186 | } |
| 1187 | |
| 1188 | bool isCFIInstruction() const { |
| 1189 | return getOpcode() == TargetOpcode::CFI_INSTRUCTION; |
| 1190 | } |
| 1191 | |
| 1192 | bool isPseudoProbe() const { |
| 1193 | return getOpcode() == TargetOpcode::PSEUDO_PROBE; |
| 1194 | } |
| 1195 | |
| 1196 | // True if the instruction represents a position in the function. |
| 1197 | bool isPosition() const { return isLabel() || isCFIInstruction(); } |
| 1198 | |
| 1199 | bool isNonListDebugValue() const { |
| 1200 | return getOpcode() == TargetOpcode::DBG_VALUE; |
| 1201 | } |
| 1202 | bool isDebugValueList() const { |
| 1203 | return getOpcode() == TargetOpcode::DBG_VALUE_LIST; |
| 1204 | } |
| 1205 | bool isDebugValue() const { |
| 1206 | return isNonListDebugValue() || isDebugValueList(); |
| 1207 | } |
| 1208 | bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; } |
| 1209 | bool isDebugRef() const { return getOpcode() == TargetOpcode::DBG_INSTR_REF; } |
| 1210 | bool isDebugInstr() const { |
| 1211 | return isDebugValue() || isDebugLabel() || isDebugRef(); |
| 1212 | } |
| 1213 | bool isDebugOrPseudoInstr() const { |
| 1214 | return isDebugInstr() || isPseudoProbe(); |
| 1215 | } |
| 1216 | |
| 1217 | bool isDebugOffsetImm() const { |
| 1218 | return isNonListDebugValue() && getDebugOffset().isImm(); |
| 1219 | } |
| 1220 | |
| 1221 | /// A DBG_VALUE is indirect iff the location operand is a register and |
| 1222 | /// the offset operand is an immediate. |
| 1223 | bool isIndirectDebugValue() const { |
| 1224 | return isDebugOffsetImm() && getDebugOperand(0).isReg(); |
| 1225 | } |
| 1226 | |
| 1227 | /// A DBG_VALUE is an entry value iff its debug expression contains the |
| 1228 | /// DW_OP_LLVM_entry_value operation. |
| 1229 | bool isDebugEntryValue() const; |
| 1230 | |
| 1231 | /// Return true if the instruction is a debug value which describes a part of |
| 1232 | /// a variable as unavailable. |
| 1233 | bool isUndefDebugValue() const { |
| 1234 | if (!isDebugValue()) |
| 1235 | return false; |
| 1236 | // If any $noreg locations are given, this DV is undef. |
| 1237 | for (const MachineOperand &Op : debug_operands()) |
| 1238 | if (Op.isReg() && !Op.getReg().isValid()) |
| 1239 | return true; |
| 1240 | return false; |
| 1241 | } |
| 1242 | |
| 1243 | bool isPHI() const { |
| 1244 | return getOpcode() == TargetOpcode::PHI || |
| 1245 | getOpcode() == TargetOpcode::G_PHI; |
| 1246 | } |
| 1247 | bool isKill() const { return getOpcode() == TargetOpcode::KILL; } |
| 1248 | bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } |
| 1249 | bool isInlineAsm() const { |
| 1250 | return getOpcode() == TargetOpcode::INLINEASM || |
| 1251 | getOpcode() == TargetOpcode::INLINEASM_BR; |
| 1252 | } |
| 1253 | |
| 1254 | /// FIXME: Seems like a layering violation that the AsmDialect, which is X86 |
| 1255 | /// specific, be attached to a generic MachineInstr. |
| 1256 | bool isMSInlineAsm() const { |
| 1257 | return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; |
| 1258 | } |
| 1259 | |
| 1260 | bool isStackAligningInlineAsm() const; |
| 1261 | InlineAsm::AsmDialect getInlineAsmDialect() const; |
| 1262 | |
| 1263 | bool isInsertSubreg() const { |
| 1264 | return getOpcode() == TargetOpcode::INSERT_SUBREG; |
| 1265 | } |
| 1266 | |
| 1267 | bool isSubregToReg() const { |
| 1268 | return getOpcode() == TargetOpcode::SUBREG_TO_REG; |
| 1269 | } |
| 1270 | |
| 1271 | bool isRegSequence() const { |
| 1272 | return getOpcode() == TargetOpcode::REG_SEQUENCE; |
| 1273 | } |
| 1274 | |
| 1275 | bool isBundle() const { |
| 1276 | return getOpcode() == TargetOpcode::BUNDLE; |
| 1277 | } |
| 1278 | |
| 1279 | bool isCopy() const { |
| 1280 | return getOpcode() == TargetOpcode::COPY; |
| 1281 | } |
| 1282 | |
| 1283 | bool isFullCopy() const { |
| 1284 | return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg(); |
| 1285 | } |
| 1286 | |
| 1287 | bool isExtractSubreg() const { |
| 1288 | return getOpcode() == TargetOpcode::EXTRACT_SUBREG; |
| 1289 | } |
| 1290 | |
| 1291 | /// Return true if the instruction behaves like a copy. |
| 1292 | /// This does not include native copy instructions. |
| 1293 | bool isCopyLike() const { |
| 1294 | return isCopy() || isSubregToReg(); |
| 1295 | } |
| 1296 | |
| 1297 | /// Return true is the instruction is an identity copy. |
| 1298 | bool isIdentityCopy() const { |
| 1299 | return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() && |
| 1300 | getOperand(0).getSubReg() == getOperand(1).getSubReg(); |
| 1301 | } |
| 1302 | |
| 1303 | /// Return true if this instruction doesn't produce any output in the form of |
| 1304 | /// executable instructions. |
| 1305 | bool isMetaInstruction() const { |
| 1306 | switch (getOpcode()) { |
| 1307 | default: |
| 1308 | return false; |
| 1309 | case TargetOpcode::IMPLICIT_DEF: |
| 1310 | case TargetOpcode::KILL: |
| 1311 | case TargetOpcode::CFI_INSTRUCTION: |
| 1312 | case TargetOpcode::EH_LABEL: |
| 1313 | case TargetOpcode::GC_LABEL: |
| 1314 | case TargetOpcode::DBG_VALUE: |
| 1315 | case TargetOpcode::DBG_VALUE_LIST: |
| 1316 | case TargetOpcode::DBG_INSTR_REF: |
| 1317 | case TargetOpcode::DBG_LABEL: |
| 1318 | case TargetOpcode::LIFETIME_START: |
| 1319 | case TargetOpcode::LIFETIME_END: |
| 1320 | case TargetOpcode::PSEUDO_PROBE: |
| 1321 | return true; |
| 1322 | } |
| 1323 | } |
| 1324 | |
| 1325 | /// Return true if this is a transient instruction that is either very likely |
| 1326 | /// to be eliminated during register allocation (such as copy-like |
| 1327 | /// instructions), or if this instruction doesn't have an execution-time cost. |
| 1328 | bool isTransient() const { |
| 1329 | switch (getOpcode()) { |
| 1330 | default: |
| 1331 | return isMetaInstruction(); |
| 1332 | // Copy-like instructions are usually eliminated during register allocation. |
| 1333 | case TargetOpcode::PHI: |
| 1334 | case TargetOpcode::G_PHI: |
| 1335 | case TargetOpcode::COPY: |
| 1336 | case TargetOpcode::INSERT_SUBREG: |
| 1337 | case TargetOpcode::SUBREG_TO_REG: |
| 1338 | case TargetOpcode::REG_SEQUENCE: |
| 1339 | return true; |
| 1340 | } |
| 1341 | } |
| 1342 | |
| 1343 | /// Return the number of instructions inside the MI bundle, excluding the |
| 1344 | /// bundle header. |
| 1345 | /// |
| 1346 | /// This is the number of instructions that MachineBasicBlock::iterator |
| 1347 | /// skips, 0 for unbundled instructions. |
| 1348 | unsigned getBundleSize() const; |
| 1349 | |
| 1350 | /// Return true if the MachineInstr reads the specified register. |
| 1351 | /// If TargetRegisterInfo is passed, then it also checks if there |
| 1352 | /// is a read of a super-register. |
| 1353 | /// This does not count partial redefines of virtual registers as reads: |
| 1354 | /// %reg1024:6 = OP. |
| 1355 | bool readsRegister(Register Reg, |
| 1356 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1357 | return findRegisterUseOperandIdx(Reg, false, TRI) != -1; |
| 1358 | } |
| 1359 | |
| 1360 | /// Return true if the MachineInstr reads the specified virtual register. |
| 1361 | /// Take into account that a partial define is a |
| 1362 | /// read-modify-write operation. |
| 1363 | bool readsVirtualRegister(Register Reg) const { |
| 1364 | return readsWritesVirtualRegister(Reg).first; |
| 1365 | } |
| 1366 | |
| 1367 | /// Return a pair of bools (reads, writes) indicating if this instruction |
| 1368 | /// reads or writes Reg. This also considers partial defines. |
| 1369 | /// If Ops is not null, all operand indices for Reg are added. |
| 1370 | std::pair<bool,bool> readsWritesVirtualRegister(Register Reg, |
| 1371 | SmallVectorImpl<unsigned> *Ops = nullptr) const; |
| 1372 | |
| 1373 | /// Return true if the MachineInstr kills the specified register. |
| 1374 | /// If TargetRegisterInfo is passed, then it also checks if there is |
| 1375 | /// a kill of a super-register. |
| 1376 | bool killsRegister(Register Reg, |
| 1377 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1378 | return findRegisterUseOperandIdx(Reg, true, TRI) != -1; |
| 1379 | } |
| 1380 | |
| 1381 | /// Return true if the MachineInstr fully defines the specified register. |
| 1382 | /// If TargetRegisterInfo is passed, then it also checks |
| 1383 | /// if there is a def of a super-register. |
| 1384 | /// NOTE: It's ignoring subreg indices on virtual registers. |
| 1385 | bool definesRegister(Register Reg, |
| 1386 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1387 | return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; |
| 1388 | } |
| 1389 | |
| 1390 | /// Return true if the MachineInstr modifies (fully define or partially |
| 1391 | /// define) the specified register. |
| 1392 | /// NOTE: It's ignoring subreg indices on virtual registers. |
| 1393 | bool modifiesRegister(Register Reg, |
| 1394 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1395 | return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; |
| 1396 | } |
| 1397 | |
| 1398 | /// Returns true if the register is dead in this machine instruction. |
| 1399 | /// If TargetRegisterInfo is passed, then it also checks |
| 1400 | /// if there is a dead def of a super-register. |
| 1401 | bool registerDefIsDead(Register Reg, |
| 1402 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1403 | return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; |
| 1404 | } |
| 1405 | |
| 1406 | /// Returns true if the MachineInstr has an implicit-use operand of exactly |
| 1407 | /// the given register (not considering sub/super-registers). |
| 1408 | bool hasRegisterImplicitUseOperand(Register Reg) const; |
| 1409 | |
| 1410 | /// Returns the operand index that is a use of the specific register or -1 |
| 1411 | /// if it is not found. It further tightens the search criteria to a use |
| 1412 | /// that kills the register if isKill is true. |
| 1413 | int findRegisterUseOperandIdx(Register Reg, bool isKill = false, |
| 1414 | const TargetRegisterInfo *TRI = nullptr) const; |
| 1415 | |
| 1416 | /// Wrapper for findRegisterUseOperandIdx, it returns |
| 1417 | /// a pointer to the MachineOperand rather than an index. |
| 1418 | MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false, |
| 1419 | const TargetRegisterInfo *TRI = nullptr) { |
| 1420 | int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI); |
| 1421 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
| 1422 | } |
| 1423 | |
| 1424 | const MachineOperand *findRegisterUseOperand( |
| 1425 | Register Reg, bool isKill = false, |
| 1426 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1427 | return const_cast<MachineInstr *>(this)-> |
| 1428 | findRegisterUseOperand(Reg, isKill, TRI); |
| 1429 | } |
| 1430 | |
| 1431 | /// Returns the operand index that is a def of the specified register or |
| 1432 | /// -1 if it is not found. If isDead is true, defs that are not dead are |
| 1433 | /// skipped. If Overlap is true, then it also looks for defs that merely |
| 1434 | /// overlap the specified register. If TargetRegisterInfo is non-null, |
| 1435 | /// then it also checks if there is a def of a super-register. |
| 1436 | /// This may also return a register mask operand when Overlap is true. |
| 1437 | int findRegisterDefOperandIdx(Register Reg, |
| 1438 | bool isDead = false, bool Overlap = false, |
| 1439 | const TargetRegisterInfo *TRI = nullptr) const; |
| 1440 | |
| 1441 | /// Wrapper for findRegisterDefOperandIdx, it returns |
| 1442 | /// a pointer to the MachineOperand rather than an index. |
| 1443 | MachineOperand * |
| 1444 | findRegisterDefOperand(Register Reg, bool isDead = false, |
| 1445 | bool Overlap = false, |
| 1446 | const TargetRegisterInfo *TRI = nullptr) { |
| 1447 | int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); |
| 1448 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
| 1449 | } |
| 1450 | |
| 1451 | const MachineOperand * |
| 1452 | findRegisterDefOperand(Register Reg, bool isDead = false, |
| 1453 | bool Overlap = false, |
| 1454 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1455 | return const_cast<MachineInstr *>(this)->findRegisterDefOperand( |
| 1456 | Reg, isDead, Overlap, TRI); |
| 1457 | } |
| 1458 | |
| 1459 | /// Find the index of the first operand in the |
| 1460 | /// operand list that is used to represent the predicate. It returns -1 if |
| 1461 | /// none is found. |
| 1462 | int findFirstPredOperandIdx() const; |
| 1463 | |
| 1464 | /// Find the index of the flag word operand that |
| 1465 | /// corresponds to operand OpIdx on an inline asm instruction. Returns -1 if |
| 1466 | /// getOperand(OpIdx) does not belong to an inline asm operand group. |
| 1467 | /// |
| 1468 | /// If GroupNo is not NULL, it will receive the number of the operand group |
| 1469 | /// containing OpIdx. |
| 1470 | /// |
| 1471 | /// The flag operand is an immediate that can be decoded with methods like |
| 1472 | /// InlineAsm::hasRegClassConstraint(). |
| 1473 | int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const; |
| 1474 | |
| 1475 | /// Compute the static register class constraint for operand OpIdx. |
| 1476 | /// For normal instructions, this is derived from the MCInstrDesc. |
| 1477 | /// For inline assembly it is derived from the flag words. |
| 1478 | /// |
| 1479 | /// Returns NULL if the static register class constraint cannot be |
| 1480 | /// determined. |
| 1481 | const TargetRegisterClass* |
| 1482 | getRegClassConstraint(unsigned OpIdx, |
| 1483 | const TargetInstrInfo *TII, |
| 1484 | const TargetRegisterInfo *TRI) const; |
| 1485 | |
| 1486 | /// Applies the constraints (def/use) implied by this MI on \p Reg to |
| 1487 | /// the given \p CurRC. |
| 1488 | /// If \p ExploreBundle is set and MI is part of a bundle, all the |
| 1489 | /// instructions inside the bundle will be taken into account. In other words, |
| 1490 | /// this method accumulates all the constraints of the operand of this MI and |
| 1491 | /// the related bundle if MI is a bundle or inside a bundle. |
| 1492 | /// |
| 1493 | /// Returns the register class that satisfies both \p CurRC and the |
| 1494 | /// constraints set by MI. Returns NULL if such a register class does not |
| 1495 | /// exist. |
| 1496 | /// |
| 1497 | /// \pre CurRC must not be NULL. |
| 1498 | const TargetRegisterClass *getRegClassConstraintEffectForVReg( |
| 1499 | Register Reg, const TargetRegisterClass *CurRC, |
| 1500 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, |
| 1501 | bool ExploreBundle = false) const; |
| 1502 | |
| 1503 | /// Applies the constraints (def/use) implied by the \p OpIdx operand |
| 1504 | /// to the given \p CurRC. |
| 1505 | /// |
| 1506 | /// Returns the register class that satisfies both \p CurRC and the |
| 1507 | /// constraints set by \p OpIdx MI. Returns NULL if such a register class |
| 1508 | /// does not exist. |
| 1509 | /// |
| 1510 | /// \pre CurRC must not be NULL. |
| 1511 | /// \pre The operand at \p OpIdx must be a register. |
| 1512 | const TargetRegisterClass * |
| 1513 | getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC, |
| 1514 | const TargetInstrInfo *TII, |
| 1515 | const TargetRegisterInfo *TRI) const; |
| 1516 | |
| 1517 | /// Add a tie between the register operands at DefIdx and UseIdx. |
| 1518 | /// The tie will cause the register allocator to ensure that the two |
| 1519 | /// operands are assigned the same physical register. |
| 1520 | /// |
| 1521 | /// Tied operands are managed automatically for explicit operands in the |
| 1522 | /// MCInstrDesc. This method is for exceptional cases like inline asm. |
| 1523 | void tieOperands(unsigned DefIdx, unsigned UseIdx); |
| 1524 | |
| 1525 | /// Given the index of a tied register operand, find the |
| 1526 | /// operand it is tied to. Defs are tied to uses and vice versa. Returns the |
| 1527 | /// index of the tied operand which must exist. |
| 1528 | unsigned findTiedOperandIdx(unsigned OpIdx) const; |
| 1529 | |
| 1530 | /// Given the index of a register def operand, |
| 1531 | /// check if the register def is tied to a source operand, due to either |
| 1532 | /// two-address elimination or inline assembly constraints. Returns the |
| 1533 | /// first tied use operand index by reference if UseOpIdx is not null. |
| 1534 | bool isRegTiedToUseOperand(unsigned DefOpIdx, |
| 1535 | unsigned *UseOpIdx = nullptr) const { |
| 1536 | const MachineOperand &MO = getOperand(DefOpIdx); |
| 1537 | if (!MO.isReg() || !MO.isDef() || !MO.isTied()) |
| 1538 | return false; |
| 1539 | if (UseOpIdx) |
| 1540 | *UseOpIdx = findTiedOperandIdx(DefOpIdx); |
| 1541 | return true; |
| 1542 | } |
| 1543 | |
| 1544 | /// Return true if the use operand of the specified index is tied to a def |
| 1545 | /// operand. It also returns the def operand index by reference if DefOpIdx |
| 1546 | /// is not null. |
| 1547 | bool isRegTiedToDefOperand(unsigned UseOpIdx, |
| 1548 | unsigned *DefOpIdx = nullptr) const { |
| 1549 | const MachineOperand &MO = getOperand(UseOpIdx); |
| 1550 | if (!MO.isReg() || !MO.isUse() || !MO.isTied()) |
| 1551 | return false; |
| 1552 | if (DefOpIdx) |
| 1553 | *DefOpIdx = findTiedOperandIdx(UseOpIdx); |
| 1554 | return true; |
| 1555 | } |
| 1556 | |
| 1557 | /// Clears kill flags on all operands. |
| 1558 | void clearKillInfo(); |
| 1559 | |
| 1560 | /// Replace all occurrences of FromReg with ToReg:SubIdx, |
| 1561 | /// properly composing subreg indices where necessary. |
| 1562 | void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, |
| 1563 | const TargetRegisterInfo &RegInfo); |
| 1564 | |
| 1565 | /// We have determined MI kills a register. Look for the |
| 1566 | /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, |
| 1567 | /// add a implicit operand if it's not found. Returns true if the operand |
| 1568 | /// exists / is added. |
| 1569 | bool addRegisterKilled(Register IncomingReg, |
| 1570 | const TargetRegisterInfo *RegInfo, |
| 1571 | bool AddIfNotFound = false); |
| 1572 | |
| 1573 | /// Clear all kill flags affecting Reg. If RegInfo is provided, this includes |
| 1574 | /// all aliasing registers. |
| 1575 | void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo); |
| 1576 | |
| 1577 | /// We have determined MI defined a register without a use. |
| 1578 | /// Look for the operand that defines it and mark it as IsDead. If |
| 1579 | /// AddIfNotFound is true, add a implicit operand if it's not found. Returns |
| 1580 | /// true if the operand exists / is added. |
| 1581 | bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, |
| 1582 | bool AddIfNotFound = false); |
| 1583 | |
| 1584 | /// Clear all dead flags on operands defining register @p Reg. |
| 1585 | void clearRegisterDeads(Register Reg); |
| 1586 | |
| 1587 | /// Mark all subregister defs of register @p Reg with the undef flag. |
| 1588 | /// This function is used when we determined to have a subregister def in an |
| 1589 | /// otherwise undefined super register. |
| 1590 | void setRegisterDefReadUndef(Register Reg, bool IsUndef = true); |
| 1591 | |
| 1592 | /// We have determined MI defines a register. Make sure there is an operand |
| 1593 | /// defining Reg. |
| 1594 | void addRegisterDefined(Register Reg, |
| 1595 | const TargetRegisterInfo *RegInfo = nullptr); |
| 1596 | |
| 1597 | /// Mark every physreg used by this instruction as |
| 1598 | /// dead except those in the UsedRegs list. |
| 1599 | /// |
| 1600 | /// On instructions with register mask operands, also add implicit-def |
| 1601 | /// operands for all registers in UsedRegs. |
| 1602 | void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs, |
| 1603 | const TargetRegisterInfo &TRI); |
| 1604 | |
| 1605 | /// Return true if it is safe to move this instruction. If |
| 1606 | /// SawStore is set to true, it means that there is a store (or call) between |
| 1607 | /// the instruction's location and its intended destination. |
| 1608 | bool isSafeToMove(AAResults *AA, bool &SawStore) const; |
| 1609 | |
| 1610 | /// Returns true if this instruction's memory access aliases the memory |
| 1611 | /// access of Other. |
| 1612 | // |
| 1613 | /// Assumes any physical registers used to compute addresses |
| 1614 | /// have the same value for both instructions. Returns false if neither |
| 1615 | /// instruction writes to memory. |
| 1616 | /// |
| 1617 | /// @param AA Optional alias analysis, used to compare memory operands. |
| 1618 | /// @param Other MachineInstr to check aliasing against. |
| 1619 | /// @param UseTBAA Whether to pass TBAA information to alias analysis. |
| 1620 | bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; |
| 1621 | |
| 1622 | /// Return true if this instruction may have an ordered |
| 1623 | /// or volatile memory reference, or if the information describing the memory |
| 1624 | /// reference is not available. Return false if it is known to have no |
| 1625 | /// ordered or volatile memory references. |
| 1626 | bool hasOrderedMemoryRef() const; |
| 1627 | |
| 1628 | /// Return true if this load instruction never traps and points to a memory |
| 1629 | /// location whose value doesn't change during the execution of this function. |
| 1630 | /// |
| 1631 | /// Examples include loading a value from the constant pool or from the |
| 1632 | /// argument area of a function (if it does not change). If the instruction |
| 1633 | /// does multiple loads, this returns true only if all of the loads are |
| 1634 | /// dereferenceable and invariant. |
| 1635 | bool isDereferenceableInvariantLoad(AAResults *AA) const; |
| 1636 | |
| 1637 | /// If the specified instruction is a PHI that always merges together the |
| 1638 | /// same virtual register, return the register, otherwise return 0. |
| 1639 | unsigned isConstantValuePHI() const; |
| 1640 | |
| 1641 | /// Return true if this instruction has side effects that are not modeled |
| 1642 | /// by mayLoad / mayStore, etc. |
| 1643 | /// For all instructions, the property is encoded in MCInstrDesc::Flags |
| 1644 | /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is |
| 1645 | /// INLINEASM instruction, in which case the side effect property is encoded |
| 1646 | /// in one of its operands (see InlineAsm::Extra_HasSideEffect). |
| 1647 | /// |
| 1648 | bool hasUnmodeledSideEffects() const; |
| 1649 | |
| 1650 | /// Returns true if it is illegal to fold a load across this instruction. |
| 1651 | bool isLoadFoldBarrier() const; |
| 1652 | |
| 1653 | /// Return true if all the defs of this instruction are dead. |
| 1654 | bool allDefsAreDead() const; |
| 1655 | |
| 1656 | /// Return a valid size if the instruction is a spill instruction. |
| 1657 | Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const; |
| 1658 | |
| 1659 | /// Return a valid size if the instruction is a folded spill instruction. |
| 1660 | Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const; |
| 1661 | |
| 1662 | /// Return a valid size if the instruction is a restore instruction. |
| 1663 | Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const; |
| 1664 | |
| 1665 | /// Return a valid size if the instruction is a folded restore instruction. |
| 1666 | Optional<unsigned> |
| 1667 | getFoldedRestoreSize(const TargetInstrInfo *TII) const; |
| 1668 | |
| 1669 | /// Copy implicit register operands from specified |
| 1670 | /// instruction to this instruction. |
| 1671 | void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI); |
| 1672 | |
| 1673 | /// Debugging support |
| 1674 | /// @{ |
| 1675 | /// Determine the generic type to be printed (if needed) on uses and defs. |
| 1676 | LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes, |
| 1677 | const MachineRegisterInfo &MRI) const; |
| 1678 | |
| 1679 | /// Return true when an instruction has tied register that can't be determined |
| 1680 | /// by the instruction's descriptor. This is useful for MIR printing, to |
| 1681 | /// determine whether we need to print the ties or not. |
| 1682 | bool hasComplexRegisterTies() const; |
| 1683 | |
| 1684 | /// Print this MI to \p OS. |
| 1685 | /// Don't print information that can be inferred from other instructions if |
| 1686 | /// \p IsStandalone is false. It is usually true when only a fragment of the |
| 1687 | /// function is printed. |
| 1688 | /// Only print the defs and the opcode if \p SkipOpers is true. |
| 1689 | /// Otherwise, also print operands if \p SkipDebugLoc is true. |
| 1690 | /// Otherwise, also print the debug loc, with a terminating newline. |
| 1691 | /// \p TII is used to print the opcode name. If it's not present, but the |
| 1692 | /// MI is in a function, the opcode will be printed using the function's TII. |
| 1693 | void print(raw_ostream &OS, bool IsStandalone = true, bool SkipOpers = false, |
| 1694 | bool SkipDebugLoc = false, bool AddNewLine = true, |
| 1695 | const TargetInstrInfo *TII = nullptr) const; |
| 1696 | void print(raw_ostream &OS, ModuleSlotTracker &MST, bool IsStandalone = true, |
| 1697 | bool SkipOpers = false, bool SkipDebugLoc = false, |
| 1698 | bool AddNewLine = true, |
| 1699 | const TargetInstrInfo *TII = nullptr) const; |
| 1700 | void dump() const; |
| 1701 | /// Print on dbgs() the current instruction and the instructions defining its |
| 1702 | /// operands and so on until we reach \p MaxDepth. |
| 1703 | void dumpr(const MachineRegisterInfo &MRI, |
| 1704 | unsigned MaxDepth = UINT_MAX(2147483647 *2U +1U)) const; |
| 1705 | /// @} |
| 1706 | |
| 1707 | //===--------------------------------------------------------------------===// |
| 1708 | // Accessors used to build up machine instructions. |
| 1709 | |
| 1710 | /// Add the specified operand to the instruction. If it is an implicit |
| 1711 | /// operand, it is added to the end of the operand list. If it is an |
| 1712 | /// explicit operand it is added at the end of the explicit operand list |
| 1713 | /// (before the first implicit operand). |
| 1714 | /// |
| 1715 | /// MF must be the machine function that was used to allocate this |
| 1716 | /// instruction. |
| 1717 | /// |
| 1718 | /// MachineInstrBuilder provides a more convenient interface for creating |
| 1719 | /// instructions and adding operands. |
| 1720 | void addOperand(MachineFunction &MF, const MachineOperand &Op); |
| 1721 | |
| 1722 | /// Add an operand without providing an MF reference. This only works for |
| 1723 | /// instructions that are inserted in a basic block. |
| 1724 | /// |
| 1725 | /// MachineInstrBuilder and the two-argument addOperand(MF, MO) should be |
| 1726 | /// preferred. |
| 1727 | void addOperand(const MachineOperand &Op); |
| 1728 | |
| 1729 | /// Replace the instruction descriptor (thus opcode) of |
| 1730 | /// the current instruction with a new one. |
| 1731 | void setDesc(const MCInstrDesc &tid) { MCID = &tid; } |
| 1732 | |
| 1733 | /// Replace current source information with new such. |
| 1734 | /// Avoid using this, the constructor argument is preferable. |
| 1735 | void setDebugLoc(DebugLoc dl) { |
| 1736 | debugLoc = std::move(dl); |
| 1737 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor" ) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 1737, __PRETTY_FUNCTION__)); |
| 1738 | } |
| 1739 | |
| 1740 | /// Erase an operand from an instruction, leaving it with one |
| 1741 | /// fewer operand than it started with. |
| 1742 | void RemoveOperand(unsigned OpNo); |
| 1743 | |
| 1744 | /// Clear this MachineInstr's memory reference descriptor list. This resets |
| 1745 | /// the memrefs to their most conservative state. This should be used only |
| 1746 | /// as a last resort since it greatly pessimizes our knowledge of the memory |
| 1747 | /// access performed by the instruction. |
| 1748 | void dropMemRefs(MachineFunction &MF); |
| 1749 | |
| 1750 | /// Assign this MachineInstr's memory reference descriptor list. |
| 1751 | /// |
| 1752 | /// Unlike other methods, this *will* allocate them into a new array |
| 1753 | /// associated with the provided `MachineFunction`. |
| 1754 | void setMemRefs(MachineFunction &MF, ArrayRef<MachineMemOperand *> MemRefs); |
| 1755 | |
| 1756 | /// Add a MachineMemOperand to the machine instruction. |
| 1757 | /// This function should be used only occasionally. The setMemRefs function |
| 1758 | /// is the primary method for setting up a MachineInstr's MemRefs list. |
| 1759 | void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); |
| 1760 | |
| 1761 | /// Clone another MachineInstr's memory reference descriptor list and replace |
| 1762 | /// ours with it. |
| 1763 | /// |
| 1764 | /// Note that `*this` may be the incoming MI! |
| 1765 | /// |
| 1766 | /// Prefer this API whenever possible as it can avoid allocations in common |
| 1767 | /// cases. |
| 1768 | void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); |
| 1769 | |
| 1770 | /// Clone the merge of multiple MachineInstrs' memory reference descriptors |
| 1771 | /// list and replace ours with it. |
| 1772 | /// |
| 1773 | /// Note that `*this` may be one of the incoming MIs! |
| 1774 | /// |
| 1775 | /// Prefer this API whenever possible as it can avoid allocations in common |
| 1776 | /// cases. |
| 1777 | void cloneMergedMemRefs(MachineFunction &MF, |
| 1778 | ArrayRef<const MachineInstr *> MIs); |
| 1779 | |
| 1780 | /// Set a symbol that will be emitted just prior to the instruction itself. |
| 1781 | /// |
| 1782 | /// Setting this to a null pointer will remove any such symbol. |
| 1783 | /// |
| 1784 | /// FIXME: This is not fully implemented yet. |
| 1785 | void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
| 1786 | |
| 1787 | /// Set a symbol that will be emitted just after the instruction itself. |
| 1788 | /// |
| 1789 | /// Setting this to a null pointer will remove any such symbol. |
| 1790 | /// |
| 1791 | /// FIXME: This is not fully implemented yet. |
| 1792 | void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
| 1793 | |
| 1794 | /// Clone another MachineInstr's pre- and post- instruction symbols and |
| 1795 | /// replace ours with it. |
| 1796 | void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); |
| 1797 | |
| 1798 | /// Set a marker on instructions that denotes where we should create and emit |
| 1799 | /// heap alloc site labels. This waits until after instruction selection and |
| 1800 | /// optimizations to create the label, so it should still work if the |
| 1801 | /// instruction is removed or duplicated. |
| 1802 | void setHeapAllocMarker(MachineFunction &MF, MDNode *MD); |
| 1803 | |
| 1804 | /// Return the MIFlags which represent both MachineInstrs. This |
| 1805 | /// should be used when merging two MachineInstrs into one. This routine does |
| 1806 | /// not modify the MIFlags of this MachineInstr. |
| 1807 | uint16_t mergeFlagsWith(const MachineInstr& Other) const; |
| 1808 | |
| 1809 | static uint16_t copyFlagsFromInstruction(const Instruction &I); |
| 1810 | |
| 1811 | /// Copy all flags to MachineInst MIFlags |
| 1812 | void copyIRFlags(const Instruction &I); |
| 1813 | |
| 1814 | /// Break any tie involving OpIdx. |
| 1815 | void untieRegOperand(unsigned OpIdx) { |
| 1816 | MachineOperand &MO = getOperand(OpIdx); |
| 1817 | if (MO.isReg() && MO.isTied()) { |
| 1818 | getOperand(findTiedOperandIdx(OpIdx)).TiedTo = 0; |
| 1819 | MO.TiedTo = 0; |
| 1820 | } |
| 1821 | } |
| 1822 | |
| 1823 | /// Add all implicit def and use operands to this instruction. |
| 1824 | void addImplicitDefUseOperands(MachineFunction &MF); |
| 1825 | |
| 1826 | /// Scan instructions immediately following MI and collect any matching |
| 1827 | /// DBG_VALUEs. |
| 1828 | void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); |
| 1829 | |
| 1830 | /// Find all DBG_VALUEs that point to the register def in this instruction |
| 1831 | /// and point them to \p Reg instead. |
| 1832 | void changeDebugValuesDefReg(Register Reg); |
| 1833 | |
| 1834 | /// Returns the Intrinsic::ID for this instruction. |
| 1835 | /// \pre Must have an intrinsic ID operand. |
| 1836 | unsigned getIntrinsicID() const { |
| 1837 | return getOperand(getNumExplicitDefs()).getIntrinsicID(); |
| 1838 | } |
| 1839 | |
| 1840 | /// Sets all register debug operands in this debug value instruction to be |
| 1841 | /// undef. |
| 1842 | void setDebugValueUndef() { |
| 1843 | assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction." ) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 1843, __PRETTY_FUNCTION__)); |
| 1844 | for (MachineOperand &MO : debug_operands()) { |
| 1845 | if (MO.isReg()) { |
| 1846 | MO.setReg(0); |
| 1847 | MO.setSubReg(0); |
| 1848 | } |
| 1849 | } |
| 1850 | } |
| 1851 | |
| 1852 | PseudoProbeAttributes getPseudoProbeAttribute() const { |
| 1853 | assert(isPseudoProbe() && "Must be a pseudo probe instruction")((isPseudoProbe() && "Must be a pseudo probe instruction" ) ? static_cast<void> (0) : __assert_fail ("isPseudoProbe() && \"Must be a pseudo probe instruction\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 1853, __PRETTY_FUNCTION__)); |
| 1854 | return (PseudoProbeAttributes)getOperand(3).getImm(); |
| 1855 | } |
| 1856 | |
| 1857 | void addPseudoProbeAttribute(PseudoProbeAttributes Attr) { |
| 1858 | assert(isPseudoProbe() && "Must be a pseudo probe instruction")((isPseudoProbe() && "Must be a pseudo probe instruction" ) ? static_cast<void> (0) : __assert_fail ("isPseudoProbe() && \"Must be a pseudo probe instruction\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/MachineInstr.h" , 1858, __PRETTY_FUNCTION__)); |
| 1859 | MachineOperand &AttrOperand = getOperand(3); |
| 1860 | AttrOperand.setImm(AttrOperand.getImm() | (uint32_t)Attr); |
| 1861 | } |
| 1862 | |
| 1863 | private: |
| 1864 | /// If this instruction is embedded into a MachineFunction, return the |
| 1865 | /// MachineRegisterInfo object for the current function, otherwise |
| 1866 | /// return null. |
| 1867 | MachineRegisterInfo *getRegInfo(); |
| 1868 | |
| 1869 | /// Unlink all of the register operands in this instruction from their |
| 1870 | /// respective use lists. This requires that the operands already be on their |
| 1871 | /// use lists. |
| 1872 | void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); |
| 1873 | |
| 1874 | /// Add all of the register operands in this instruction from their |
| 1875 | /// respective use lists. This requires that the operands not be on their |
| 1876 | /// use lists yet. |
| 1877 | void AddRegOperandsToUseLists(MachineRegisterInfo&); |
| 1878 | |
| 1879 | /// Slow path for hasProperty when we're dealing with a bundle. |
| 1880 | bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; |
| 1881 | |
| 1882 | /// Implements the logic of getRegClassConstraintEffectForVReg for the |
| 1883 | /// this MI and the given operand index \p OpIdx. |
| 1884 | /// If the related operand does not constrained Reg, this returns CurRC. |
| 1885 | const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( |
| 1886 | unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, |
| 1887 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; |
| 1888 | |
| 1889 | /// Stores extra instruction information inline or allocates as ExtraInfo |
| 1890 | /// based on the number of pointers. |
| 1891 | void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs, |
| 1892 | MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, |
| 1893 | MDNode *HeapAllocMarker); |
| 1894 | }; |
| 1895 | |
| 1896 | /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the |
| 1897 | /// instruction rather than by pointer value. |
| 1898 | /// The hashing and equality testing functions ignore definitions so this is |
| 1899 | /// useful for CSE, etc. |
| 1900 | struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> { |
| 1901 | static inline MachineInstr *getEmptyKey() { |
| 1902 | return nullptr; |
| 1903 | } |
| 1904 | |
| 1905 | static inline MachineInstr *getTombstoneKey() { |
| 1906 | return reinterpret_cast<MachineInstr*>(-1); |
| 1907 | } |
| 1908 | |
| 1909 | static unsigned getHashValue(const MachineInstr* const &MI); |
| 1910 | |
| 1911 | static bool isEqual(const MachineInstr* const &LHS, |
| 1912 | const MachineInstr* const &RHS) { |
| 1913 | if (RHS == getEmptyKey() || RHS == getTombstoneKey() || |
| 1914 | LHS == getEmptyKey() || LHS == getTombstoneKey()) |
| 1915 | return LHS == RHS; |
| 1916 | return LHS->isIdenticalTo(*RHS, MachineInstr::IgnoreVRegDefs); |
| 1917 | } |
| 1918 | }; |
| 1919 | |
| 1920 | //===----------------------------------------------------------------------===// |
| 1921 | // Debugging Support |
| 1922 | |
| 1923 | inline raw_ostream& operator<<(raw_ostream &OS, const MachineInstr &MI) { |
| 1924 | MI.print(OS); |
| 1925 | return OS; |
| 1926 | } |
| 1927 | |
| 1928 | } // end namespace llvm |
| 1929 | |
| 1930 | #endif // LLVM_CODEGEN_MACHINEINSTR_H |
| 1 | //===-- llvm/CodeGen/Register.h ---------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef LLVM_CODEGEN_REGISTER_H |
| 10 | #define LLVM_CODEGEN_REGISTER_H |
| 11 | |
| 12 | #include "llvm/MC/MCRegister.h" |
| 13 | #include <cassert> |
| 14 | |
| 15 | namespace llvm { |
| 16 | |
| 17 | /// Wrapper class representing virtual and physical registers. Should be passed |
| 18 | /// by value. |
| 19 | class Register { |
| 20 | unsigned Reg; |
| 21 | |
| 22 | public: |
| 23 | constexpr Register(unsigned Val = 0): Reg(Val) {} |
| 24 | constexpr Register(MCRegister Val): Reg(Val) {} |
| 25 | |
| 26 | // Register numbers can represent physical registers, virtual registers, and |
| 27 | // sometimes stack slots. The unsigned values are divided into these ranges: |
| 28 | // |
| 29 | // 0 Not a register, can be used as a sentinel. |
| 30 | // [1;2^30) Physical registers assigned by TableGen. |
| 31 | // [2^30;2^31) Stack slots. (Rarely used.) |
| 32 | // [2^31;2^32) Virtual registers assigned by MachineRegisterInfo. |
| 33 | // |
| 34 | // Further sentinels can be allocated from the small negative integers. |
| 35 | // DenseMapInfo<unsigned> uses -1u and -2u. |
| 36 | static_assert(std::numeric_limits<decltype(Reg)>::max() >= 0xFFFFFFFF, |
| 37 | "Reg isn't large enough to hold full range."); |
| 38 | |
| 39 | /// isStackSlot - Sometimes it is useful the be able to store a non-negative |
| 40 | /// frame index in a variable that normally holds a register. isStackSlot() |
| 41 | /// returns true if Reg is in the range used for stack slots. |
| 42 | /// |
| 43 | /// FIXME: remove in favor of member. |
| 44 | static bool isStackSlot(unsigned Reg) { |
| 45 | return MCRegister::isStackSlot(Reg); |
| 46 | } |
| 47 | |
| 48 | /// Return true if this is a stack slot. |
| 49 | bool isStack() const { return MCRegister::isStackSlot(Reg); } |
| 50 | |
| 51 | /// Compute the frame index from a register value representing a stack slot. |
| 52 | static int stackSlot2Index(Register Reg) { |
| 53 | assert(Reg.isStack() && "Not a stack slot")((Reg.isStack() && "Not a stack slot") ? static_cast< void> (0) : __assert_fail ("Reg.isStack() && \"Not a stack slot\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/Register.h" , 53, __PRETTY_FUNCTION__)); |
| 54 | return int(Reg - MCRegister::FirstStackSlot); |
| 55 | } |
| 56 | |
| 57 | /// Convert a non-negative frame index to a stack slot register value. |
| 58 | static Register index2StackSlot(int FI) { |
| 59 | assert(FI >= 0 && "Cannot hold a negative frame index.")((FI >= 0 && "Cannot hold a negative frame index." ) ? static_cast<void> (0) : __assert_fail ("FI >= 0 && \"Cannot hold a negative frame index.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/Register.h" , 59, __PRETTY_FUNCTION__)); |
| 60 | return Register(FI + MCRegister::FirstStackSlot); |
| 61 | } |
| 62 | |
| 63 | /// Return true if the specified register number is in |
| 64 | /// the physical register namespace. |
| 65 | static bool isPhysicalRegister(unsigned Reg) { |
| 66 | return MCRegister::isPhysicalRegister(Reg); |
| 67 | } |
| 68 | |
| 69 | /// Return true if the specified register number is in |
| 70 | /// the virtual register namespace. |
| 71 | static bool isVirtualRegister(unsigned Reg) { |
| 72 | return Reg & MCRegister::VirtualRegFlag && !isStackSlot(Reg); |
| 73 | } |
| 74 | |
| 75 | /// Convert a virtual register number to a 0-based index. |
| 76 | /// The first virtual register in a function will get the index 0. |
| 77 | static unsigned virtReg2Index(Register Reg) { |
| 78 | assert(isVirtualRegister(Reg) && "Not a virtual register")((isVirtualRegister(Reg) && "Not a virtual register") ? static_cast<void> (0) : __assert_fail ("isVirtualRegister(Reg) && \"Not a virtual register\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/Register.h" , 78, __PRETTY_FUNCTION__)); |
| 79 | return Reg & ~MCRegister::VirtualRegFlag; |
| 80 | } |
| 81 | |
| 82 | /// Convert a 0-based index to a virtual register number. |
| 83 | /// This is the inverse operation of VirtReg2IndexFunctor below. |
| 84 | static Register index2VirtReg(unsigned Index) { |
| 85 | assert(Index < (1u << 31) && "Index too large for virtual register range.")((Index < (1u << 31) && "Index too large for virtual register range." ) ? static_cast<void> (0) : __assert_fail ("Index < (1u << 31) && \"Index too large for virtual register range.\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/Register.h" , 85, __PRETTY_FUNCTION__)); |
| 86 | return Index | MCRegister::VirtualRegFlag; |
| 87 | } |
| 88 | |
| 89 | /// Return true if the specified register number is in the virtual register |
| 90 | /// namespace. |
| 91 | bool isVirtual() const { |
| 92 | return isVirtualRegister(Reg); |
| 93 | } |
| 94 | |
| 95 | /// Return true if the specified register number is in the physical register |
| 96 | /// namespace. |
| 97 | bool isPhysical() const { |
| 98 | return isPhysicalRegister(Reg); |
| 99 | } |
| 100 | |
| 101 | /// Convert a virtual register number to a 0-based index. The first virtual |
| 102 | /// register in a function will get the index 0. |
| 103 | unsigned virtRegIndex() const { |
| 104 | return virtReg2Index(Reg); |
| 105 | } |
| 106 | |
| 107 | constexpr operator unsigned() const { |
| 108 | return Reg; |
| 109 | } |
| 110 | |
| 111 | unsigned id() const { return Reg; } |
| 112 | |
| 113 | operator MCRegister() const { |
| 114 | return MCRegister(Reg); |
| 115 | } |
| 116 | |
| 117 | /// Utility to check-convert this value to a MCRegister. The caller is |
| 118 | /// expected to have already validated that this Register is, indeed, |
| 119 | /// physical. |
| 120 | MCRegister asMCReg() const { |
| 121 | assert(Reg == MCRegister::NoRegister ||((Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister (Reg)) ? static_cast<void> (0) : __assert_fail ("Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister(Reg)" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/Register.h" , 122, __PRETTY_FUNCTION__)) |
| 122 | MCRegister::isPhysicalRegister(Reg))((Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister (Reg)) ? static_cast<void> (0) : __assert_fail ("Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister(Reg)" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/Register.h" , 122, __PRETTY_FUNCTION__)); |
| 123 | return MCRegister(Reg); |
| 124 | } |
| 125 | |
| 126 | bool isValid() const { return Reg != MCRegister::NoRegister; } |
| 127 | |
| 128 | /// Comparisons between register objects |
| 129 | bool operator==(const Register &Other) const { return Reg == Other.Reg; } |
| 130 | bool operator!=(const Register &Other) const { return Reg != Other.Reg; } |
| 131 | bool operator==(const MCRegister &Other) const { return Reg == Other.id(); } |
| 132 | bool operator!=(const MCRegister &Other) const { return Reg != Other.id(); } |
| 133 | |
| 134 | /// Comparisons against register constants. E.g. |
| 135 | /// * R == AArch64::WZR |
| 136 | /// * R == 0 |
| 137 | /// * R == VirtRegMap::NO_PHYS_REG |
| 138 | bool operator==(unsigned Other) const { return Reg == Other; } |
| 139 | bool operator!=(unsigned Other) const { return Reg != Other; } |
| 140 | bool operator==(int Other) const { return Reg == unsigned(Other); } |
| 141 | bool operator!=(int Other) const { return Reg != unsigned(Other); } |
| 142 | // MSVC requires that we explicitly declare these two as well. |
| 143 | bool operator==(MCPhysReg Other) const { return Reg == unsigned(Other); } |
| 144 | bool operator!=(MCPhysReg Other) const { return Reg != unsigned(Other); } |
| 145 | }; |
| 146 | |
| 147 | // Provide DenseMapInfo for Register |
| 148 | template<> struct DenseMapInfo<Register> { |
| 149 | static inline unsigned getEmptyKey() { |
| 150 | return DenseMapInfo<unsigned>::getEmptyKey(); |
| 151 | } |
| 152 | static inline unsigned getTombstoneKey() { |
| 153 | return DenseMapInfo<unsigned>::getTombstoneKey(); |
| 154 | } |
| 155 | static unsigned getHashValue(const Register &Val) { |
| 156 | return DenseMapInfo<unsigned>::getHashValue(Val.id()); |
| 157 | } |
| 158 | static bool isEqual(const Register &LHS, const Register &RHS) { |
| 159 | return DenseMapInfo<unsigned>::isEqual(LHS.id(), RHS.id()); |
| 160 | } |
| 161 | }; |
| 162 | |
| 163 | } |
| 164 | |
| 165 | #endif // LLVM_CODEGEN_REGISTER_H |
| 1 | //===- llvm/CodeGen/TargetInstrInfo.h - Instruction Info --------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file describes the target machine instruction set to the code generator. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #ifndef LLVM_CODEGEN_TARGETINSTRINFO_H |
| 14 | #define LLVM_CODEGEN_TARGETINSTRINFO_H |
| 15 | |
| 16 | #include "llvm/ADT/ArrayRef.h" |
| 17 | #include "llvm/ADT/DenseMap.h" |
| 18 | #include "llvm/ADT/DenseMapInfo.h" |
| 19 | #include "llvm/ADT/None.h" |
| 20 | #include "llvm/CodeGen/MIRFormatter.h" |
| 21 | #include "llvm/CodeGen/MachineBasicBlock.h" |
| 22 | #include "llvm/CodeGen/MachineCombinerPattern.h" |
| 23 | #include "llvm/CodeGen/MachineFunction.h" |
| 24 | #include "llvm/CodeGen/MachineInstr.h" |
| 25 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
| 26 | #include "llvm/CodeGen/MachineOperand.h" |
| 27 | #include "llvm/CodeGen/MachineOutliner.h" |
| 28 | #include "llvm/CodeGen/RegisterClassInfo.h" |
| 29 | #include "llvm/CodeGen/VirtRegMap.h" |
| 30 | #include "llvm/MC/MCInstrInfo.h" |
| 31 | #include "llvm/Support/BranchProbability.h" |
| 32 | #include "llvm/Support/ErrorHandling.h" |
| 33 | #include <cassert> |
| 34 | #include <cstddef> |
| 35 | #include <cstdint> |
| 36 | #include <utility> |
| 37 | #include <vector> |
| 38 | |
| 39 | namespace llvm { |
| 40 | |
| 41 | class AAResults; |
| 42 | class DFAPacketizer; |
| 43 | class InstrItineraryData; |
| 44 | class LiveIntervals; |
| 45 | class LiveVariables; |
| 46 | class MachineLoop; |
| 47 | class MachineMemOperand; |
| 48 | class MachineRegisterInfo; |
| 49 | class MCAsmInfo; |
| 50 | class MCInst; |
| 51 | struct MCSchedModel; |
| 52 | class Module; |
| 53 | class ScheduleDAG; |
| 54 | class ScheduleDAGMI; |
| 55 | class ScheduleHazardRecognizer; |
| 56 | class SDNode; |
| 57 | class SelectionDAG; |
| 58 | class RegScavenger; |
| 59 | class TargetRegisterClass; |
| 60 | class TargetRegisterInfo; |
| 61 | class TargetSchedModel; |
| 62 | class TargetSubtargetInfo; |
| 63 | |
| 64 | template <class T> class SmallVectorImpl; |
| 65 | |
| 66 | using ParamLoadedValue = std::pair<MachineOperand, DIExpression*>; |
| 67 | |
| 68 | struct DestSourcePair { |
| 69 | const MachineOperand *Destination; |
| 70 | const MachineOperand *Source; |
| 71 | |
| 72 | DestSourcePair(const MachineOperand &Dest, const MachineOperand &Src) |
| 73 | : Destination(&Dest), Source(&Src) {} |
| 74 | }; |
| 75 | |
| 76 | /// Used to describe a register and immediate addition. |
| 77 | struct RegImmPair { |
| 78 | Register Reg; |
| 79 | int64_t Imm; |
| 80 | |
| 81 | RegImmPair(Register Reg, int64_t Imm) : Reg(Reg), Imm(Imm) {} |
| 82 | }; |
| 83 | |
| 84 | /// Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare. |
| 85 | /// It holds the register values, the scale value and the displacement. |
| 86 | struct ExtAddrMode { |
| 87 | Register BaseReg; |
| 88 | Register ScaledReg; |
| 89 | int64_t Scale; |
| 90 | int64_t Displacement; |
| 91 | }; |
| 92 | |
| 93 | //--------------------------------------------------------------------------- |
| 94 | /// |
| 95 | /// TargetInstrInfo - Interface to description of machine instruction set |
| 96 | /// |
| 97 | class TargetInstrInfo : public MCInstrInfo { |
| 98 | public: |
| 99 | TargetInstrInfo(unsigned CFSetupOpcode = ~0u, unsigned CFDestroyOpcode = ~0u, |
| 100 | unsigned CatchRetOpcode = ~0u, unsigned ReturnOpcode = ~0u) |
| 101 | : CallFrameSetupOpcode(CFSetupOpcode), |
| 102 | CallFrameDestroyOpcode(CFDestroyOpcode), CatchRetOpcode(CatchRetOpcode), |
| 103 | ReturnOpcode(ReturnOpcode) {} |
| 104 | TargetInstrInfo(const TargetInstrInfo &) = delete; |
| 105 | TargetInstrInfo &operator=(const TargetInstrInfo &) = delete; |
| 106 | virtual ~TargetInstrInfo(); |
| 107 | |
| 108 | static bool isGenericOpcode(unsigned Opc) { |
| 109 | return Opc <= TargetOpcode::GENERIC_OP_END; |
| 110 | } |
| 111 | |
| 112 | /// Given a machine instruction descriptor, returns the register |
| 113 | /// class constraint for OpNum, or NULL. |
| 114 | virtual |
| 115 | const TargetRegisterClass *getRegClass(const MCInstrDesc &MCID, unsigned OpNum, |
| 116 | const TargetRegisterInfo *TRI, |
| 117 | const MachineFunction &MF) const; |
| 118 | |
| 119 | /// Return true if the instruction is trivially rematerializable, meaning it |
| 120 | /// has no side effects and requires no operands that aren't always available. |
| 121 | /// This means the only allowed uses are constants and unallocatable physical |
| 122 | /// registers so that the instructions result is independent of the place |
| 123 | /// in the function. |
| 124 | bool isTriviallyReMaterializable(const MachineInstr &MI, |
| 125 | AAResults *AA = nullptr) const { |
| 126 | return MI.getOpcode() == TargetOpcode::IMPLICIT_DEF || |
| 127 | (MI.getDesc().isRematerializable() && |
| 128 | (isReallyTriviallyReMaterializable(MI, AA) || |
| 129 | isReallyTriviallyReMaterializableGeneric(MI, AA))); |
| 130 | } |
| 131 | |
| 132 | protected: |
| 133 | /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is |
| 134 | /// set, this hook lets the target specify whether the instruction is actually |
| 135 | /// trivially rematerializable, taking into consideration its operands. This |
| 136 | /// predicate must return false if the instruction has any side effects other |
| 137 | /// than producing a value, or if it requres any address registers that are |
| 138 | /// not always available. |
| 139 | /// Requirements must be check as stated in isTriviallyReMaterializable() . |
| 140 | virtual bool isReallyTriviallyReMaterializable(const MachineInstr &MI, |
| 141 | AAResults *AA) const { |
| 142 | return false; |
| 143 | } |
| 144 | |
| 145 | /// This method commutes the operands of the given machine instruction MI. |
| 146 | /// The operands to be commuted are specified by their indices OpIdx1 and |
| 147 | /// OpIdx2. |
| 148 | /// |
| 149 | /// If a target has any instructions that are commutable but require |
| 150 | /// converting to different instructions or making non-trivial changes |
| 151 | /// to commute them, this method can be overloaded to do that. |
| 152 | /// The default implementation simply swaps the commutable operands. |
| 153 | /// |
| 154 | /// If NewMI is false, MI is modified in place and returned; otherwise, a |
| 155 | /// new machine instruction is created and returned. |
| 156 | /// |
| 157 | /// Do not call this method for a non-commutable instruction. |
| 158 | /// Even though the instruction is commutable, the method may still |
| 159 | /// fail to commute the operands, null pointer is returned in such cases. |
| 160 | virtual MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI, |
| 161 | unsigned OpIdx1, |
| 162 | unsigned OpIdx2) const; |
| 163 | |
| 164 | /// Assigns the (CommutableOpIdx1, CommutableOpIdx2) pair of commutable |
| 165 | /// operand indices to (ResultIdx1, ResultIdx2). |
| 166 | /// One or both input values of the pair: (ResultIdx1, ResultIdx2) may be |
| 167 | /// predefined to some indices or be undefined (designated by the special |
| 168 | /// value 'CommuteAnyOperandIndex'). |
| 169 | /// The predefined result indices cannot be re-defined. |
| 170 | /// The function returns true iff after the result pair redefinition |
| 171 | /// the fixed result pair is equal to or equivalent to the source pair of |
| 172 | /// indices: (CommutableOpIdx1, CommutableOpIdx2). It is assumed here that |
| 173 | /// the pairs (x,y) and (y,x) are equivalent. |
| 174 | static bool fixCommutedOpIndices(unsigned &ResultIdx1, unsigned &ResultIdx2, |
| 175 | unsigned CommutableOpIdx1, |
| 176 | unsigned CommutableOpIdx2); |
| 177 | |
| 178 | private: |
| 179 | /// For instructions with opcodes for which the M_REMATERIALIZABLE flag is |
| 180 | /// set and the target hook isReallyTriviallyReMaterializable returns false, |
| 181 | /// this function does target-independent tests to determine if the |
| 182 | /// instruction is really trivially rematerializable. |
| 183 | bool isReallyTriviallyReMaterializableGeneric(const MachineInstr &MI, |
| 184 | AAResults *AA) const; |
| 185 | |
| 186 | public: |
| 187 | /// These methods return the opcode of the frame setup/destroy instructions |
| 188 | /// if they exist (-1 otherwise). Some targets use pseudo instructions in |
| 189 | /// order to abstract away the difference between operating with a frame |
| 190 | /// pointer and operating without, through the use of these two instructions. |
| 191 | /// |
| 192 | unsigned getCallFrameSetupOpcode() const { return CallFrameSetupOpcode; } |
| 193 | unsigned getCallFrameDestroyOpcode() const { return CallFrameDestroyOpcode; } |
| 194 | |
| 195 | /// Returns true if the argument is a frame pseudo instruction. |
| 196 | bool isFrameInstr(const MachineInstr &I) const { |
| 197 | return I.getOpcode() == getCallFrameSetupOpcode() || |
| 198 | I.getOpcode() == getCallFrameDestroyOpcode(); |
| 199 | } |
| 200 | |
| 201 | /// Returns true if the argument is a frame setup pseudo instruction. |
| 202 | bool isFrameSetup(const MachineInstr &I) const { |
| 203 | return I.getOpcode() == getCallFrameSetupOpcode(); |
| 204 | } |
| 205 | |
| 206 | /// Returns size of the frame associated with the given frame instruction. |
| 207 | /// For frame setup instruction this is frame that is set up space set up |
| 208 | /// after the instruction. For frame destroy instruction this is the frame |
| 209 | /// freed by the caller. |
| 210 | /// Note, in some cases a call frame (or a part of it) may be prepared prior |
| 211 | /// to the frame setup instruction. It occurs in the calls that involve |
| 212 | /// inalloca arguments. This function reports only the size of the frame part |
| 213 | /// that is set up between the frame setup and destroy pseudo instructions. |
| 214 | int64_t getFrameSize(const MachineInstr &I) const { |
| 215 | assert(isFrameInstr(I) && "Not a frame instruction")((isFrameInstr(I) && "Not a frame instruction") ? static_cast <void> (0) : __assert_fail ("isFrameInstr(I) && \"Not a frame instruction\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 215, __PRETTY_FUNCTION__)); |
| 216 | assert(I.getOperand(0).getImm() >= 0)((I.getOperand(0).getImm() >= 0) ? static_cast<void> (0) : __assert_fail ("I.getOperand(0).getImm() >= 0", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 216, __PRETTY_FUNCTION__)); |
| 217 | return I.getOperand(0).getImm(); |
| 218 | } |
| 219 | |
| 220 | /// Returns the total frame size, which is made up of the space set up inside |
| 221 | /// the pair of frame start-stop instructions and the space that is set up |
| 222 | /// prior to the pair. |
| 223 | int64_t getFrameTotalSize(const MachineInstr &I) const { |
| 224 | if (isFrameSetup(I)) { |
| 225 | assert(I.getOperand(1).getImm() >= 0 &&((I.getOperand(1).getImm() >= 0 && "Frame size must not be negative" ) ? static_cast<void> (0) : __assert_fail ("I.getOperand(1).getImm() >= 0 && \"Frame size must not be negative\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 226, __PRETTY_FUNCTION__)) |
| 226 | "Frame size must not be negative")((I.getOperand(1).getImm() >= 0 && "Frame size must not be negative" ) ? static_cast<void> (0) : __assert_fail ("I.getOperand(1).getImm() >= 0 && \"Frame size must not be negative\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 226, __PRETTY_FUNCTION__)); |
| 227 | return getFrameSize(I) + I.getOperand(1).getImm(); |
| 228 | } |
| 229 | return getFrameSize(I); |
| 230 | } |
| 231 | |
| 232 | unsigned getCatchReturnOpcode() const { return CatchRetOpcode; } |
| 233 | unsigned getReturnOpcode() const { return ReturnOpcode; } |
| 234 | |
| 235 | /// Returns the actual stack pointer adjustment made by an instruction |
| 236 | /// as part of a call sequence. By default, only call frame setup/destroy |
| 237 | /// instructions adjust the stack, but targets may want to override this |
| 238 | /// to enable more fine-grained adjustment, or adjust by a different value. |
| 239 | virtual int getSPAdjust(const MachineInstr &MI) const; |
| 240 | |
| 241 | /// Return true if the instruction is a "coalescable" extension instruction. |
| 242 | /// That is, it's like a copy where it's legal for the source to overlap the |
| 243 | /// destination. e.g. X86::MOVSX64rr32. If this returns true, then it's |
| 244 | /// expected the pre-extension value is available as a subreg of the result |
| 245 | /// register. This also returns the sub-register index in SubIdx. |
| 246 | virtual bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, |
| 247 | Register &DstReg, unsigned &SubIdx) const { |
| 248 | return false; |
| 249 | } |
| 250 | |
| 251 | /// If the specified machine instruction is a direct |
| 252 | /// load from a stack slot, return the virtual or physical register number of |
| 253 | /// the destination along with the FrameIndex of the loaded stack slot. If |
| 254 | /// not, return 0. This predicate must return 0 if the instruction has |
| 255 | /// any side effects other than loading from the stack slot. |
| 256 | virtual unsigned isLoadFromStackSlot(const MachineInstr &MI, |
| 257 | int &FrameIndex) const { |
| 258 | return 0; |
| 259 | } |
| 260 | |
| 261 | /// Optional extension of isLoadFromStackSlot that returns the number of |
| 262 | /// bytes loaded from the stack. This must be implemented if a backend |
| 263 | /// supports partial stack slot spills/loads to further disambiguate |
| 264 | /// what the load does. |
| 265 | virtual unsigned isLoadFromStackSlot(const MachineInstr &MI, |
| 266 | int &FrameIndex, |
| 267 | unsigned &MemBytes) const { |
| 268 | MemBytes = 0; |
| 269 | return isLoadFromStackSlot(MI, FrameIndex); |
| 270 | } |
| 271 | |
| 272 | /// Check for post-frame ptr elimination stack locations as well. |
| 273 | /// This uses a heuristic so it isn't reliable for correctness. |
| 274 | virtual unsigned isLoadFromStackSlotPostFE(const MachineInstr &MI, |
| 275 | int &FrameIndex) const { |
| 276 | return 0; |
| 277 | } |
| 278 | |
| 279 | /// If the specified machine instruction has a load from a stack slot, |
| 280 | /// return true along with the FrameIndices of the loaded stack slot and the |
| 281 | /// machine mem operands containing the reference. |
| 282 | /// If not, return false. Unlike isLoadFromStackSlot, this returns true for |
| 283 | /// any instructions that loads from the stack. This is just a hint, as some |
| 284 | /// cases may be missed. |
| 285 | virtual bool hasLoadFromStackSlot( |
| 286 | const MachineInstr &MI, |
| 287 | SmallVectorImpl<const MachineMemOperand *> &Accesses) const; |
| 288 | |
| 289 | /// If the specified machine instruction is a direct |
| 290 | /// store to a stack slot, return the virtual or physical register number of |
| 291 | /// the source reg along with the FrameIndex of the loaded stack slot. If |
| 292 | /// not, return 0. This predicate must return 0 if the instruction has |
| 293 | /// any side effects other than storing to the stack slot. |
| 294 | virtual unsigned isStoreToStackSlot(const MachineInstr &MI, |
| 295 | int &FrameIndex) const { |
| 296 | return 0; |
| 297 | } |
| 298 | |
| 299 | /// Optional extension of isStoreToStackSlot that returns the number of |
| 300 | /// bytes stored to the stack. This must be implemented if a backend |
| 301 | /// supports partial stack slot spills/loads to further disambiguate |
| 302 | /// what the store does. |
| 303 | virtual unsigned isStoreToStackSlot(const MachineInstr &MI, |
| 304 | int &FrameIndex, |
| 305 | unsigned &MemBytes) const { |
| 306 | MemBytes = 0; |
| 307 | return isStoreToStackSlot(MI, FrameIndex); |
| 308 | } |
| 309 | |
| 310 | /// Check for post-frame ptr elimination stack locations as well. |
| 311 | /// This uses a heuristic, so it isn't reliable for correctness. |
| 312 | virtual unsigned isStoreToStackSlotPostFE(const MachineInstr &MI, |
| 313 | int &FrameIndex) const { |
| 314 | return 0; |
| 315 | } |
| 316 | |
| 317 | /// If the specified machine instruction has a store to a stack slot, |
| 318 | /// return true along with the FrameIndices of the loaded stack slot and the |
| 319 | /// machine mem operands containing the reference. |
| 320 | /// If not, return false. Unlike isStoreToStackSlot, |
| 321 | /// this returns true for any instructions that stores to the |
| 322 | /// stack. This is just a hint, as some cases may be missed. |
| 323 | virtual bool hasStoreToStackSlot( |
| 324 | const MachineInstr &MI, |
| 325 | SmallVectorImpl<const MachineMemOperand *> &Accesses) const; |
| 326 | |
| 327 | /// Return true if the specified machine instruction |
| 328 | /// is a copy of one stack slot to another and has no other effect. |
| 329 | /// Provide the identity of the two frame indices. |
| 330 | virtual bool isStackSlotCopy(const MachineInstr &MI, int &DestFrameIndex, |
| 331 | int &SrcFrameIndex) const { |
| 332 | return false; |
| 333 | } |
| 334 | |
| 335 | /// Compute the size in bytes and offset within a stack slot of a spilled |
| 336 | /// register or subregister. |
| 337 | /// |
| 338 | /// \param [out] Size in bytes of the spilled value. |
| 339 | /// \param [out] Offset in bytes within the stack slot. |
| 340 | /// \returns true if both Size and Offset are successfully computed. |
| 341 | /// |
| 342 | /// Not all subregisters have computable spill slots. For example, |
| 343 | /// subregisters registers may not be byte-sized, and a pair of discontiguous |
| 344 | /// subregisters has no single offset. |
| 345 | /// |
| 346 | /// Targets with nontrivial bigendian implementations may need to override |
| 347 | /// this, particularly to support spilled vector registers. |
| 348 | virtual bool getStackSlotRange(const TargetRegisterClass *RC, unsigned SubIdx, |
| 349 | unsigned &Size, unsigned &Offset, |
| 350 | const MachineFunction &MF) const; |
| 351 | |
| 352 | /// Return true if the given instruction is terminator that is unspillable, |
| 353 | /// according to isUnspillableTerminatorImpl. |
| 354 | bool isUnspillableTerminator(const MachineInstr *MI) const { |
| 355 | return MI->isTerminator() && isUnspillableTerminatorImpl(MI); |
| 356 | } |
| 357 | |
| 358 | /// Returns the size in bytes of the specified MachineInstr, or ~0U |
| 359 | /// when this function is not implemented by a target. |
| 360 | virtual unsigned getInstSizeInBytes(const MachineInstr &MI) const { |
| 361 | return ~0U; |
| 362 | } |
| 363 | |
| 364 | /// Return true if the instruction is as cheap as a move instruction. |
| 365 | /// |
| 366 | /// Targets for different archs need to override this, and different |
| 367 | /// micro-architectures can also be finely tuned inside. |
| 368 | virtual bool isAsCheapAsAMove(const MachineInstr &MI) const { |
| 369 | return MI.isAsCheapAsAMove(); |
| 370 | } |
| 371 | |
| 372 | /// Return true if the instruction should be sunk by MachineSink. |
| 373 | /// |
| 374 | /// MachineSink determines on its own whether the instruction is safe to sink; |
| 375 | /// this gives the target a hook to override the default behavior with regards |
| 376 | /// to which instructions should be sunk. |
| 377 | virtual bool shouldSink(const MachineInstr &MI) const { return true; } |
| 378 | |
| 379 | /// Re-issue the specified 'original' instruction at the |
| 380 | /// specific location targeting a new destination register. |
| 381 | /// The register in Orig->getOperand(0).getReg() will be substituted by |
| 382 | /// DestReg:SubIdx. Any existing subreg index is preserved or composed with |
| 383 | /// SubIdx. |
| 384 | virtual void reMaterialize(MachineBasicBlock &MBB, |
| 385 | MachineBasicBlock::iterator MI, Register DestReg, |
| 386 | unsigned SubIdx, const MachineInstr &Orig, |
| 387 | const TargetRegisterInfo &TRI) const; |
| 388 | |
| 389 | /// Clones instruction or the whole instruction bundle \p Orig and |
| 390 | /// insert into \p MBB before \p InsertBefore. The target may update operands |
| 391 | /// that are required to be unique. |
| 392 | /// |
| 393 | /// \p Orig must not return true for MachineInstr::isNotDuplicable(). |
| 394 | virtual MachineInstr &duplicate(MachineBasicBlock &MBB, |
| 395 | MachineBasicBlock::iterator InsertBefore, |
| 396 | const MachineInstr &Orig) const; |
| 397 | |
| 398 | /// This method must be implemented by targets that |
| 399 | /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target |
| 400 | /// may be able to convert a two-address instruction into one or more true |
| 401 | /// three-address instructions on demand. This allows the X86 target (for |
| 402 | /// example) to convert ADD and SHL instructions into LEA instructions if they |
| 403 | /// would require register copies due to two-addressness. |
| 404 | /// |
| 405 | /// This method returns a null pointer if the transformation cannot be |
| 406 | /// performed, otherwise it returns the last new instruction. |
| 407 | /// |
| 408 | virtual MachineInstr *convertToThreeAddress(MachineFunction::iterator &MFI, |
| 409 | MachineInstr &MI, |
| 410 | LiveVariables *LV) const { |
| 411 | return nullptr; |
| 412 | } |
| 413 | |
| 414 | // This constant can be used as an input value of operand index passed to |
| 415 | // the method findCommutedOpIndices() to tell the method that the |
| 416 | // corresponding operand index is not pre-defined and that the method |
| 417 | // can pick any commutable operand. |
| 418 | static const unsigned CommuteAnyOperandIndex = ~0U; |
| 419 | |
| 420 | /// This method commutes the operands of the given machine instruction MI. |
| 421 | /// |
| 422 | /// The operands to be commuted are specified by their indices OpIdx1 and |
| 423 | /// OpIdx2. OpIdx1 and OpIdx2 arguments may be set to a special value |
| 424 | /// 'CommuteAnyOperandIndex', which means that the method is free to choose |
| 425 | /// any arbitrarily chosen commutable operand. If both arguments are set to |
| 426 | /// 'CommuteAnyOperandIndex' then the method looks for 2 different commutable |
| 427 | /// operands; then commutes them if such operands could be found. |
| 428 | /// |
| 429 | /// If NewMI is false, MI is modified in place and returned; otherwise, a |
| 430 | /// new machine instruction is created and returned. |
| 431 | /// |
| 432 | /// Do not call this method for a non-commutable instruction or |
| 433 | /// for non-commuable operands. |
| 434 | /// Even though the instruction is commutable, the method may still |
| 435 | /// fail to commute the operands, null pointer is returned in such cases. |
| 436 | MachineInstr * |
| 437 | commuteInstruction(MachineInstr &MI, bool NewMI = false, |
| 438 | unsigned OpIdx1 = CommuteAnyOperandIndex, |
| 439 | unsigned OpIdx2 = CommuteAnyOperandIndex) const; |
| 440 | |
| 441 | /// Returns true iff the routine could find two commutable operands in the |
| 442 | /// given machine instruction. |
| 443 | /// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. |
| 444 | /// If any of the INPUT values is set to the special value |
| 445 | /// 'CommuteAnyOperandIndex' then the method arbitrarily picks a commutable |
| 446 | /// operand, then returns its index in the corresponding argument. |
| 447 | /// If both of INPUT values are set to 'CommuteAnyOperandIndex' then method |
| 448 | /// looks for 2 commutable operands. |
| 449 | /// If INPUT values refer to some operands of MI, then the method simply |
| 450 | /// returns true if the corresponding operands are commutable and returns |
| 451 | /// false otherwise. |
| 452 | /// |
| 453 | /// For example, calling this method this way: |
| 454 | /// unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex; |
| 455 | /// findCommutedOpIndices(MI, Op1, Op2); |
| 456 | /// can be interpreted as a query asking to find an operand that would be |
| 457 | /// commutable with the operand#1. |
| 458 | virtual bool findCommutedOpIndices(const MachineInstr &MI, |
| 459 | unsigned &SrcOpIdx1, |
| 460 | unsigned &SrcOpIdx2) const; |
| 461 | |
| 462 | /// A pair composed of a register and a sub-register index. |
| 463 | /// Used to give some type checking when modeling Reg:SubReg. |
| 464 | struct RegSubRegPair { |
| 465 | Register Reg; |
| 466 | unsigned SubReg; |
| 467 | |
| 468 | RegSubRegPair(Register Reg = Register(), unsigned SubReg = 0) |
| 469 | : Reg(Reg), SubReg(SubReg) {} |
| 470 | |
| 471 | bool operator==(const RegSubRegPair& P) const { |
| 472 | return Reg == P.Reg && SubReg == P.SubReg; |
| 473 | } |
| 474 | bool operator!=(const RegSubRegPair& P) const { |
| 475 | return !(*this == P); |
| 476 | } |
| 477 | }; |
| 478 | |
| 479 | /// A pair composed of a pair of a register and a sub-register index, |
| 480 | /// and another sub-register index. |
| 481 | /// Used to give some type checking when modeling Reg:SubReg1, SubReg2. |
| 482 | struct RegSubRegPairAndIdx : RegSubRegPair { |
| 483 | unsigned SubIdx; |
| 484 | |
| 485 | RegSubRegPairAndIdx(Register Reg = Register(), unsigned SubReg = 0, |
| 486 | unsigned SubIdx = 0) |
| 487 | : RegSubRegPair(Reg, SubReg), SubIdx(SubIdx) {} |
| 488 | }; |
| 489 | |
| 490 | /// Build the equivalent inputs of a REG_SEQUENCE for the given \p MI |
| 491 | /// and \p DefIdx. |
| 492 | /// \p [out] InputRegs of the equivalent REG_SEQUENCE. Each element of |
| 493 | /// the list is modeled as <Reg:SubReg, SubIdx>. Operands with the undef |
| 494 | /// flag are not added to this list. |
| 495 | /// E.g., REG_SEQUENCE %1:sub1, sub0, %2, sub1 would produce |
| 496 | /// two elements: |
| 497 | /// - %1:sub1, sub0 |
| 498 | /// - %2<:0>, sub1 |
| 499 | /// |
| 500 | /// \returns true if it is possible to build such an input sequence |
| 501 | /// with the pair \p MI, \p DefIdx. False otherwise. |
| 502 | /// |
| 503 | /// \pre MI.isRegSequence() or MI.isRegSequenceLike(). |
| 504 | /// |
| 505 | /// \note The generic implementation does not provide any support for |
| 506 | /// MI.isRegSequenceLike(). In other words, one has to override |
| 507 | /// getRegSequenceLikeInputs for target specific instructions. |
| 508 | bool |
| 509 | getRegSequenceInputs(const MachineInstr &MI, unsigned DefIdx, |
| 510 | SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const; |
| 511 | |
| 512 | /// Build the equivalent inputs of a EXTRACT_SUBREG for the given \p MI |
| 513 | /// and \p DefIdx. |
| 514 | /// \p [out] InputReg of the equivalent EXTRACT_SUBREG. |
| 515 | /// E.g., EXTRACT_SUBREG %1:sub1, sub0, sub1 would produce: |
| 516 | /// - %1:sub1, sub0 |
| 517 | /// |
| 518 | /// \returns true if it is possible to build such an input sequence |
| 519 | /// with the pair \p MI, \p DefIdx and the operand has no undef flag set. |
| 520 | /// False otherwise. |
| 521 | /// |
| 522 | /// \pre MI.isExtractSubreg() or MI.isExtractSubregLike(). |
| 523 | /// |
| 524 | /// \note The generic implementation does not provide any support for |
| 525 | /// MI.isExtractSubregLike(). In other words, one has to override |
| 526 | /// getExtractSubregLikeInputs for target specific instructions. |
| 527 | bool getExtractSubregInputs(const MachineInstr &MI, unsigned DefIdx, |
| 528 | RegSubRegPairAndIdx &InputReg) const; |
| 529 | |
| 530 | /// Build the equivalent inputs of a INSERT_SUBREG for the given \p MI |
| 531 | /// and \p DefIdx. |
| 532 | /// \p [out] BaseReg and \p [out] InsertedReg contain |
| 533 | /// the equivalent inputs of INSERT_SUBREG. |
| 534 | /// E.g., INSERT_SUBREG %0:sub0, %1:sub1, sub3 would produce: |
| 535 | /// - BaseReg: %0:sub0 |
| 536 | /// - InsertedReg: %1:sub1, sub3 |
| 537 | /// |
| 538 | /// \returns true if it is possible to build such an input sequence |
| 539 | /// with the pair \p MI, \p DefIdx and the operand has no undef flag set. |
| 540 | /// False otherwise. |
| 541 | /// |
| 542 | /// \pre MI.isInsertSubreg() or MI.isInsertSubregLike(). |
| 543 | /// |
| 544 | /// \note The generic implementation does not provide any support for |
| 545 | /// MI.isInsertSubregLike(). In other words, one has to override |
| 546 | /// getInsertSubregLikeInputs for target specific instructions. |
| 547 | bool getInsertSubregInputs(const MachineInstr &MI, unsigned DefIdx, |
| 548 | RegSubRegPair &BaseReg, |
| 549 | RegSubRegPairAndIdx &InsertedReg) const; |
| 550 | |
| 551 | /// Return true if two machine instructions would produce identical values. |
| 552 | /// By default, this is only true when the two instructions |
| 553 | /// are deemed identical except for defs. If this function is called when the |
| 554 | /// IR is still in SSA form, the caller can pass the MachineRegisterInfo for |
| 555 | /// aggressive checks. |
| 556 | virtual bool produceSameValue(const MachineInstr &MI0, |
| 557 | const MachineInstr &MI1, |
| 558 | const MachineRegisterInfo *MRI = nullptr) const; |
| 559 | |
| 560 | /// \returns true if a branch from an instruction with opcode \p BranchOpc |
| 561 | /// bytes is capable of jumping to a position \p BrOffset bytes away. |
| 562 | virtual bool isBranchOffsetInRange(unsigned BranchOpc, |
| 563 | int64_t BrOffset) const { |
| 564 | llvm_unreachable("target did not implement")::llvm::llvm_unreachable_internal("target did not implement", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 564); |
| 565 | } |
| 566 | |
| 567 | /// \returns The block that branch instruction \p MI jumps to. |
| 568 | virtual MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const { |
| 569 | llvm_unreachable("target did not implement")::llvm::llvm_unreachable_internal("target did not implement", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 569); |
| 570 | } |
| 571 | |
| 572 | /// Insert an unconditional indirect branch at the end of \p MBB to \p |
| 573 | /// NewDestBB. \p BrOffset indicates the offset of \p NewDestBB relative to |
| 574 | /// the offset of the position to insert the new branch. |
| 575 | /// |
| 576 | /// \returns The number of bytes added to the block. |
| 577 | virtual unsigned insertIndirectBranch(MachineBasicBlock &MBB, |
| 578 | MachineBasicBlock &NewDestBB, |
| 579 | const DebugLoc &DL, |
| 580 | int64_t BrOffset = 0, |
| 581 | RegScavenger *RS = nullptr) const { |
| 582 | llvm_unreachable("target did not implement")::llvm::llvm_unreachable_internal("target did not implement", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 582); |
| 583 | } |
| 584 | |
| 585 | /// Analyze the branching code at the end of MBB, returning |
| 586 | /// true if it cannot be understood (e.g. it's a switch dispatch or isn't |
| 587 | /// implemented for a target). Upon success, this returns false and returns |
| 588 | /// with the following information in various cases: |
| 589 | /// |
| 590 | /// 1. If this block ends with no branches (it just falls through to its succ) |
| 591 | /// just return false, leaving TBB/FBB null. |
| 592 | /// 2. If this block ends with only an unconditional branch, it sets TBB to be |
| 593 | /// the destination block. |
| 594 | /// 3. If this block ends with a conditional branch and it falls through to a |
| 595 | /// successor block, it sets TBB to be the branch destination block and a |
| 596 | /// list of operands that evaluate the condition. These operands can be |
| 597 | /// passed to other TargetInstrInfo methods to create new branches. |
| 598 | /// 4. If this block ends with a conditional branch followed by an |
| 599 | /// unconditional branch, it returns the 'true' destination in TBB, the |
| 600 | /// 'false' destination in FBB, and a list of operands that evaluate the |
| 601 | /// condition. These operands can be passed to other TargetInstrInfo |
| 602 | /// methods to create new branches. |
| 603 | /// |
| 604 | /// Note that removeBranch and insertBranch must be implemented to support |
| 605 | /// cases where this method returns success. |
| 606 | /// |
| 607 | /// If AllowModify is true, then this routine is allowed to modify the basic |
| 608 | /// block (e.g. delete instructions after the unconditional branch). |
| 609 | /// |
| 610 | /// The CFG information in MBB.Predecessors and MBB.Successors must be valid |
| 611 | /// before calling this function. |
| 612 | virtual bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, |
| 613 | MachineBasicBlock *&FBB, |
| 614 | SmallVectorImpl<MachineOperand> &Cond, |
| 615 | bool AllowModify = false) const { |
| 616 | return true; |
| 617 | } |
| 618 | |
| 619 | /// Represents a predicate at the MachineFunction level. The control flow a |
| 620 | /// MachineBranchPredicate represents is: |
| 621 | /// |
| 622 | /// Reg = LHS `Predicate` RHS == ConditionDef |
| 623 | /// if Reg then goto TrueDest else goto FalseDest |
| 624 | /// |
| 625 | struct MachineBranchPredicate { |
| 626 | enum ComparePredicate { |
| 627 | PRED_EQ, // True if two values are equal |
| 628 | PRED_NE, // True if two values are not equal |
| 629 | PRED_INVALID // Sentinel value |
| 630 | }; |
| 631 | |
| 632 | ComparePredicate Predicate = PRED_INVALID; |
| 633 | MachineOperand LHS = MachineOperand::CreateImm(0); |
| 634 | MachineOperand RHS = MachineOperand::CreateImm(0); |
| 635 | MachineBasicBlock *TrueDest = nullptr; |
| 636 | MachineBasicBlock *FalseDest = nullptr; |
| 637 | MachineInstr *ConditionDef = nullptr; |
| 638 | |
| 639 | /// SingleUseCondition is true if ConditionDef is dead except for the |
| 640 | /// branch(es) at the end of the basic block. |
| 641 | /// |
| 642 | bool SingleUseCondition = false; |
| 643 | |
| 644 | explicit MachineBranchPredicate() = default; |
| 645 | }; |
| 646 | |
| 647 | /// Analyze the branching code at the end of MBB and parse it into the |
| 648 | /// MachineBranchPredicate structure if possible. Returns false on success |
| 649 | /// and true on failure. |
| 650 | /// |
| 651 | /// If AllowModify is true, then this routine is allowed to modify the basic |
| 652 | /// block (e.g. delete instructions after the unconditional branch). |
| 653 | /// |
| 654 | virtual bool analyzeBranchPredicate(MachineBasicBlock &MBB, |
| 655 | MachineBranchPredicate &MBP, |
| 656 | bool AllowModify = false) const { |
| 657 | return true; |
| 658 | } |
| 659 | |
| 660 | /// Remove the branching code at the end of the specific MBB. |
| 661 | /// This is only invoked in cases where analyzeBranch returns success. It |
| 662 | /// returns the number of instructions that were removed. |
| 663 | /// If \p BytesRemoved is non-null, report the change in code size from the |
| 664 | /// removed instructions. |
| 665 | virtual unsigned removeBranch(MachineBasicBlock &MBB, |
| 666 | int *BytesRemoved = nullptr) const { |
| 667 | llvm_unreachable("Target didn't implement TargetInstrInfo::removeBranch!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::removeBranch!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 667); |
| 668 | } |
| 669 | |
| 670 | /// Insert branch code into the end of the specified MachineBasicBlock. The |
| 671 | /// operands to this method are the same as those returned by analyzeBranch. |
| 672 | /// This is only invoked in cases where analyzeBranch returns success. It |
| 673 | /// returns the number of instructions inserted. If \p BytesAdded is non-null, |
| 674 | /// report the change in code size from the added instructions. |
| 675 | /// |
| 676 | /// It is also invoked by tail merging to add unconditional branches in |
| 677 | /// cases where analyzeBranch doesn't apply because there was no original |
| 678 | /// branch to analyze. At least this much must be implemented, else tail |
| 679 | /// merging needs to be disabled. |
| 680 | /// |
| 681 | /// The CFG information in MBB.Predecessors and MBB.Successors must be valid |
| 682 | /// before calling this function. |
| 683 | virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, |
| 684 | MachineBasicBlock *FBB, |
| 685 | ArrayRef<MachineOperand> Cond, |
| 686 | const DebugLoc &DL, |
| 687 | int *BytesAdded = nullptr) const { |
| 688 | llvm_unreachable("Target didn't implement TargetInstrInfo::insertBranch!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::insertBranch!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 688); |
| 689 | } |
| 690 | |
| 691 | unsigned insertUnconditionalBranch(MachineBasicBlock &MBB, |
| 692 | MachineBasicBlock *DestBB, |
| 693 | const DebugLoc &DL, |
| 694 | int *BytesAdded = nullptr) const { |
| 695 | return insertBranch(MBB, DestBB, nullptr, ArrayRef<MachineOperand>(), DL, |
| 696 | BytesAdded); |
| 697 | } |
| 698 | |
| 699 | /// Object returned by analyzeLoopForPipelining. Allows software pipelining |
| 700 | /// implementations to query attributes of the loop being pipelined and to |
| 701 | /// apply target-specific updates to the loop once pipelining is complete. |
| 702 | class PipelinerLoopInfo { |
| 703 | public: |
| 704 | virtual ~PipelinerLoopInfo(); |
| 705 | /// Return true if the given instruction should not be pipelined and should |
| 706 | /// be ignored. An example could be a loop comparison, or induction variable |
| 707 | /// update with no users being pipelined. |
| 708 | virtual bool shouldIgnoreForPipelining(const MachineInstr *MI) const = 0; |
| 709 | |
| 710 | /// Create a condition to determine if the trip count of the loop is greater |
| 711 | /// than TC. |
| 712 | /// |
| 713 | /// If the trip count is statically known to be greater than TC, return |
| 714 | /// true. If the trip count is statically known to be not greater than TC, |
| 715 | /// return false. Otherwise return nullopt and fill out Cond with the test |
| 716 | /// condition. |
| 717 | virtual Optional<bool> |
| 718 | createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB, |
| 719 | SmallVectorImpl<MachineOperand> &Cond) = 0; |
| 720 | |
| 721 | /// Modify the loop such that the trip count is |
| 722 | /// OriginalTC + TripCountAdjust. |
| 723 | virtual void adjustTripCount(int TripCountAdjust) = 0; |
| 724 | |
| 725 | /// Called when the loop's preheader has been modified to NewPreheader. |
| 726 | virtual void setPreheader(MachineBasicBlock *NewPreheader) = 0; |
| 727 | |
| 728 | /// Called when the loop is being removed. Any instructions in the preheader |
| 729 | /// should be removed. |
| 730 | /// |
| 731 | /// Once this function is called, no other functions on this object are |
| 732 | /// valid; the loop has been removed. |
| 733 | virtual void disposed() = 0; |
| 734 | }; |
| 735 | |
| 736 | /// Analyze loop L, which must be a single-basic-block loop, and if the |
| 737 | /// conditions can be understood enough produce a PipelinerLoopInfo object. |
| 738 | virtual std::unique_ptr<PipelinerLoopInfo> |
| 739 | analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const { |
| 740 | return nullptr; |
| 741 | } |
| 742 | |
| 743 | /// Analyze the loop code, return true if it cannot be understood. Upon |
| 744 | /// success, this function returns false and returns information about the |
| 745 | /// induction variable and compare instruction used at the end. |
| 746 | virtual bool analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst, |
| 747 | MachineInstr *&CmpInst) const { |
| 748 | return true; |
| 749 | } |
| 750 | |
| 751 | /// Generate code to reduce the loop iteration by one and check if the loop |
| 752 | /// is finished. Return the value/register of the new loop count. We need |
| 753 | /// this function when peeling off one or more iterations of a loop. This |
| 754 | /// function assumes the nth iteration is peeled first. |
| 755 | virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, |
| 756 | MachineBasicBlock &PreHeader, |
| 757 | MachineInstr *IndVar, MachineInstr &Cmp, |
| 758 | SmallVectorImpl<MachineOperand> &Cond, |
| 759 | SmallVectorImpl<MachineInstr *> &PrevInsts, |
| 760 | unsigned Iter, unsigned MaxIter) const { |
| 761 | llvm_unreachable("Target didn't implement ReduceLoopCount")::llvm::llvm_unreachable_internal("Target didn't implement ReduceLoopCount" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 761); |
| 762 | } |
| 763 | |
| 764 | /// Delete the instruction OldInst and everything after it, replacing it with |
| 765 | /// an unconditional branch to NewDest. This is used by the tail merging pass. |
| 766 | virtual void ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, |
| 767 | MachineBasicBlock *NewDest) const; |
| 768 | |
| 769 | /// Return true if it's legal to split the given basic |
| 770 | /// block at the specified instruction (i.e. instruction would be the start |
| 771 | /// of a new basic block). |
| 772 | virtual bool isLegalToSplitMBBAt(MachineBasicBlock &MBB, |
| 773 | MachineBasicBlock::iterator MBBI) const { |
| 774 | return true; |
| 775 | } |
| 776 | |
| 777 | /// Return true if it's profitable to predicate |
| 778 | /// instructions with accumulated instruction latency of "NumCycles" |
| 779 | /// of the specified basic block, where the probability of the instructions |
| 780 | /// being executed is given by Probability, and Confidence is a measure |
| 781 | /// of our confidence that it will be properly predicted. |
| 782 | virtual bool isProfitableToIfCvt(MachineBasicBlock &MBB, unsigned NumCycles, |
| 783 | unsigned ExtraPredCycles, |
| 784 | BranchProbability Probability) const { |
| 785 | return false; |
| 786 | } |
| 787 | |
| 788 | /// Second variant of isProfitableToIfCvt. This one |
| 789 | /// checks for the case where two basic blocks from true and false path |
| 790 | /// of a if-then-else (diamond) are predicated on mutually exclusive |
| 791 | /// predicates, where the probability of the true path being taken is given |
| 792 | /// by Probability, and Confidence is a measure of our confidence that it |
| 793 | /// will be properly predicted. |
| 794 | virtual bool isProfitableToIfCvt(MachineBasicBlock &TMBB, unsigned NumTCycles, |
| 795 | unsigned ExtraTCycles, |
| 796 | MachineBasicBlock &FMBB, unsigned NumFCycles, |
| 797 | unsigned ExtraFCycles, |
| 798 | BranchProbability Probability) const { |
| 799 | return false; |
| 800 | } |
| 801 | |
| 802 | /// Return true if it's profitable for if-converter to duplicate instructions |
| 803 | /// of specified accumulated instruction latencies in the specified MBB to |
| 804 | /// enable if-conversion. |
| 805 | /// The probability of the instructions being executed is given by |
| 806 | /// Probability, and Confidence is a measure of our confidence that it |
| 807 | /// will be properly predicted. |
| 808 | virtual bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, |
| 809 | unsigned NumCycles, |
| 810 | BranchProbability Probability) const { |
| 811 | return false; |
| 812 | } |
| 813 | |
| 814 | /// Return the increase in code size needed to predicate a contiguous run of |
| 815 | /// NumInsts instructions. |
| 816 | virtual unsigned extraSizeToPredicateInstructions(const MachineFunction &MF, |
| 817 | unsigned NumInsts) const { |
| 818 | return 0; |
| 819 | } |
| 820 | |
| 821 | /// Return an estimate for the code size reduction (in bytes) which will be |
| 822 | /// caused by removing the given branch instruction during if-conversion. |
| 823 | virtual unsigned predictBranchSizeForIfCvt(MachineInstr &MI) const { |
| 824 | return getInstSizeInBytes(MI); |
| 825 | } |
| 826 | |
| 827 | /// Return true if it's profitable to unpredicate |
| 828 | /// one side of a 'diamond', i.e. two sides of if-else predicated on mutually |
| 829 | /// exclusive predicates. |
| 830 | /// e.g. |
| 831 | /// subeq r0, r1, #1 |
| 832 | /// addne r0, r1, #1 |
| 833 | /// => |
| 834 | /// sub r0, r1, #1 |
| 835 | /// addne r0, r1, #1 |
| 836 | /// |
| 837 | /// This may be profitable is conditional instructions are always executed. |
| 838 | virtual bool isProfitableToUnpredicate(MachineBasicBlock &TMBB, |
| 839 | MachineBasicBlock &FMBB) const { |
| 840 | return false; |
| 841 | } |
| 842 | |
| 843 | /// Return true if it is possible to insert a select |
| 844 | /// instruction that chooses between TrueReg and FalseReg based on the |
| 845 | /// condition code in Cond. |
| 846 | /// |
| 847 | /// When successful, also return the latency in cycles from TrueReg, |
| 848 | /// FalseReg, and Cond to the destination register. In most cases, a select |
| 849 | /// instruction will be 1 cycle, so CondCycles = TrueCycles = FalseCycles = 1 |
| 850 | /// |
| 851 | /// Some x86 implementations have 2-cycle cmov instructions. |
| 852 | /// |
| 853 | /// @param MBB Block where select instruction would be inserted. |
| 854 | /// @param Cond Condition returned by analyzeBranch. |
| 855 | /// @param DstReg Virtual dest register that the result should write to. |
| 856 | /// @param TrueReg Virtual register to select when Cond is true. |
| 857 | /// @param FalseReg Virtual register to select when Cond is false. |
| 858 | /// @param CondCycles Latency from Cond+Branch to select output. |
| 859 | /// @param TrueCycles Latency from TrueReg to select output. |
| 860 | /// @param FalseCycles Latency from FalseReg to select output. |
| 861 | virtual bool canInsertSelect(const MachineBasicBlock &MBB, |
| 862 | ArrayRef<MachineOperand> Cond, Register DstReg, |
| 863 | Register TrueReg, Register FalseReg, |
| 864 | int &CondCycles, int &TrueCycles, |
| 865 | int &FalseCycles) const { |
| 866 | return false; |
| 867 | } |
| 868 | |
| 869 | /// Insert a select instruction into MBB before I that will copy TrueReg to |
| 870 | /// DstReg when Cond is true, and FalseReg to DstReg when Cond is false. |
| 871 | /// |
| 872 | /// This function can only be called after canInsertSelect() returned true. |
| 873 | /// The condition in Cond comes from analyzeBranch, and it can be assumed |
| 874 | /// that the same flags or registers required by Cond are available at the |
| 875 | /// insertion point. |
| 876 | /// |
| 877 | /// @param MBB Block where select instruction should be inserted. |
| 878 | /// @param I Insertion point. |
| 879 | /// @param DL Source location for debugging. |
| 880 | /// @param DstReg Virtual register to be defined by select instruction. |
| 881 | /// @param Cond Condition as computed by analyzeBranch. |
| 882 | /// @param TrueReg Virtual register to copy when Cond is true. |
| 883 | /// @param FalseReg Virtual register to copy when Cons is false. |
| 884 | virtual void insertSelect(MachineBasicBlock &MBB, |
| 885 | MachineBasicBlock::iterator I, const DebugLoc &DL, |
| 886 | Register DstReg, ArrayRef<MachineOperand> Cond, |
| 887 | Register TrueReg, Register FalseReg) const { |
| 888 | llvm_unreachable("Target didn't implement TargetInstrInfo::insertSelect!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::insertSelect!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 888); |
| 889 | } |
| 890 | |
| 891 | /// Analyze the given select instruction, returning true if |
| 892 | /// it cannot be understood. It is assumed that MI->isSelect() is true. |
| 893 | /// |
| 894 | /// When successful, return the controlling condition and the operands that |
| 895 | /// determine the true and false result values. |
| 896 | /// |
| 897 | /// Result = SELECT Cond, TrueOp, FalseOp |
| 898 | /// |
| 899 | /// Some targets can optimize select instructions, for example by predicating |
| 900 | /// the instruction defining one of the operands. Such targets should set |
| 901 | /// Optimizable. |
| 902 | /// |
| 903 | /// @param MI Select instruction to analyze. |
| 904 | /// @param Cond Condition controlling the select. |
| 905 | /// @param TrueOp Operand number of the value selected when Cond is true. |
| 906 | /// @param FalseOp Operand number of the value selected when Cond is false. |
| 907 | /// @param Optimizable Returned as true if MI is optimizable. |
| 908 | /// @returns False on success. |
| 909 | virtual bool analyzeSelect(const MachineInstr &MI, |
| 910 | SmallVectorImpl<MachineOperand> &Cond, |
| 911 | unsigned &TrueOp, unsigned &FalseOp, |
| 912 | bool &Optimizable) const { |
| 913 | assert(MI.getDesc().isSelect() && "MI must be a select instruction")((MI.getDesc().isSelect() && "MI must be a select instruction" ) ? static_cast<void> (0) : __assert_fail ("MI.getDesc().isSelect() && \"MI must be a select instruction\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 913, __PRETTY_FUNCTION__)); |
| 914 | return true; |
| 915 | } |
| 916 | |
| 917 | /// Given a select instruction that was understood by |
| 918 | /// analyzeSelect and returned Optimizable = true, attempt to optimize MI by |
| 919 | /// merging it with one of its operands. Returns NULL on failure. |
| 920 | /// |
| 921 | /// When successful, returns the new select instruction. The client is |
| 922 | /// responsible for deleting MI. |
| 923 | /// |
| 924 | /// If both sides of the select can be optimized, PreferFalse is used to pick |
| 925 | /// a side. |
| 926 | /// |
| 927 | /// @param MI Optimizable select instruction. |
| 928 | /// @param NewMIs Set that record all MIs in the basic block up to \p |
| 929 | /// MI. Has to be updated with any newly created MI or deleted ones. |
| 930 | /// @param PreferFalse Try to optimize FalseOp instead of TrueOp. |
| 931 | /// @returns Optimized instruction or NULL. |
| 932 | virtual MachineInstr *optimizeSelect(MachineInstr &MI, |
| 933 | SmallPtrSetImpl<MachineInstr *> &NewMIs, |
| 934 | bool PreferFalse = false) const { |
| 935 | // This function must be implemented if Optimizable is ever set. |
| 936 | llvm_unreachable("Target must implement TargetInstrInfo::optimizeSelect!")::llvm::llvm_unreachable_internal("Target must implement TargetInstrInfo::optimizeSelect!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 936); |
| 937 | } |
| 938 | |
| 939 | /// Emit instructions to copy a pair of physical registers. |
| 940 | /// |
| 941 | /// This function should support copies within any legal register class as |
| 942 | /// well as any cross-class copies created during instruction selection. |
| 943 | /// |
| 944 | /// The source and destination registers may overlap, which may require a |
| 945 | /// careful implementation when multiple copy instructions are required for |
| 946 | /// large registers. See for example the ARM target. |
| 947 | virtual void copyPhysReg(MachineBasicBlock &MBB, |
| 948 | MachineBasicBlock::iterator MI, const DebugLoc &DL, |
| 949 | MCRegister DestReg, MCRegister SrcReg, |
| 950 | bool KillSrc) const { |
| 951 | llvm_unreachable("Target didn't implement TargetInstrInfo::copyPhysReg!")::llvm::llvm_unreachable_internal("Target didn't implement TargetInstrInfo::copyPhysReg!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 951); |
| 952 | } |
| 953 | |
| 954 | /// Allow targets to tell MachineVerifier whether a specific register |
| 955 | /// MachineOperand can be used as part of PC-relative addressing. |
| 956 | /// PC-relative addressing modes in many CISC architectures contain |
| 957 | /// (non-PC) registers as offsets or scaling values, which inherently |
| 958 | /// tags the corresponding MachineOperand with OPERAND_PCREL. |
| 959 | /// |
| 960 | /// @param MO The MachineOperand in question. MO.isReg() should always |
| 961 | /// be true. |
| 962 | /// @return Whether this operand is allowed to be used PC-relatively. |
| 963 | virtual bool isPCRelRegisterOperandLegal(const MachineOperand &MO) const { |
| 964 | return false; |
| 965 | } |
| 966 | |
| 967 | protected: |
| 968 | /// Target-dependent implementation for IsCopyInstr. |
| 969 | /// If the specific machine instruction is a instruction that moves/copies |
| 970 | /// value from one register to another register return destination and source |
| 971 | /// registers as machine operands. |
| 972 | virtual Optional<DestSourcePair> |
| 973 | isCopyInstrImpl(const MachineInstr &MI) const { |
| 974 | return None; |
| 975 | } |
| 976 | |
| 977 | /// Return true if the given terminator MI is not expected to spill. This |
| 978 | /// sets the live interval as not spillable and adjusts phi node lowering to |
| 979 | /// not introduce copies after the terminator. Use with care, these are |
| 980 | /// currently used for hardware loop intrinsics in very controlled situations, |
| 981 | /// created prior to registry allocation in loops that only have single phi |
| 982 | /// users for the terminators value. They may run out of registers if not used |
| 983 | /// carefully. |
| 984 | virtual bool isUnspillableTerminatorImpl(const MachineInstr *MI) const { |
| 985 | return false; |
| 986 | } |
| 987 | |
| 988 | public: |
| 989 | /// If the specific machine instruction is a instruction that moves/copies |
| 990 | /// value from one register to another register return destination and source |
| 991 | /// registers as machine operands. |
| 992 | /// For COPY-instruction the method naturally returns destination and source |
| 993 | /// registers as machine operands, for all other instructions the method calls |
| 994 | /// target-dependent implementation. |
| 995 | Optional<DestSourcePair> isCopyInstr(const MachineInstr &MI) const { |
| 996 | if (MI.isCopy()) { |
| 997 | return DestSourcePair{MI.getOperand(0), MI.getOperand(1)}; |
| 998 | } |
| 999 | return isCopyInstrImpl(MI); |
| 1000 | } |
| 1001 | |
| 1002 | /// If the specific machine instruction is an instruction that adds an |
| 1003 | /// immediate value and a physical register, and stores the result in |
| 1004 | /// the given physical register \c Reg, return a pair of the source |
| 1005 | /// register and the offset which has been added. |
| 1006 | virtual Optional<RegImmPair> isAddImmediate(const MachineInstr &MI, |
| 1007 | Register Reg) const { |
| 1008 | return None; |
| 1009 | } |
| 1010 | |
| 1011 | /// Returns true if MI is an instruction that defines Reg to have a constant |
| 1012 | /// value and the value is recorded in ImmVal. The ImmVal is a result that |
| 1013 | /// should be interpreted as modulo size of Reg. |
| 1014 | virtual bool getConstValDefinedInReg(const MachineInstr &MI, |
| 1015 | const Register Reg, |
| 1016 | int64_t &ImmVal) const { |
| 1017 | return false; |
| 1018 | } |
| 1019 | |
| 1020 | /// Store the specified register of the given register class to the specified |
| 1021 | /// stack frame index. The store instruction is to be added to the given |
| 1022 | /// machine basic block before the specified machine instruction. If isKill |
| 1023 | /// is true, the register operand is the last use and must be marked kill. |
| 1024 | virtual void storeRegToStackSlot(MachineBasicBlock &MBB, |
| 1025 | MachineBasicBlock::iterator MI, |
| 1026 | Register SrcReg, bool isKill, int FrameIndex, |
| 1027 | const TargetRegisterClass *RC, |
| 1028 | const TargetRegisterInfo *TRI) const { |
| 1029 | llvm_unreachable("Target didn't implement "::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::storeRegToStackSlot!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1030) |
| 1030 | "TargetInstrInfo::storeRegToStackSlot!")::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::storeRegToStackSlot!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1030); |
| 1031 | } |
| 1032 | |
| 1033 | /// Load the specified register of the given register class from the specified |
| 1034 | /// stack frame index. The load instruction is to be added to the given |
| 1035 | /// machine basic block before the specified machine instruction. |
| 1036 | virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, |
| 1037 | MachineBasicBlock::iterator MI, |
| 1038 | Register DestReg, int FrameIndex, |
| 1039 | const TargetRegisterClass *RC, |
| 1040 | const TargetRegisterInfo *TRI) const { |
| 1041 | llvm_unreachable("Target didn't implement "::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::loadRegFromStackSlot!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1042) |
| 1042 | "TargetInstrInfo::loadRegFromStackSlot!")::llvm::llvm_unreachable_internal("Target didn't implement " "TargetInstrInfo::loadRegFromStackSlot!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1042); |
| 1043 | } |
| 1044 | |
| 1045 | /// This function is called for all pseudo instructions |
| 1046 | /// that remain after register allocation. Many pseudo instructions are |
| 1047 | /// created to help register allocation. This is the place to convert them |
| 1048 | /// into real instructions. The target can edit MI in place, or it can insert |
| 1049 | /// new instructions and erase MI. The function should return true if |
| 1050 | /// anything was changed. |
| 1051 | virtual bool expandPostRAPseudo(MachineInstr &MI) const { return false; } |
| 1052 | |
| 1053 | /// Check whether the target can fold a load that feeds a subreg operand |
| 1054 | /// (or a subreg operand that feeds a store). |
| 1055 | /// For example, X86 may want to return true if it can fold |
| 1056 | /// movl (%esp), %eax |
| 1057 | /// subb, %al, ... |
| 1058 | /// Into: |
| 1059 | /// subb (%esp), ... |
| 1060 | /// |
| 1061 | /// Ideally, we'd like the target implementation of foldMemoryOperand() to |
| 1062 | /// reject subregs - but since this behavior used to be enforced in the |
| 1063 | /// target-independent code, moving this responsibility to the targets |
| 1064 | /// has the potential of causing nasty silent breakage in out-of-tree targets. |
| 1065 | virtual bool isSubregFoldable() const { return false; } |
| 1066 | |
| 1067 | /// For a patchpoint, stackmap, or statepoint intrinsic, return the range of |
| 1068 | /// operands which can't be folded into stack references. Operands outside |
| 1069 | /// of the range are most likely foldable but it is not guaranteed. |
| 1070 | /// These instructions are unique in that stack references for some operands |
| 1071 | /// have the same execution cost (e.g. none) as the unfolded register forms. |
| 1072 | /// The ranged return is guaranteed to include all operands which can't be |
| 1073 | /// folded at zero cost. |
| 1074 | virtual std::pair<unsigned, unsigned> |
| 1075 | getPatchpointUnfoldableRange(const MachineInstr &MI) const; |
| 1076 | |
| 1077 | /// Attempt to fold a load or store of the specified stack |
| 1078 | /// slot into the specified machine instruction for the specified operand(s). |
| 1079 | /// If this is possible, a new instruction is returned with the specified |
| 1080 | /// operand folded, otherwise NULL is returned. |
| 1081 | /// The new instruction is inserted before MI, and the client is responsible |
| 1082 | /// for removing the old instruction. |
| 1083 | /// If VRM is passed, the assigned physregs can be inspected by target to |
| 1084 | /// decide on using an opcode (note that those assignments can still change). |
| 1085 | MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, |
| 1086 | int FI, |
| 1087 | LiveIntervals *LIS = nullptr, |
| 1088 | VirtRegMap *VRM = nullptr) const; |
| 1089 | |
| 1090 | /// Same as the previous version except it allows folding of any load and |
| 1091 | /// store from / to any address, not just from a specific stack slot. |
| 1092 | MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, |
| 1093 | MachineInstr &LoadMI, |
| 1094 | LiveIntervals *LIS = nullptr) const; |
| 1095 | |
| 1096 | /// Return true when there is potentially a faster code sequence |
| 1097 | /// for an instruction chain ending in \p Root. All potential patterns are |
| 1098 | /// returned in the \p Pattern vector. Pattern should be sorted in priority |
| 1099 | /// order since the pattern evaluator stops checking as soon as it finds a |
| 1100 | /// faster sequence. |
| 1101 | /// \param Root - Instruction that could be combined with one of its operands |
| 1102 | /// \param Patterns - Vector of possible combination patterns |
| 1103 | virtual bool |
| 1104 | getMachineCombinerPatterns(MachineInstr &Root, |
| 1105 | SmallVectorImpl<MachineCombinerPattern> &Patterns, |
| 1106 | bool DoRegPressureReduce) const; |
| 1107 | |
| 1108 | /// Return true if target supports reassociation of instructions in machine |
| 1109 | /// combiner pass to reduce register pressure for a given BB. |
| 1110 | virtual bool |
| 1111 | shouldReduceRegisterPressure(MachineBasicBlock *MBB, |
| 1112 | RegisterClassInfo *RegClassInfo) const { |
| 1113 | return false; |
| 1114 | } |
| 1115 | |
| 1116 | /// Fix up the placeholder we may add in genAlternativeCodeSequence(). |
| 1117 | virtual void |
| 1118 | finalizeInsInstrs(MachineInstr &Root, MachineCombinerPattern &P, |
| 1119 | SmallVectorImpl<MachineInstr *> &InsInstrs) const {} |
| 1120 | |
| 1121 | /// Return true when a code sequence can improve throughput. It |
| 1122 | /// should be called only for instructions in loops. |
| 1123 | /// \param Pattern - combiner pattern |
| 1124 | virtual bool isThroughputPattern(MachineCombinerPattern Pattern) const; |
| 1125 | |
| 1126 | /// Return true if the input \P Inst is part of a chain of dependent ops |
| 1127 | /// that are suitable for reassociation, otherwise return false. |
| 1128 | /// If the instruction's operands must be commuted to have a previous |
| 1129 | /// instruction of the same type define the first source operand, \P Commuted |
| 1130 | /// will be set to true. |
| 1131 | bool isReassociationCandidate(const MachineInstr &Inst, bool &Commuted) const; |
| 1132 | |
| 1133 | /// Return true when \P Inst is both associative and commutative. |
| 1134 | virtual bool isAssociativeAndCommutative(const MachineInstr &Inst) const { |
| 1135 | return false; |
| 1136 | } |
| 1137 | |
| 1138 | /// Return true when \P Inst has reassociable operands in the same \P MBB. |
| 1139 | virtual bool hasReassociableOperands(const MachineInstr &Inst, |
| 1140 | const MachineBasicBlock *MBB) const; |
| 1141 | |
| 1142 | /// Return true when \P Inst has reassociable sibling. |
| 1143 | bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const; |
| 1144 | |
| 1145 | /// When getMachineCombinerPatterns() finds patterns, this function generates |
| 1146 | /// the instructions that could replace the original code sequence. The client |
| 1147 | /// has to decide whether the actual replacement is beneficial or not. |
| 1148 | /// \param Root - Instruction that could be combined with one of its operands |
| 1149 | /// \param Pattern - Combination pattern for Root |
| 1150 | /// \param InsInstrs - Vector of new instructions that implement P |
| 1151 | /// \param DelInstrs - Old instructions, including Root, that could be |
| 1152 | /// replaced by InsInstr |
| 1153 | /// \param InstIdxForVirtReg - map of virtual register to instruction in |
| 1154 | /// InsInstr that defines it |
| 1155 | virtual void genAlternativeCodeSequence( |
| 1156 | MachineInstr &Root, MachineCombinerPattern Pattern, |
| 1157 | SmallVectorImpl<MachineInstr *> &InsInstrs, |
| 1158 | SmallVectorImpl<MachineInstr *> &DelInstrs, |
| 1159 | DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const; |
| 1160 | |
| 1161 | /// Attempt to reassociate \P Root and \P Prev according to \P Pattern to |
| 1162 | /// reduce critical path length. |
| 1163 | void reassociateOps(MachineInstr &Root, MachineInstr &Prev, |
| 1164 | MachineCombinerPattern Pattern, |
| 1165 | SmallVectorImpl<MachineInstr *> &InsInstrs, |
| 1166 | SmallVectorImpl<MachineInstr *> &DelInstrs, |
| 1167 | DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const; |
| 1168 | |
| 1169 | /// The limit on resource length extension we accept in MachineCombiner Pass. |
| 1170 | virtual int getExtendResourceLenLimit() const { return 0; } |
| 1171 | |
| 1172 | /// This is an architecture-specific helper function of reassociateOps. |
| 1173 | /// Set special operand attributes for new instructions after reassociation. |
| 1174 | virtual void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, |
| 1175 | MachineInstr &NewMI1, |
| 1176 | MachineInstr &NewMI2) const {} |
| 1177 | |
| 1178 | virtual void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const {} |
| 1179 | |
| 1180 | /// Return true when a target supports MachineCombiner. |
| 1181 | virtual bool useMachineCombiner() const { return false; } |
| 1182 | |
| 1183 | /// Return true if the given SDNode can be copied during scheduling |
| 1184 | /// even if it has glue. |
| 1185 | virtual bool canCopyGluedNodeDuringSchedule(SDNode *N) const { return false; } |
| 1186 | |
| 1187 | protected: |
| 1188 | /// Target-dependent implementation for foldMemoryOperand. |
| 1189 | /// Target-independent code in foldMemoryOperand will |
| 1190 | /// take care of adding a MachineMemOperand to the newly created instruction. |
| 1191 | /// The instruction and any auxiliary instructions necessary will be inserted |
| 1192 | /// at InsertPt. |
| 1193 | virtual MachineInstr * |
| 1194 | foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, |
| 1195 | ArrayRef<unsigned> Ops, |
| 1196 | MachineBasicBlock::iterator InsertPt, int FrameIndex, |
| 1197 | LiveIntervals *LIS = nullptr, |
| 1198 | VirtRegMap *VRM = nullptr) const { |
| 1199 | return nullptr; |
| 1200 | } |
| 1201 | |
| 1202 | /// Target-dependent implementation for foldMemoryOperand. |
| 1203 | /// Target-independent code in foldMemoryOperand will |
| 1204 | /// take care of adding a MachineMemOperand to the newly created instruction. |
| 1205 | /// The instruction and any auxiliary instructions necessary will be inserted |
| 1206 | /// at InsertPt. |
| 1207 | virtual MachineInstr *foldMemoryOperandImpl( |
| 1208 | MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, |
| 1209 | MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI, |
| 1210 | LiveIntervals *LIS = nullptr) const { |
| 1211 | return nullptr; |
| 1212 | } |
| 1213 | |
| 1214 | /// Target-dependent implementation of getRegSequenceInputs. |
| 1215 | /// |
| 1216 | /// \returns true if it is possible to build the equivalent |
| 1217 | /// REG_SEQUENCE inputs with the pair \p MI, \p DefIdx. False otherwise. |
| 1218 | /// |
| 1219 | /// \pre MI.isRegSequenceLike(). |
| 1220 | /// |
| 1221 | /// \see TargetInstrInfo::getRegSequenceInputs. |
| 1222 | virtual bool getRegSequenceLikeInputs( |
| 1223 | const MachineInstr &MI, unsigned DefIdx, |
| 1224 | SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { |
| 1225 | return false; |
| 1226 | } |
| 1227 | |
| 1228 | /// Target-dependent implementation of getExtractSubregInputs. |
| 1229 | /// |
| 1230 | /// \returns true if it is possible to build the equivalent |
| 1231 | /// EXTRACT_SUBREG inputs with the pair \p MI, \p DefIdx. False otherwise. |
| 1232 | /// |
| 1233 | /// \pre MI.isExtractSubregLike(). |
| 1234 | /// |
| 1235 | /// \see TargetInstrInfo::getExtractSubregInputs. |
| 1236 | virtual bool getExtractSubregLikeInputs(const MachineInstr &MI, |
| 1237 | unsigned DefIdx, |
| 1238 | RegSubRegPairAndIdx &InputReg) const { |
| 1239 | return false; |
| 1240 | } |
| 1241 | |
| 1242 | /// Target-dependent implementation of getInsertSubregInputs. |
| 1243 | /// |
| 1244 | /// \returns true if it is possible to build the equivalent |
| 1245 | /// INSERT_SUBREG inputs with the pair \p MI, \p DefIdx. False otherwise. |
| 1246 | /// |
| 1247 | /// \pre MI.isInsertSubregLike(). |
| 1248 | /// |
| 1249 | /// \see TargetInstrInfo::getInsertSubregInputs. |
| 1250 | virtual bool |
| 1251 | getInsertSubregLikeInputs(const MachineInstr &MI, unsigned DefIdx, |
| 1252 | RegSubRegPair &BaseReg, |
| 1253 | RegSubRegPairAndIdx &InsertedReg) const { |
| 1254 | return false; |
| 1255 | } |
| 1256 | |
| 1257 | public: |
| 1258 | /// getAddressSpaceForPseudoSourceKind - Given the kind of memory |
| 1259 | /// (e.g. stack) the target returns the corresponding address space. |
| 1260 | virtual unsigned |
| 1261 | getAddressSpaceForPseudoSourceKind(unsigned Kind) const { |
| 1262 | return 0; |
| 1263 | } |
| 1264 | |
| 1265 | /// unfoldMemoryOperand - Separate a single instruction which folded a load or |
| 1266 | /// a store or a load and a store into two or more instruction. If this is |
| 1267 | /// possible, returns true as well as the new instructions by reference. |
| 1268 | virtual bool |
| 1269 | unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, unsigned Reg, |
| 1270 | bool UnfoldLoad, bool UnfoldStore, |
| 1271 | SmallVectorImpl<MachineInstr *> &NewMIs) const { |
| 1272 | return false; |
| 1273 | } |
| 1274 | |
| 1275 | virtual bool unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, |
| 1276 | SmallVectorImpl<SDNode *> &NewNodes) const { |
| 1277 | return false; |
| 1278 | } |
| 1279 | |
| 1280 | /// Returns the opcode of the would be new |
| 1281 | /// instruction after load / store are unfolded from an instruction of the |
| 1282 | /// specified opcode. It returns zero if the specified unfolding is not |
| 1283 | /// possible. If LoadRegIndex is non-null, it is filled in with the operand |
| 1284 | /// index of the operand which will hold the register holding the loaded |
| 1285 | /// value. |
| 1286 | virtual unsigned |
| 1287 | getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, |
| 1288 | unsigned *LoadRegIndex = nullptr) const { |
| 1289 | return 0; |
| 1290 | } |
| 1291 | |
| 1292 | /// This is used by the pre-regalloc scheduler to determine if two loads are |
| 1293 | /// loading from the same base address. It should only return true if the base |
| 1294 | /// pointers are the same and the only differences between the two addresses |
| 1295 | /// are the offset. It also returns the offsets by reference. |
| 1296 | virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, |
| 1297 | int64_t &Offset1, |
| 1298 | int64_t &Offset2) const { |
| 1299 | return false; |
| 1300 | } |
| 1301 | |
| 1302 | /// This is a used by the pre-regalloc scheduler to determine (in conjunction |
| 1303 | /// with areLoadsFromSameBasePtr) if two loads should be scheduled together. |
| 1304 | /// On some targets if two loads are loading from |
| 1305 | /// addresses in the same cache line, it's better if they are scheduled |
| 1306 | /// together. This function takes two integers that represent the load offsets |
| 1307 | /// from the common base address. It returns true if it decides it's desirable |
| 1308 | /// to schedule the two loads together. "NumLoads" is the number of loads that |
| 1309 | /// have already been scheduled after Load1. |
| 1310 | virtual bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, |
| 1311 | int64_t Offset1, int64_t Offset2, |
| 1312 | unsigned NumLoads) const { |
| 1313 | return false; |
| 1314 | } |
| 1315 | |
| 1316 | /// Get the base operand and byte offset of an instruction that reads/writes |
| 1317 | /// memory. This is a convenience function for callers that are only prepared |
| 1318 | /// to handle a single base operand. |
| 1319 | bool getMemOperandWithOffset(const MachineInstr &MI, |
| 1320 | const MachineOperand *&BaseOp, int64_t &Offset, |
| 1321 | bool &OffsetIsScalable, |
| 1322 | const TargetRegisterInfo *TRI) const; |
| 1323 | |
| 1324 | /// Get zero or more base operands and the byte offset of an instruction that |
| 1325 | /// reads/writes memory. Note that there may be zero base operands if the |
| 1326 | /// instruction accesses a constant address. |
| 1327 | /// It returns false if MI does not read/write memory. |
| 1328 | /// It returns false if base operands and offset could not be determined. |
| 1329 | /// It is not guaranteed to always recognize base operands and offsets in all |
| 1330 | /// cases. |
| 1331 | virtual bool getMemOperandsWithOffsetWidth( |
| 1332 | const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps, |
| 1333 | int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, |
| 1334 | const TargetRegisterInfo *TRI) const { |
| 1335 | return false; |
| 1336 | } |
| 1337 | |
| 1338 | /// Return true if the instruction contains a base register and offset. If |
| 1339 | /// true, the function also sets the operand position in the instruction |
| 1340 | /// for the base register and offset. |
| 1341 | virtual bool getBaseAndOffsetPosition(const MachineInstr &MI, |
| 1342 | unsigned &BasePos, |
| 1343 | unsigned &OffsetPos) const { |
| 1344 | return false; |
| 1345 | } |
| 1346 | |
| 1347 | /// Target dependent implementation to get the values constituting the address |
| 1348 | /// MachineInstr that is accessing memory. These values are returned as a |
| 1349 | /// struct ExtAddrMode which contains all relevant information to make up the |
| 1350 | /// address. |
| 1351 | virtual Optional<ExtAddrMode> |
| 1352 | getAddrModeFromMemoryOp(const MachineInstr &MemI, |
| 1353 | const TargetRegisterInfo *TRI) const { |
| 1354 | return None; |
| 1355 | } |
| 1356 | |
| 1357 | /// Returns true if MI's Def is NullValueReg, and the MI |
| 1358 | /// does not change the Zero value. i.e. cases such as rax = shr rax, X where |
| 1359 | /// NullValueReg = rax. Note that if the NullValueReg is non-zero, this |
| 1360 | /// function can return true even if becomes zero. Specifically cases such as |
| 1361 | /// NullValueReg = shl NullValueReg, 63. |
| 1362 | virtual bool preservesZeroValueInReg(const MachineInstr *MI, |
| 1363 | const Register NullValueReg, |
| 1364 | const TargetRegisterInfo *TRI) const { |
| 1365 | return false; |
| 1366 | } |
| 1367 | |
| 1368 | /// If the instruction is an increment of a constant value, return the amount. |
| 1369 | virtual bool getIncrementValue(const MachineInstr &MI, int &Value) const { |
| 1370 | return false; |
| 1371 | } |
| 1372 | |
| 1373 | /// Returns true if the two given memory operations should be scheduled |
| 1374 | /// adjacent. Note that you have to add: |
| 1375 | /// DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); |
| 1376 | /// or |
| 1377 | /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); |
| 1378 | /// to TargetPassConfig::createMachineScheduler() to have an effect. |
| 1379 | /// |
| 1380 | /// \p BaseOps1 and \p BaseOps2 are memory operands of two memory operations. |
| 1381 | /// \p NumLoads is the number of loads that will be in the cluster if this |
| 1382 | /// hook returns true. |
| 1383 | /// \p NumBytes is the number of bytes that will be loaded from all the |
| 1384 | /// clustered loads if this hook returns true. |
| 1385 | virtual bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, |
| 1386 | ArrayRef<const MachineOperand *> BaseOps2, |
| 1387 | unsigned NumLoads, unsigned NumBytes) const { |
| 1388 | llvm_unreachable("target did not implement shouldClusterMemOps()")::llvm::llvm_unreachable_internal("target did not implement shouldClusterMemOps()" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1388); |
| 1389 | } |
| 1390 | |
| 1391 | /// Reverses the branch condition of the specified condition list, |
| 1392 | /// returning false on success and true if it cannot be reversed. |
| 1393 | virtual bool |
| 1394 | reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { |
| 1395 | return true; |
| 1396 | } |
| 1397 | |
| 1398 | /// Insert a noop into the instruction stream at the specified point. |
| 1399 | virtual void insertNoop(MachineBasicBlock &MBB, |
| 1400 | MachineBasicBlock::iterator MI) const; |
| 1401 | |
| 1402 | /// Insert noops into the instruction stream at the specified point. |
| 1403 | virtual void insertNoops(MachineBasicBlock &MBB, |
| 1404 | MachineBasicBlock::iterator MI, |
| 1405 | unsigned Quantity) const; |
| 1406 | |
| 1407 | /// Return the noop instruction to use for a noop. |
| 1408 | virtual MCInst getNop() const; |
| 1409 | |
| 1410 | /// Return true for post-incremented instructions. |
| 1411 | virtual bool isPostIncrement(const MachineInstr &MI) const { return false; } |
| 1412 | |
| 1413 | /// Returns true if the instruction is already predicated. |
| 1414 | virtual bool isPredicated(const MachineInstr &MI) const { return false; } |
| 1415 | |
| 1416 | // Returns a MIRPrinter comment for this machine operand. |
| 1417 | virtual std::string |
| 1418 | createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, |
| 1419 | unsigned OpIdx, const TargetRegisterInfo *TRI) const; |
| 1420 | |
| 1421 | /// Returns true if the instruction is a |
| 1422 | /// terminator instruction that has not been predicated. |
| 1423 | bool isUnpredicatedTerminator(const MachineInstr &MI) const; |
| 1424 | |
| 1425 | /// Returns true if MI is an unconditional tail call. |
| 1426 | virtual bool isUnconditionalTailCall(const MachineInstr &MI) const { |
| 1427 | return false; |
| 1428 | } |
| 1429 | |
| 1430 | /// Returns true if the tail call can be made conditional on BranchCond. |
| 1431 | virtual bool canMakeTailCallConditional(SmallVectorImpl<MachineOperand> &Cond, |
| 1432 | const MachineInstr &TailCall) const { |
| 1433 | return false; |
| 1434 | } |
| 1435 | |
| 1436 | /// Replace the conditional branch in MBB with a conditional tail call. |
| 1437 | virtual void replaceBranchWithTailCall(MachineBasicBlock &MBB, |
| 1438 | SmallVectorImpl<MachineOperand> &Cond, |
| 1439 | const MachineInstr &TailCall) const { |
| 1440 | llvm_unreachable("Target didn't implement replaceBranchWithTailCall!")::llvm::llvm_unreachable_internal("Target didn't implement replaceBranchWithTailCall!" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1440); |
| 1441 | } |
| 1442 | |
| 1443 | /// Convert the instruction into a predicated instruction. |
| 1444 | /// It returns true if the operation was successful. |
| 1445 | virtual bool PredicateInstruction(MachineInstr &MI, |
| 1446 | ArrayRef<MachineOperand> Pred) const; |
| 1447 | |
| 1448 | /// Returns true if the first specified predicate |
| 1449 | /// subsumes the second, e.g. GE subsumes GT. |
| 1450 | virtual bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1, |
| 1451 | ArrayRef<MachineOperand> Pred2) const { |
| 1452 | return false; |
| 1453 | } |
| 1454 | |
| 1455 | /// If the specified instruction defines any predicate |
| 1456 | /// or condition code register(s) used for predication, returns true as well |
| 1457 | /// as the definition predicate(s) by reference. |
| 1458 | /// SkipDead should be set to false at any point that dead |
| 1459 | /// predicate instructions should be considered as being defined. |
| 1460 | /// A dead predicate instruction is one that is guaranteed to be removed |
| 1461 | /// after a call to PredicateInstruction. |
| 1462 | virtual bool ClobbersPredicate(MachineInstr &MI, |
| 1463 | std::vector<MachineOperand> &Pred, |
| 1464 | bool SkipDead) const { |
| 1465 | return false; |
| 1466 | } |
| 1467 | |
| 1468 | /// Return true if the specified instruction can be predicated. |
| 1469 | /// By default, this returns true for every instruction with a |
| 1470 | /// PredicateOperand. |
| 1471 | virtual bool isPredicable(const MachineInstr &MI) const { |
| 1472 | return MI.getDesc().isPredicable(); |
| 1473 | } |
| 1474 | |
| 1475 | /// Return true if it's safe to move a machine |
| 1476 | /// instruction that defines the specified register class. |
| 1477 | virtual bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const { |
| 1478 | return true; |
| 1479 | } |
| 1480 | |
| 1481 | /// Test if the given instruction should be considered a scheduling boundary. |
| 1482 | /// This primarily includes labels and terminators. |
| 1483 | virtual bool isSchedulingBoundary(const MachineInstr &MI, |
| 1484 | const MachineBasicBlock *MBB, |
| 1485 | const MachineFunction &MF) const; |
| 1486 | |
| 1487 | /// Measure the specified inline asm to determine an approximation of its |
| 1488 | /// length. |
| 1489 | virtual unsigned getInlineAsmLength( |
| 1490 | const char *Str, const MCAsmInfo &MAI, |
| 1491 | const TargetSubtargetInfo *STI = nullptr) const; |
| 1492 | |
| 1493 | /// Allocate and return a hazard recognizer to use for this target when |
| 1494 | /// scheduling the machine instructions before register allocation. |
| 1495 | virtual ScheduleHazardRecognizer * |
| 1496 | CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, |
| 1497 | const ScheduleDAG *DAG) const; |
| 1498 | |
| 1499 | /// Allocate and return a hazard recognizer to use for this target when |
| 1500 | /// scheduling the machine instructions before register allocation. |
| 1501 | virtual ScheduleHazardRecognizer * |
| 1502 | CreateTargetMIHazardRecognizer(const InstrItineraryData *, |
| 1503 | const ScheduleDAGMI *DAG) const; |
| 1504 | |
| 1505 | /// Allocate and return a hazard recognizer to use for this target when |
| 1506 | /// scheduling the machine instructions after register allocation. |
| 1507 | virtual ScheduleHazardRecognizer * |
| 1508 | CreateTargetPostRAHazardRecognizer(const InstrItineraryData *, |
| 1509 | const ScheduleDAG *DAG) const; |
| 1510 | |
| 1511 | /// Allocate and return a hazard recognizer to use for by non-scheduling |
| 1512 | /// passes. |
| 1513 | virtual ScheduleHazardRecognizer * |
| 1514 | CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { |
| 1515 | return nullptr; |
| 1516 | } |
| 1517 | |
| 1518 | /// Provide a global flag for disabling the PreRA hazard recognizer that |
| 1519 | /// targets may choose to honor. |
| 1520 | bool usePreRAHazardRecognizer() const; |
| 1521 | |
| 1522 | /// For a comparison instruction, return the source registers |
| 1523 | /// in SrcReg and SrcReg2 if having two register operands, and the value it |
| 1524 | /// compares against in CmpValue. Return true if the comparison instruction |
| 1525 | /// can be analyzed. |
| 1526 | virtual bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, |
| 1527 | Register &SrcReg2, int &Mask, int &Value) const { |
| 1528 | return false; |
| 1529 | } |
| 1530 | |
| 1531 | /// See if the comparison instruction can be converted |
| 1532 | /// into something more efficient. E.g., on ARM most instructions can set the |
| 1533 | /// flags register, obviating the need for a separate CMP. |
| 1534 | virtual bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, |
| 1535 | Register SrcReg2, int Mask, int Value, |
| 1536 | const MachineRegisterInfo *MRI) const { |
| 1537 | return false; |
| 1538 | } |
| 1539 | virtual bool optimizeCondBranch(MachineInstr &MI) const { return false; } |
| 1540 | |
| 1541 | /// Try to remove the load by folding it to a register operand at the use. |
| 1542 | /// We fold the load instructions if and only if the |
| 1543 | /// def and use are in the same BB. We only look at one load and see |
| 1544 | /// whether it can be folded into MI. FoldAsLoadDefReg is the virtual register |
| 1545 | /// defined by the load we are trying to fold. DefMI returns the machine |
| 1546 | /// instruction that defines FoldAsLoadDefReg, and the function returns |
| 1547 | /// the machine instruction generated due to folding. |
| 1548 | virtual MachineInstr *optimizeLoadInstr(MachineInstr &MI, |
| 1549 | const MachineRegisterInfo *MRI, |
| 1550 | Register &FoldAsLoadDefReg, |
| 1551 | MachineInstr *&DefMI) const { |
| 1552 | return nullptr; |
| 1553 | } |
| 1554 | |
| 1555 | /// 'Reg' is known to be defined by a move immediate instruction, |
| 1556 | /// try to fold the immediate into the use instruction. |
| 1557 | /// If MRI->hasOneNonDBGUse(Reg) is true, and this function returns true, |
| 1558 | /// then the caller may assume that DefMI has been erased from its parent |
| 1559 | /// block. The caller may assume that it will not be erased by this |
| 1560 | /// function otherwise. |
| 1561 | virtual bool FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, |
| 1562 | Register Reg, MachineRegisterInfo *MRI) const { |
| 1563 | return false; |
| 1564 | } |
| 1565 | |
| 1566 | /// Return the number of u-operations the given machine |
| 1567 | /// instruction will be decoded to on the target cpu. The itinerary's |
| 1568 | /// IssueWidth is the number of microops that can be dispatched each |
| 1569 | /// cycle. An instruction with zero microops takes no dispatch resources. |
| 1570 | virtual unsigned getNumMicroOps(const InstrItineraryData *ItinData, |
| 1571 | const MachineInstr &MI) const; |
| 1572 | |
| 1573 | /// Return true for pseudo instructions that don't consume any |
| 1574 | /// machine resources in their current form. These are common cases that the |
| 1575 | /// scheduler should consider free, rather than conservatively handling them |
| 1576 | /// as instructions with no itinerary. |
| 1577 | bool isZeroCost(unsigned Opcode) const { |
| 1578 | return Opcode <= TargetOpcode::COPY; |
| 1579 | } |
| 1580 | |
| 1581 | virtual int getOperandLatency(const InstrItineraryData *ItinData, |
| 1582 | SDNode *DefNode, unsigned DefIdx, |
| 1583 | SDNode *UseNode, unsigned UseIdx) const; |
| 1584 | |
| 1585 | /// Compute and return the use operand latency of a given pair of def and use. |
| 1586 | /// In most cases, the static scheduling itinerary was enough to determine the |
| 1587 | /// operand latency. But it may not be possible for instructions with variable |
| 1588 | /// number of defs / uses. |
| 1589 | /// |
| 1590 | /// This is a raw interface to the itinerary that may be directly overridden |
| 1591 | /// by a target. Use computeOperandLatency to get the best estimate of |
| 1592 | /// latency. |
| 1593 | virtual int getOperandLatency(const InstrItineraryData *ItinData, |
| 1594 | const MachineInstr &DefMI, unsigned DefIdx, |
| 1595 | const MachineInstr &UseMI, |
| 1596 | unsigned UseIdx) const; |
| 1597 | |
| 1598 | /// Compute the instruction latency of a given instruction. |
| 1599 | /// If the instruction has higher cost when predicated, it's returned via |
| 1600 | /// PredCost. |
| 1601 | virtual unsigned getInstrLatency(const InstrItineraryData *ItinData, |
| 1602 | const MachineInstr &MI, |
| 1603 | unsigned *PredCost = nullptr) const; |
| 1604 | |
| 1605 | virtual unsigned getPredicationCost(const MachineInstr &MI) const; |
| 1606 | |
| 1607 | virtual int getInstrLatency(const InstrItineraryData *ItinData, |
| 1608 | SDNode *Node) const; |
| 1609 | |
| 1610 | /// Return the default expected latency for a def based on its opcode. |
| 1611 | unsigned defaultDefLatency(const MCSchedModel &SchedModel, |
| 1612 | const MachineInstr &DefMI) const; |
| 1613 | |
| 1614 | int computeDefOperandLatency(const InstrItineraryData *ItinData, |
| 1615 | const MachineInstr &DefMI) const; |
| 1616 | |
| 1617 | /// Return true if this opcode has high latency to its result. |
| 1618 | virtual bool isHighLatencyDef(int opc) const { return false; } |
| 1619 | |
| 1620 | /// Compute operand latency between a def of 'Reg' |
| 1621 | /// and a use in the current loop. Return true if the target considered |
| 1622 | /// it 'high'. This is used by optimization passes such as machine LICM to |
| 1623 | /// determine whether it makes sense to hoist an instruction out even in a |
| 1624 | /// high register pressure situation. |
| 1625 | virtual bool hasHighOperandLatency(const TargetSchedModel &SchedModel, |
| 1626 | const MachineRegisterInfo *MRI, |
| 1627 | const MachineInstr &DefMI, unsigned DefIdx, |
| 1628 | const MachineInstr &UseMI, |
| 1629 | unsigned UseIdx) const { |
| 1630 | return false; |
| 1631 | } |
| 1632 | |
| 1633 | /// Compute operand latency of a def of 'Reg'. Return true |
| 1634 | /// if the target considered it 'low'. |
| 1635 | virtual bool hasLowDefLatency(const TargetSchedModel &SchedModel, |
| 1636 | const MachineInstr &DefMI, |
| 1637 | unsigned DefIdx) const; |
| 1638 | |
| 1639 | /// Perform target-specific instruction verification. |
| 1640 | virtual bool verifyInstruction(const MachineInstr &MI, |
| 1641 | StringRef &ErrInfo) const { |
| 1642 | return true; |
| 1643 | } |
| 1644 | |
| 1645 | /// Return the current execution domain and bit mask of |
| 1646 | /// possible domains for instruction. |
| 1647 | /// |
| 1648 | /// Some micro-architectures have multiple execution domains, and multiple |
| 1649 | /// opcodes that perform the same operation in different domains. For |
| 1650 | /// example, the x86 architecture provides the por, orps, and orpd |
| 1651 | /// instructions that all do the same thing. There is a latency penalty if a |
| 1652 | /// register is written in one domain and read in another. |
| 1653 | /// |
| 1654 | /// This function returns a pair (domain, mask) containing the execution |
| 1655 | /// domain of MI, and a bit mask of possible domains. The setExecutionDomain |
| 1656 | /// function can be used to change the opcode to one of the domains in the |
| 1657 | /// bit mask. Instructions whose execution domain can't be changed should |
| 1658 | /// return a 0 mask. |
| 1659 | /// |
| 1660 | /// The execution domain numbers don't have any special meaning except domain |
| 1661 | /// 0 is used for instructions that are not associated with any interesting |
| 1662 | /// execution domain. |
| 1663 | /// |
| 1664 | virtual std::pair<uint16_t, uint16_t> |
| 1665 | getExecutionDomain(const MachineInstr &MI) const { |
| 1666 | return std::make_pair(0, 0); |
| 1667 | } |
| 1668 | |
| 1669 | /// Change the opcode of MI to execute in Domain. |
| 1670 | /// |
| 1671 | /// The bit (1 << Domain) must be set in the mask returned from |
| 1672 | /// getExecutionDomain(MI). |
| 1673 | virtual void setExecutionDomain(MachineInstr &MI, unsigned Domain) const {} |
| 1674 | |
| 1675 | /// Returns the preferred minimum clearance |
| 1676 | /// before an instruction with an unwanted partial register update. |
| 1677 | /// |
| 1678 | /// Some instructions only write part of a register, and implicitly need to |
| 1679 | /// read the other parts of the register. This may cause unwanted stalls |
| 1680 | /// preventing otherwise unrelated instructions from executing in parallel in |
| 1681 | /// an out-of-order CPU. |
| 1682 | /// |
| 1683 | /// For example, the x86 instruction cvtsi2ss writes its result to bits |
| 1684 | /// [31:0] of the destination xmm register. Bits [127:32] are unaffected, so |
| 1685 | /// the instruction needs to wait for the old value of the register to become |
| 1686 | /// available: |
| 1687 | /// |
| 1688 | /// addps %xmm1, %xmm0 |
| 1689 | /// movaps %xmm0, (%rax) |
| 1690 | /// cvtsi2ss %rbx, %xmm0 |
| 1691 | /// |
| 1692 | /// In the code above, the cvtsi2ss instruction needs to wait for the addps |
| 1693 | /// instruction before it can issue, even though the high bits of %xmm0 |
| 1694 | /// probably aren't needed. |
| 1695 | /// |
| 1696 | /// This hook returns the preferred clearance before MI, measured in |
| 1697 | /// instructions. Other defs of MI's operand OpNum are avoided in the last N |
| 1698 | /// instructions before MI. It should only return a positive value for |
| 1699 | /// unwanted dependencies. If the old bits of the defined register have |
| 1700 | /// useful values, or if MI is determined to otherwise read the dependency, |
| 1701 | /// the hook should return 0. |
| 1702 | /// |
| 1703 | /// The unwanted dependency may be handled by: |
| 1704 | /// |
| 1705 | /// 1. Allocating the same register for an MI def and use. That makes the |
| 1706 | /// unwanted dependency identical to a required dependency. |
| 1707 | /// |
| 1708 | /// 2. Allocating a register for the def that has no defs in the previous N |
| 1709 | /// instructions. |
| 1710 | /// |
| 1711 | /// 3. Calling breakPartialRegDependency() with the same arguments. This |
| 1712 | /// allows the target to insert a dependency breaking instruction. |
| 1713 | /// |
| 1714 | virtual unsigned |
| 1715 | getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum, |
| 1716 | const TargetRegisterInfo *TRI) const { |
| 1717 | // The default implementation returns 0 for no partial register dependency. |
| 1718 | return 0; |
| 1719 | } |
| 1720 | |
| 1721 | /// Return the minimum clearance before an instruction that reads an |
| 1722 | /// unused register. |
| 1723 | /// |
| 1724 | /// For example, AVX instructions may copy part of a register operand into |
| 1725 | /// the unused high bits of the destination register. |
| 1726 | /// |
| 1727 | /// vcvtsi2sdq %rax, undef %xmm0, %xmm14 |
| 1728 | /// |
| 1729 | /// In the code above, vcvtsi2sdq copies %xmm0[127:64] into %xmm14 creating a |
| 1730 | /// false dependence on any previous write to %xmm0. |
| 1731 | /// |
| 1732 | /// This hook works similarly to getPartialRegUpdateClearance, except that it |
| 1733 | /// does not take an operand index. Instead sets \p OpNum to the index of the |
| 1734 | /// unused register. |
| 1735 | virtual unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum, |
| 1736 | const TargetRegisterInfo *TRI) const { |
| 1737 | // The default implementation returns 0 for no undef register dependency. |
| 1738 | return 0; |
| 1739 | } |
| 1740 | |
| 1741 | /// Insert a dependency-breaking instruction |
| 1742 | /// before MI to eliminate an unwanted dependency on OpNum. |
| 1743 | /// |
| 1744 | /// If it wasn't possible to avoid a def in the last N instructions before MI |
| 1745 | /// (see getPartialRegUpdateClearance), this hook will be called to break the |
| 1746 | /// unwanted dependency. |
| 1747 | /// |
| 1748 | /// On x86, an xorps instruction can be used as a dependency breaker: |
| 1749 | /// |
| 1750 | /// addps %xmm1, %xmm0 |
| 1751 | /// movaps %xmm0, (%rax) |
| 1752 | /// xorps %xmm0, %xmm0 |
| 1753 | /// cvtsi2ss %rbx, %xmm0 |
| 1754 | /// |
| 1755 | /// An <imp-kill> operand should be added to MI if an instruction was |
| 1756 | /// inserted. This ties the instructions together in the post-ra scheduler. |
| 1757 | /// |
| 1758 | virtual void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum, |
| 1759 | const TargetRegisterInfo *TRI) const {} |
| 1760 | |
| 1761 | /// Create machine specific model for scheduling. |
| 1762 | virtual DFAPacketizer * |
| 1763 | CreateTargetScheduleState(const TargetSubtargetInfo &) const { |
| 1764 | return nullptr; |
| 1765 | } |
| 1766 | |
| 1767 | /// Sometimes, it is possible for the target |
| 1768 | /// to tell, even without aliasing information, that two MIs access different |
| 1769 | /// memory addresses. This function returns true if two MIs access different |
| 1770 | /// memory addresses and false otherwise. |
| 1771 | /// |
| 1772 | /// Assumes any physical registers used to compute addresses have the same |
| 1773 | /// value for both instructions. (This is the most useful assumption for |
| 1774 | /// post-RA scheduling.) |
| 1775 | /// |
| 1776 | /// See also MachineInstr::mayAlias, which is implemented on top of this |
| 1777 | /// function. |
| 1778 | virtual bool |
| 1779 | areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, |
| 1780 | const MachineInstr &MIb) const { |
| 1781 | assert(MIa.mayLoadOrStore() &&((MIa.mayLoadOrStore() && "MIa must load from or modify a memory location" ) ? static_cast<void> (0) : __assert_fail ("MIa.mayLoadOrStore() && \"MIa must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1782, __PRETTY_FUNCTION__)) |
| 1782 | "MIa must load from or modify a memory location")((MIa.mayLoadOrStore() && "MIa must load from or modify a memory location" ) ? static_cast<void> (0) : __assert_fail ("MIa.mayLoadOrStore() && \"MIa must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1782, __PRETTY_FUNCTION__)); |
| 1783 | assert(MIb.mayLoadOrStore() &&((MIb.mayLoadOrStore() && "MIb must load from or modify a memory location" ) ? static_cast<void> (0) : __assert_fail ("MIb.mayLoadOrStore() && \"MIb must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1784, __PRETTY_FUNCTION__)) |
| 1784 | "MIb must load from or modify a memory location")((MIb.mayLoadOrStore() && "MIb must load from or modify a memory location" ) ? static_cast<void> (0) : __assert_fail ("MIb.mayLoadOrStore() && \"MIb must load from or modify a memory location\"" , "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include/llvm/CodeGen/TargetInstrInfo.h" , 1784, __PRETTY_FUNCTION__)); |
| 1785 | return false; |
| 1786 | } |
| 1787 | |
| 1788 | /// Return the value to use for the MachineCSE's LookAheadLimit, |
| 1789 | /// which is a heuristic used for CSE'ing phys reg defs. |
| 1790 | virtual unsigned getMachineCSELookAheadLimit() const { |
| 1791 | // The default lookahead is small to prevent unprofitable quadratic |
| 1792 | // behavior. |
| 1793 | return 5; |
| 1794 | } |
| 1795 | |
| 1796 | /// Return the maximal number of alias checks on memory operands. For |
| 1797 | /// instructions with more than one memory operands, the alias check on a |
| 1798 | /// single MachineInstr pair has quadratic overhead and results in |
| 1799 | /// unacceptable performance in the worst case. The limit here is to clamp |
| 1800 | /// that maximal checks performed. Usually, that's the product of memory |
| 1801 | /// operand numbers from that pair of MachineInstr to be checked. For |
| 1802 | /// instance, with two MachineInstrs with 4 and 5 memory operands |
| 1803 | /// correspondingly, a total of 20 checks are required. With this limit set to |
| 1804 | /// 16, their alias check is skipped. We choose to limit the product instead |
| 1805 | /// of the individual instruction as targets may have special MachineInstrs |
| 1806 | /// with a considerably high number of memory operands, such as `ldm` in ARM. |
| 1807 | /// Setting this limit per MachineInstr would result in either too high |
| 1808 | /// overhead or too rigid restriction. |
| 1809 | virtual unsigned getMemOperandAACheckLimit() const { return 16; } |
| 1810 | |
| 1811 | /// Return an array that contains the ids of the target indices (used for the |
| 1812 | /// TargetIndex machine operand) and their names. |
| 1813 | /// |
| 1814 | /// MIR Serialization is able to serialize only the target indices that are |
| 1815 | /// defined by this method. |
| 1816 | virtual ArrayRef<std::pair<int, const char *>> |
| 1817 | getSerializableTargetIndices() |