LCOV - code coverage report
Current view: top level - lib/CodeGen - RegisterPressure.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 533 567 94.0 %
Date: 2018-07-13 00:08:38 Functions: 60 67 89.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- RegisterPressure.cpp - Dynamic Register Pressure -------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the RegisterPressure class which can be used to track
      11             : // MachineInstr level register pressure.
      12             : //
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "llvm/CodeGen/RegisterPressure.h"
      16             : #include "llvm/ADT/ArrayRef.h"
      17             : #include "llvm/ADT/STLExtras.h"
      18             : #include "llvm/ADT/SmallVector.h"
      19             : #include "llvm/CodeGen/LiveInterval.h"
      20             : #include "llvm/CodeGen/LiveIntervals.h"
      21             : #include "llvm/CodeGen/MachineBasicBlock.h"
      22             : #include "llvm/CodeGen/MachineFunction.h"
      23             : #include "llvm/CodeGen/MachineInstr.h"
      24             : #include "llvm/CodeGen/MachineInstrBundle.h"
      25             : #include "llvm/CodeGen/MachineOperand.h"
      26             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      27             : #include "llvm/CodeGen/RegisterClassInfo.h"
      28             : #include "llvm/CodeGen/SlotIndexes.h"
      29             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      30             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      31             : #include "llvm/Config/llvm-config.h"
      32             : #include "llvm/MC/LaneBitmask.h"
      33             : #include "llvm/MC/MCRegisterInfo.h"
      34             : #include "llvm/Support/Compiler.h"
      35             : #include "llvm/Support/Debug.h"
      36             : #include "llvm/Support/ErrorHandling.h"
      37             : #include "llvm/Support/raw_ostream.h"
      38             : #include <algorithm>
      39             : #include <cassert>
      40             : #include <cstdint>
      41             : #include <cstdlib>
      42             : #include <cstring>
      43             : #include <iterator>
      44             : #include <limits>
      45             : #include <utility>
      46             : #include <vector>
      47             : 
      48             : using namespace llvm;
      49             : 
      50             : /// Increase pressure for each pressure set provided by TargetRegisterInfo.
      51      307326 : static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure,
      52             :                                 const MachineRegisterInfo &MRI, unsigned Reg,
      53             :                                 LaneBitmask PrevMask, LaneBitmask NewMask) {
      54             :   assert((PrevMask & ~NewMask).none() && "Must not remove bits");
      55      612169 :   if (PrevMask.any() || NewMask.none())
      56             :     return;
      57             : 
      58             :   PSetIterator PSetI = MRI.getPressureSets(Reg);
      59             :   unsigned Weight = PSetI.getWeight();
      60      810873 :   for (; PSetI.isValid(); ++PSetI)
      61     1012060 :     CurrSetPressure[*PSetI] += Weight;
      62             : }
      63             : 
      64             : /// Decrease pressure for each pressure set provided by TargetRegisterInfo.
      65     4952884 : static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
      66             :                                 const MachineRegisterInfo &MRI, unsigned Reg,
      67             :                                 LaneBitmask PrevMask, LaneBitmask NewMask) {
      68             :   //assert((NewMask & !PrevMask) == 0 && "Must not add bits");
      69     8905938 :   if (NewMask.any() || PrevMask.none())
      70             :     return;
      71             : 
      72             :   PSetIterator PSetI = MRI.getPressureSets(Reg);
      73             :   unsigned Weight = PSetI.getWeight();
      74    13401572 :   for (; PSetI.isValid(); ++PSetI) {
      75             :     assert(CurrSetPressure[*PSetI] >= Weight && "register pressure underflow");
      76    18897234 :     CurrSetPressure[*PSetI] -= Weight;
      77             :   }
      78             : }
      79             : 
      80             : #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
      81             : LLVM_DUMP_METHOD
      82             : void llvm::dumpRegSetPressure(ArrayRef<unsigned> SetPressure,
      83             :                               const TargetRegisterInfo *TRI) {
      84             :   bool Empty = true;
      85             :   for (unsigned i = 0, e = SetPressure.size(); i < e; ++i) {
      86             :     if (SetPressure[i] != 0) {
      87             :       dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << '\n';
      88             :       Empty = false;
      89             :     }
      90             :   }
      91             :   if (Empty)
      92             :     dbgs() << "\n";
      93             : }
      94             : 
      95             : LLVM_DUMP_METHOD
      96             : void RegisterPressure::dump(const TargetRegisterInfo *TRI) const {
      97             :   dbgs() << "Max Pressure: ";
      98             :   dumpRegSetPressure(MaxSetPressure, TRI);
      99             :   dbgs() << "Live In: ";
     100             :   for (const RegisterMaskPair &P : LiveInRegs) {
     101             :     dbgs() << printVRegOrUnit(P.RegUnit, TRI);
     102             :     if (!P.LaneMask.all())
     103             :       dbgs() << ':' << PrintLaneMask(P.LaneMask);
     104             :     dbgs() << ' ';
     105             :   }
     106             :   dbgs() << '\n';
     107             :   dbgs() << "Live Out: ";
     108             :   for (const RegisterMaskPair &P : LiveOutRegs) {
     109             :     dbgs() << printVRegOrUnit(P.RegUnit, TRI);
     110             :     if (!P.LaneMask.all())
     111             :       dbgs() << ':' << PrintLaneMask(P.LaneMask);
     112             :     dbgs() << ' ';
     113             :   }
     114             :   dbgs() << '\n';
     115             : }
     116             : 
     117             : LLVM_DUMP_METHOD
     118             : void RegPressureTracker::dump() const {
     119             :   if (!isTopClosed() || !isBottomClosed()) {
     120             :     dbgs() << "Curr Pressure: ";
     121             :     dumpRegSetPressure(CurrSetPressure, TRI);
     122             :   }
     123             :   P.dump(TRI);
     124             : }
     125             : 
     126             : LLVM_DUMP_METHOD
     127             : void PressureDiff::dump(const TargetRegisterInfo &TRI) const {
     128             :   const char *sep = "";
     129             :   for (const PressureChange &Change : *this) {
     130             :     if (!Change.isValid())
     131             :       break;
     132             :     dbgs() << sep << TRI.getRegPressureSetName(Change.getPSet())
     133             :            << " " << Change.getUnitInc();
     134             :     sep = "    ";
     135             :   }
     136             :   dbgs() << '\n';
     137             : }
     138             : #endif
     139             : 
     140     6286879 : void RegPressureTracker::increaseRegPressure(unsigned RegUnit,
     141             :                                              LaneBitmask PreviousMask,
     142             :                                              LaneBitmask NewMask) {
     143    11551886 :   if (PreviousMask.any() || NewMask.none())
     144             :     return;
     145             : 
     146     5264922 :   PSetIterator PSetI = MRI->getPressureSets(RegUnit);
     147             :   unsigned Weight = PSetI.getWeight();
     148    18262315 :   for (; PSetI.isValid(); ++PSetI) {
     149    25994786 :     CurrSetPressure[*PSetI] += Weight;
     150    12997393 :     P.MaxSetPressure[*PSetI] =
     151    38992179 :         std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]);
     152             :   }
     153             : }
     154             : 
     155     4952884 : void RegPressureTracker::decreaseRegPressure(unsigned RegUnit,
     156             :                                              LaneBitmask PreviousMask,
     157             :                                              LaneBitmask NewMask) {
     158     4952884 :   decreaseSetPressure(CurrSetPressure, *MRI, RegUnit, PreviousMask, NewMask);
     159     4952884 : }
     160             : 
     161             : /// Clear the result so it can be used for another round of pressure tracking.
     162      600154 : void IntervalPressure::reset() {
     163      600154 :   TopIdx = BottomIdx = SlotIndex();
     164             :   MaxSetPressure.clear();
     165             :   LiveInRegs.clear();
     166             :   LiveOutRegs.clear();
     167      600154 : }
     168             : 
     169             : /// Clear the result so it can be used for another round of pressure tracking.
     170           0 : void RegionPressure::reset() {
     171           0 :   TopPos = BottomPos = MachineBasicBlock::const_iterator();
     172             :   MaxSetPressure.clear();
     173             :   LiveInRegs.clear();
     174             :   LiveOutRegs.clear();
     175           0 : }
     176             : 
     177             : /// If the current top is not less than or equal to the next index, open it.
     178             : /// We happen to need the SlotIndex for the next top for pressure update.
     179           0 : void IntervalPressure::openTop(SlotIndex NextTop) {
     180           0 :   if (TopIdx <= NextTop)
     181             :     return;
     182           0 :   TopIdx = SlotIndex();
     183             :   LiveInRegs.clear();
     184             : }
     185             : 
     186             : /// If the current top is the previous instruction (before receding), open it.
     187           0 : void RegionPressure::openTop(MachineBasicBlock::const_iterator PrevTop) {
     188           0 :   if (TopPos != PrevTop)
     189             :     return;
     190           0 :   TopPos = MachineBasicBlock::const_iterator();
     191             :   LiveInRegs.clear();
     192             : }
     193             : 
     194             : /// If the current bottom is not greater than the previous index, open it.
     195           0 : void IntervalPressure::openBottom(SlotIndex PrevBottom) {
     196           0 :   if (BottomIdx > PrevBottom)
     197             :     return;
     198           0 :   BottomIdx = SlotIndex();
     199             :   LiveInRegs.clear();
     200             : }
     201             : 
     202             : /// If the current bottom is the previous instr (before advancing), open it.
     203           0 : void RegionPressure::openBottom(MachineBasicBlock::const_iterator PrevBottom) {
     204           0 :   if (BottomPos != PrevBottom)
     205             :     return;
     206           0 :   BottomPos = MachineBasicBlock::const_iterator();
     207             :   LiveInRegs.clear();
     208             : }
     209             : 
     210      341084 : void LiveRegSet::init(const MachineRegisterInfo &MRI) {
     211      341084 :   const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
     212      341084 :   unsigned NumRegUnits = TRI.getNumRegs();
     213             :   unsigned NumVirtRegs = MRI.getNumVirtRegs();
     214      341084 :   Regs.setUniverse(NumRegUnits + NumVirtRegs);
     215      341084 :   this->NumRegUnits = NumRegUnits;
     216      341084 : }
     217             : 
     218      600154 : void LiveRegSet::clear() {
     219             :   Regs.clear();
     220      600154 : }
     221             : 
     222             : static const LiveRange *getLiveRange(const LiveIntervals &LIS, unsigned Reg) {
     223      892107 :   if (TargetRegisterInfo::isVirtualRegister(Reg))
     224             :     return &LIS.getInterval(Reg);
     225             :   return LIS.getCachedRegUnit(Reg);
     226             : }
     227             : 
     228      600154 : void RegPressureTracker::reset() {
     229      600154 :   MBB = nullptr;
     230      600154 :   LIS = nullptr;
     231             : 
     232             :   CurrSetPressure.clear();
     233             :   LiveThruPressure.clear();
     234      600154 :   P.MaxSetPressure.clear();
     235             : 
     236      600154 :   if (RequireIntervals)
     237      600154 :     static_cast<IntervalPressure&>(P).reset();
     238             :   else
     239           0 :     static_cast<RegionPressure&>(P).reset();
     240             : 
     241      600154 :   LiveRegs.clear();
     242             :   UntiedDefs.clear();
     243      600154 : }
     244             : 
     245             : /// Setup the RegPressureTracker.
     246             : ///
     247             : /// TODO: Add support for pressure without LiveIntervals.
     248      341084 : void RegPressureTracker::init(const MachineFunction *mf,
     249             :                               const RegisterClassInfo *rci,
     250             :                               const LiveIntervals *lis,
     251             :                               const MachineBasicBlock *mbb,
     252             :                               MachineBasicBlock::const_iterator pos,
     253             :                               bool TrackLaneMasks, bool TrackUntiedDefs) {
     254      341084 :   reset();
     255             : 
     256      341084 :   MF = mf;
     257      341084 :   TRI = MF->getSubtarget().getRegisterInfo();
     258      341084 :   RCI = rci;
     259      341084 :   MRI = &MF->getRegInfo();
     260      341084 :   MBB = mbb;
     261      341084 :   this->TrackUntiedDefs = TrackUntiedDefs;
     262      341084 :   this->TrackLaneMasks = TrackLaneMasks;
     263             : 
     264      341084 :   if (RequireIntervals) {
     265             :     assert(lis && "IntervalPressure requires LiveIntervals");
     266      341084 :     LIS = lis;
     267             :   }
     268             : 
     269      341084 :   CurrPos = pos;
     270      682168 :   CurrSetPressure.assign(TRI->getNumRegPressureSets(), 0);
     271             : 
     272      341084 :   P.MaxSetPressure = CurrSetPressure;
     273             : 
     274      341084 :   LiveRegs.init(*MRI);
     275      341084 :   if (TrackUntiedDefs)
     276      227466 :     UntiedDefs.setUniverse(MRI->getNumVirtRegs());
     277      341084 : }
     278             : 
     279             : /// Does this pressure result have a valid top position and live ins.
     280     3250834 : bool RegPressureTracker::isTopClosed() const {
     281     3250834 :   if (RequireIntervals)
     282     6501668 :     return static_cast<IntervalPressure&>(P).TopIdx.isValid();
     283           0 :   return (static_cast<RegionPressure&>(P).TopPos ==
     284             :           MachineBasicBlock::const_iterator());
     285             : }
     286             : 
     287             : /// Does this pressure result have a valid bottom position and live outs.
     288     3144955 : bool RegPressureTracker::isBottomClosed() const {
     289     3144955 :   if (RequireIntervals)
     290     6289910 :     return static_cast<IntervalPressure&>(P).BottomIdx.isValid();
     291           0 :   return (static_cast<RegionPressure&>(P).BottomPos ==
     292             :           MachineBasicBlock::const_iterator());
     293             : }
     294             : 
     295      840542 : SlotIndex RegPressureTracker::getCurrSlot() const {
     296             :   MachineBasicBlock::const_iterator IdxPos =
     297     1681084 :     skipDebugInstructionsForward(CurrPos, MBB->end());
     298     1681084 :   if (IdxPos == MBB->end())
     299      127401 :     return LIS->getMBBEndIdx(MBB);
     300      713141 :   return LIS->getInstructionIndex(*IdxPos).getRegSlot();
     301             : }
     302             : 
     303             : /// Set the boundary for the top of the region and summarize live ins.
     304      227346 : void RegPressureTracker::closeTop() {
     305      227346 :   if (RequireIntervals)
     306      227346 :     static_cast<IntervalPressure&>(P).TopIdx = getCurrSlot();
     307             :   else
     308           0 :     static_cast<RegionPressure&>(P).TopPos = CurrPos;
     309             : 
     310             :   assert(P.LiveInRegs.empty() && "inconsistent max pressure result");
     311      454692 :   P.LiveInRegs.reserve(LiveRegs.size());
     312      227346 :   LiveRegs.appendTo(P.LiveInRegs);
     313      227346 : }
     314             : 
     315             : /// Set the boundary for the bottom of the region and summarize live outs.
     316      227406 : void RegPressureTracker::closeBottom() {
     317      227406 :   if (RequireIntervals)
     318      227406 :     static_cast<IntervalPressure&>(P).BottomIdx = getCurrSlot();
     319             :   else
     320           0 :     static_cast<RegionPressure&>(P).BottomPos = CurrPos;
     321             : 
     322             :   assert(P.LiveOutRegs.empty() && "inconsistent max pressure result");
     323      454812 :   P.LiveOutRegs.reserve(LiveRegs.size());
     324      227406 :   LiveRegs.appendTo(P.LiveOutRegs);
     325      227406 : }
     326             : 
     327             : /// Finalize the region boundaries and record live ins and live outs.
     328      113673 : void RegPressureTracker::closeRegion() {
     329      113673 :   if (!isTopClosed() && !isBottomClosed()) {
     330             :     assert(LiveRegs.size() == 0 && "no region boundary");
     331             :     return;
     332             :   }
     333      113673 :   if (!isBottomClosed())
     334           5 :     closeBottom();
     335      113668 :   else if (!isTopClosed())
     336      113668 :     closeTop();
     337             :   // If both top and bottom are closed, do nothing.
     338             : }
     339             : 
     340             : /// The register tracker is unaware of global liveness so ignores normal
     341             : /// live-thru ranges. However, two-address or coalesced chains can also lead
     342             : /// to live ranges with no holes. Count these to inform heuristics that we
     343             : /// can never drop below this pressure.
     344      113668 : void RegPressureTracker::initLiveThru(const RegPressureTracker &RPTracker) {
     345      227336 :   LiveThruPressure.assign(TRI->getNumRegPressureSets(), 0);
     346             :   assert(isBottomClosed() && "need bottom-up tracking to intialize.");
     347      558884 :   for (const RegisterMaskPair &Pair : P.LiveOutRegs) {
     348      165774 :     unsigned RegUnit = Pair.RegUnit;
     349             :     if (TargetRegisterInfo::isVirtualRegister(RegUnit)
     350      324757 :         && !RPTracker.hasUntiedDef(RegUnit))
     351       72916 :       increaseSetPressure(LiveThruPressure, *MRI, RegUnit,
     352             :                           LaneBitmask::getNone(), Pair.LaneMask);
     353             :   }
     354      113668 : }
     355             : 
     356     2312502 : static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits,
     357             :                                unsigned RegUnit) {
     358             :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     359             :     return Other.RegUnit == RegUnit;
     360             :   });
     361     2312502 :   if (I == RegUnits.end())
     362     2254305 :     return LaneBitmask::getNone();
     363       58197 :   return I->LaneMask;
     364             : }
     365             : 
     366    13211993 : static void addRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits,
     367             :                         RegisterMaskPair Pair) {
     368    13211993 :   unsigned RegUnit = Pair.RegUnit;
     369             :   assert(Pair.LaneMask.any());
     370             :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     371             :     return Other.RegUnit == RegUnit;
     372             :   });
     373    13211993 :   if (I == RegUnits.end()) {
     374    13097586 :     RegUnits.push_back(Pair);
     375             :   } else {
     376             :     I->LaneMask |= Pair.LaneMask;
     377             :   }
     378    13211993 : }
     379             : 
     380      157467 : static void setRegZero(SmallVectorImpl<RegisterMaskPair> &RegUnits,
     381             :                        unsigned RegUnit) {
     382             :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     383             :     return Other.RegUnit == RegUnit;
     384             :   });
     385      157467 :   if (I == RegUnits.end()) {
     386      314934 :     RegUnits.push_back(RegisterMaskPair(RegUnit, LaneBitmask::getNone()));
     387             :   } else {
     388           0 :     I->LaneMask = LaneBitmask::getNone();
     389             :   }
     390      157467 : }
     391             : 
     392     5141114 : static void removeRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits,
     393             :                            RegisterMaskPair Pair) {
     394     5141114 :   unsigned RegUnit = Pair.RegUnit;
     395             :   assert(Pair.LaneMask.any());
     396             :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     397             :     return Other.RegUnit == RegUnit;
     398             :   });
     399     5141114 :   if (I != RegUnits.end()) {
     400             :     I->LaneMask &= ~Pair.LaneMask;
     401        7551 :     if (I->LaneMask.none())
     402             :       RegUnits.erase(I);
     403             :   }
     404     5141114 : }
     405             : 
     406     9907570 : static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS,
     407             :     const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit,
     408             :     SlotIndex Pos, LaneBitmask SafeDefault,
     409             :     bool(*Property)(const LiveRange &LR, SlotIndex Pos)) {
     410     9907570 :   if (TargetRegisterInfo::isVirtualRegister(RegUnit)) {
     411             :     const LiveInterval &LI = LIS.getInterval(RegUnit);
     412             :     LaneBitmask Result;
     413     8651055 :     if (TrackLaneMasks && LI.hasSubRanges()) {
     414    16994015 :         for (const LiveInterval::SubRange &SR : LI.subranges()) {
     415    13774775 :           if (Property(SR, Pos))
     416             :             Result |= SR.LaneMask;
     417             :         }
     418     5431815 :     } else if (Property(LI, Pos)) {
     419     3767953 :       Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit)
     420             :                               : LaneBitmask::getAll();
     421             :     }
     422             : 
     423     8651055 :     return Result;
     424             :   } else {
     425             :     const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
     426             :     // Be prepared for missing liveranges: We usually do not compute liveranges
     427             :     // for physical registers on targets with many registers (GPUs).
     428     1256515 :     if (LR == nullptr)
     429      249781 :       return SafeDefault;
     430     1006734 :     return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
     431             :   }
     432             : }
     433             : 
     434             : static LaneBitmask getLiveLanesAt(const LiveIntervals &LIS,
     435             :                                   const MachineRegisterInfo &MRI,
     436             :                                   bool TrackLaneMasks, unsigned RegUnit,
     437             :                                   SlotIndex Pos) {
     438             :   return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos,
     439             :                               LaneBitmask::getAll(),
     440    16886713 :                               [](const LiveRange &LR, SlotIndex Pos) {
     441             :                                 return LR.liveAt(Pos);
     442    23883750 :                               });
     443             : }
     444             : 
     445             : 
     446             : namespace {
     447             : 
     448             : /// Collect this instruction's unique uses and defs into SmallVectors for
     449             : /// processing defs and uses in order.
     450             : ///
     451             : /// FIXME: always ignore tied opers
     452             : class RegisterOperandsCollector {
     453             :   friend class llvm::RegisterOperands;
     454             : 
     455             :   RegisterOperands &RegOpers;
     456             :   const TargetRegisterInfo &TRI;
     457             :   const MachineRegisterInfo &MRI;
     458             :   bool IgnoreDead;
     459             : 
     460             :   RegisterOperandsCollector(RegisterOperands &RegOpers,
     461             :                             const TargetRegisterInfo &TRI,
     462             :                             const MachineRegisterInfo &MRI, bool IgnoreDead)
     463     6286302 :     : RegOpers(RegOpers), TRI(TRI), MRI(MRI), IgnoreDead(IgnoreDead) {}
     464             : 
     465     2576825 :   void collectInstr(const MachineInstr &MI) const {
     466    14478006 :     for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI)
     467    11901181 :       collectOperand(*OperI);
     468             : 
     469             :     // Remove redundant physreg dead defs.
     470     8884766 :     for (const RegisterMaskPair &P : RegOpers.Defs)
     471     1865558 :       removeRegLanes(RegOpers.DeadDefs, P);
     472     2576825 :   }
     473             : 
     474     3709477 :   void collectInstrLanes(const MachineInstr &MI) const {
     475    18821365 :     for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI)
     476    15111888 :       collectOperandLanes(*OperI);
     477             : 
     478             :     // Remove redundant physreg dead defs.
     479    13965712 :     for (const RegisterMaskPair &P : RegOpers.Defs)
     480     3273379 :       removeRegLanes(RegOpers.DeadDefs, P);
     481     3709477 :   }
     482             : 
     483             :   /// Push this operand's register onto the correct vectors.
     484    11901181 :   void collectOperand(const MachineOperand &MO) const {
     485    11901181 :     if (!MO.isReg() || !MO.getReg())
     486             :       return;
     487             :     unsigned Reg = MO.getReg();
     488     5917958 :     if (MO.isUse()) {
     489     7306909 :       if (!MO.isUndef() && !MO.isInternalRead())
     490     3647924 :         pushReg(Reg, RegOpers.Uses);
     491             :     } else {
     492             :       assert(MO.isDef());
     493             :       // Subregister definitions may imply a register read.
     494             :       if (MO.readsReg())
     495       62276 :         pushReg(Reg, RegOpers.Uses);
     496             : 
     497     2258973 :       if (MO.isDead()) {
     498      340695 :         if (!IgnoreDead)
     499      338797 :           pushReg(Reg, RegOpers.DeadDefs);
     500             :       } else
     501     1918278 :         pushReg(Reg, RegOpers.Defs);
     502             :     }
     503             :   }
     504             : 
     505     5967275 :   void pushReg(unsigned Reg,
     506             :                SmallVectorImpl<RegisterMaskPair> &RegUnits) const {
     507     5967275 :     if (TargetRegisterInfo::isVirtualRegister(Reg)) {
     508     4100423 :       addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneBitmask::getAll()));
     509     1866852 :     } else if (MRI.isAllocatable(Reg)) {
     510     1995206 :       for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
     511      949692 :         addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
     512             :     }
     513     5967275 :   }
     514             : 
     515    15111888 :   void collectOperandLanes(const MachineOperand &MO) const {
     516    15111888 :     if (!MO.isReg() || !MO.getReg())
     517             :       return;
     518             :     unsigned Reg = MO.getReg();
     519             :     unsigned SubRegIdx = MO.getSubReg();
     520    10033841 :     if (MO.isUse()) {
     521    12780025 :       if (!MO.isUndef() && !MO.isInternalRead())
     522     6387226 :         pushRegLanes(Reg, SubRegIdx, RegOpers.Uses);
     523             :     } else {
     524             :       assert(MO.isDef());
     525             :       // Treat read-undef subreg defs as definitions of the whole register.
     526     3641042 :       if (MO.isUndef())
     527             :         SubRegIdx = 0;
     528             : 
     529     3641042 :       if (MO.isDead()) {
     530      358317 :         if (!IgnoreDead)
     531       70786 :           pushRegLanes(Reg, SubRegIdx, RegOpers.DeadDefs);
     532             :       } else
     533     3282725 :         pushRegLanes(Reg, SubRegIdx, RegOpers.Defs);
     534             :     }
     535             :   }
     536             : 
     537     9740737 :   void pushRegLanes(unsigned Reg, unsigned SubRegIdx,
     538             :                     SmallVectorImpl<RegisterMaskPair> &RegUnits) const {
     539     9740737 :     if (TargetRegisterInfo::isVirtualRegister(Reg)) {
     540             :       LaneBitmask LaneMask = SubRegIdx != 0
     541     2453801 :                              ? TRI.getSubRegIndexLaneMask(SubRegIdx)
     542    13273996 :                              : MRI.getMaxLaneMaskForVReg(Reg);
     543     6636998 :       addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneMask));
     544     3103739 :     } else if (MRI.isAllocatable(Reg)) {
     545     1053536 :       for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
     546      420614 :         addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
     547             :     }
     548     9740737 :   }
     549             : };
     550             : 
     551             : } // end anonymous namespace
     552             : 
     553     6286302 : void RegisterOperands::collect(const MachineInstr &MI,
     554             :                                const TargetRegisterInfo &TRI,
     555             :                                const MachineRegisterInfo &MRI,
     556             :                                bool TrackLaneMasks, bool IgnoreDead) {
     557             :   RegisterOperandsCollector Collector(*this, TRI, MRI, IgnoreDead);
     558     6286302 :   if (TrackLaneMasks)
     559     3709477 :     Collector.collectInstrLanes(MI);
     560             :   else
     561     2576825 :     Collector.collectInstr(MI);
     562     6286302 : }
     563             : 
     564     1343393 : void RegisterOperands::detectDeadDefs(const MachineInstr &MI,
     565             :                                       const LiveIntervals &LIS) {
     566     2686786 :   SlotIndex SlotIdx = LIS.getInstructionIndex(MI);
     567     2235500 :   for (auto RI = Defs.begin(); RI != Defs.end(); /*empty*/) {
     568      892107 :     unsigned Reg = RI->RegUnit;
     569             :     const LiveRange *LR = getLiveRange(LIS, Reg);
     570      892107 :     if (LR != nullptr) {
     571      803652 :       LiveQueryResult LRQ = LR->Query(SlotIdx);
     572      803652 :       if (LRQ.isDeadDef()) {
     573             :         // LiveIntervals knows this is a dead even though it's MachineOperand is
     574             :         // not flagged as such.
     575           0 :         DeadDefs.push_back(*RI);
     576             :         RI = Defs.erase(RI);
     577           0 :         continue;
     578             :       }
     579             :     }
     580      892107 :     ++RI;
     581             :   }
     582     1343393 : }
     583             : 
     584     3709477 : void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS,
     585             :                                           const MachineRegisterInfo &MRI,
     586             :                                           SlotIndex Pos,
     587             :                                           MachineInstr *AddFlagsMI) {
     588     6982856 :   for (auto I = Defs.begin(); I != Defs.end(); ) {
     589             :     LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
     590     3273379 :                                            Pos.getDeadSlot());
     591             :     // If the def is all that is live after the instruction, then in case
     592             :     // of a subregister def we need a read-undef flag.
     593     3273379 :     unsigned RegUnit = I->RegUnit;
     594     3202164 :     if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
     595     3543149 :         AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask).none())
     596      194990 :       AddFlagsMI->setRegisterDefReadUndef(RegUnit);
     597             : 
     598             :     LaneBitmask ActualDef = I->LaneMask & LiveAfter;
     599     3273379 :     if (ActualDef.none()) {
     600             :       I = Defs.erase(I);
     601             :     } else {
     602     3273173 :       I->LaneMask = ActualDef;
     603     3273173 :       ++I;
     604             :     }
     605             :   }
     606     7428072 :   for (auto I = Uses.begin(); I != Uses.end(); ) {
     607             :     LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
     608     3718595 :                                             Pos.getBaseIndex());
     609             :     LaneBitmask LaneMask = I->LaneMask & LiveBefore;
     610     3718595 :     if (LaneMask.none()) {
     611             :       I = Uses.erase(I);
     612             :     } else {
     613     3718595 :       I->LaneMask = LaneMask;
     614     3718595 :       ++I;
     615             :     }
     616             :   }
     617     3709477 :   if (AddFlagsMI != nullptr) {
     618      338388 :     for (const RegisterMaskPair &P : DeadDefs) {
     619       13526 :       unsigned RegUnit = P.RegUnit;
     620       13526 :       if (!TargetRegisterInfo::isVirtualRegister(RegUnit))
     621             :         continue;
     622             :       LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit,
     623             :                                              Pos.getDeadSlot());
     624        5063 :       if (LiveAfter.none())
     625        5063 :         AddFlagsMI->setRegisterDefReadUndef(RegUnit);
     626             :     }
     627             :   }
     628     3709477 : }
     629             : 
     630             : /// Initialize an array of N PressureDiffs.
     631      113668 : void PressureDiffs::init(unsigned N) {
     632      113668 :   Size = N;
     633      113668 :   if (N <= Max) {
     634       50657 :     memset(PDiffArray, 0, N * sizeof(PressureDiff));
     635       50657 :     return;
     636             :   }
     637       63011 :   Max = Size;
     638       63011 :   free(PDiffArray);
     639       63011 :   PDiffArray = static_cast<PressureDiff*>(safe_calloc(N, sizeof(PressureDiff)));
     640             : }
     641             : 
     642     1423163 : void PressureDiffs::addInstruction(unsigned Idx,
     643             :                                    const RegisterOperands &RegOpers,
     644             :                                    const MachineRegisterInfo &MRI) {
     645     1423163 :   PressureDiff &PDiff = (*this)[Idx];
     646             :   assert(!PDiff.begin()->isValid() && "stale PDiff");
     647     3698829 :   for (const RegisterMaskPair &P : RegOpers.Defs)
     648     1137833 :     PDiff.addPressureChange(P.RegUnit, true, &MRI);
     649             : 
     650     4802459 :   for (const RegisterMaskPair &P : RegOpers.Uses)
     651     1689648 :     PDiff.addPressureChange(P.RegUnit, false, &MRI);
     652     1423163 : }
     653             : 
     654             : /// Add a change in pressure to the pressure diff of a given instruction.
     655     4289196 : void PressureDiff::addPressureChange(unsigned RegUnit, bool IsDec,
     656             :                                      const MachineRegisterInfo *MRI) {
     657             :   PSetIterator PSetI = MRI->getPressureSets(RegUnit);
     658     4289196 :   int Weight = IsDec ? -PSetI.getWeight() : PSetI.getWeight();
     659    14419351 :   for (; PSetI.isValid(); ++PSetI) {
     660             :     // Find an existing entry in the pressure diff for this PSet.
     661             :     PressureDiff::iterator I = nonconst_begin(), E = nonconst_end();
     662    58936931 :     for (; I != E && I->isValid(); ++I) {
     663    29550708 :       if (I->getPSet() >= *PSetI)
     664             :         break;
     665             :     }
     666             :     // If all pressure sets are more constrained, skip the remaining PSets.
     667    10272087 :     if (I == E)
     668             :       break;
     669             :     // Insert this PressureChange.
     670    15348441 :     if (!I->isValid() || I->getPSet() != *PSetI) {
     671             :       PressureChange PTmp = PressureChange(*PSetI);
     672    22233298 :       for (PressureDiff::iterator J = I; J != E && PTmp.isValid(); ++J)
     673             :         std::swap(*J, PTmp);
     674             :     }
     675             :     // Update the units for this pressure set.
     676    20260310 :     unsigned NewUnitInc = I->getUnitInc() + Weight;
     677    10130155 :     if (NewUnitInc != 0) {
     678             :       I->setUnitInc(NewUnitInc);
     679             :     } else {
     680             :       // Remove entry
     681             :       PressureDiff::iterator J;
     682    12236898 :       for (J = std::next(I); J != E && J->isValid(); ++J, ++I)
     683     4583336 :         *I = *J;
     684     3070226 :       if (J != E)
     685     3032465 :         *I = *J;
     686             :     }
     687             :   }
     688     4289196 : }
     689             : 
     690             : /// Force liveness of registers.
     691      227411 : void RegPressureTracker::addLiveRegs(ArrayRef<RegisterMaskPair> Regs) {
     692     1125365 :   for (const RegisterMaskPair &P : Regs) {
     693      448977 :     LaneBitmask PrevMask = LiveRegs.insert(P);
     694      448977 :     LaneBitmask NewMask = PrevMask | P.LaneMask;
     695      448977 :     increaseRegPressure(P.RegUnit, PrevMask, NewMask);
     696             :   }
     697      227411 : }
     698             : 
     699      168519 : void RegPressureTracker::discoverLiveInOrOut(RegisterMaskPair Pair,
     700             :     SmallVectorImpl<RegisterMaskPair> &LiveInOrOut) {
     701             :   assert(Pair.LaneMask.any());
     702             : 
     703      168519 :   unsigned RegUnit = Pair.RegUnit;
     704             :   auto I = llvm::find_if(LiveInOrOut, [RegUnit](const RegisterMaskPair &Other) {
     705             :     return Other.RegUnit == RegUnit;
     706             :   });
     707             :   LaneBitmask PrevMask;
     708             :   LaneBitmask NewMask;
     709      168519 :   if (I == LiveInOrOut.end()) {
     710             :     PrevMask = LaneBitmask::getNone();
     711      166036 :     NewMask = Pair.LaneMask;
     712      166036 :     LiveInOrOut.push_back(Pair);
     713             :   } else {
     714        2483 :     PrevMask = I->LaneMask;
     715             :     NewMask = PrevMask | Pair.LaneMask;
     716        2483 :     I->LaneMask = NewMask;
     717             :   }
     718      168519 :   increaseSetPressure(P.MaxSetPressure, *MRI, RegUnit, PrevMask, NewMask);
     719      168519 : }
     720             : 
     721          15 : void RegPressureTracker::discoverLiveIn(RegisterMaskPair Pair) {
     722          15 :   discoverLiveInOrOut(Pair, P.LiveInRegs);
     723          15 : }
     724             : 
     725      168504 : void RegPressureTracker::discoverLiveOut(RegisterMaskPair Pair) {
     726      168504 :   discoverLiveInOrOut(Pair, P.LiveOutRegs);
     727      168504 : }
     728             : 
     729     6283010 : void RegPressureTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
     730     6403674 :   for (const RegisterMaskPair &P : DeadDefs) {
     731       60332 :     unsigned Reg = P.RegUnit;
     732       60332 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
     733       60332 :     LaneBitmask BumpedMask = LiveMask | P.LaneMask;
     734       60332 :     increaseRegPressure(Reg, LiveMask, BumpedMask);
     735             :   }
     736     6403674 :   for (const RegisterMaskPair &P : DeadDefs) {
     737       60332 :     unsigned Reg = P.RegUnit;
     738       60332 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
     739       60332 :     LaneBitmask BumpedMask = LiveMask | P.LaneMask;
     740       60332 :     decreaseRegPressure(Reg, BumpedMask, LiveMask);
     741             :   }
     742     6283010 : }
     743             : 
     744             : /// Recede across the previous instruction. If LiveUses is provided, record any
     745             : /// RegUnits that are made live by the current instruction's uses. This includes
     746             : /// registers that are both defined and used by the instruction.  If a pressure
     747             : /// difference pointer is provided record the changes is pressure caused by this
     748             : /// instruction independent of liveness.
     749     2990182 : void RegPressureTracker::recede(const RegisterOperands &RegOpers,
     750             :                                 SmallVectorImpl<RegisterMaskPair> *LiveUses) {
     751             :   assert(!CurrPos->isDebugInstr());
     752             : 
     753             :   // Boost pressure for all dead defs together.
     754     2990182 :   bumpDeadDefs(RegOpers.DeadDefs);
     755             : 
     756             :   // Kill liveness at live defs.
     757             :   // TODO: consider earlyclobbers?
     758     7408172 :   for (const RegisterMaskPair &Def : RegOpers.Defs) {
     759     2208995 :     unsigned Reg = Def.RegUnit;
     760             : 
     761     2208995 :     LaneBitmask PreviousMask = LiveRegs.erase(Def);
     762             :     LaneBitmask NewMask = PreviousMask & ~Def.LaneMask;
     763             : 
     764             :     LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask;
     765     2208995 :     if (LiveOut.any()) {
     766       65891 :       discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
     767             :       // Retroactively model effects on pressure of the live out lanes.
     768       65891 :       increaseSetPressure(CurrSetPressure, *MRI, Reg, LaneBitmask::getNone(),
     769             :                           LiveOut);
     770             :       PreviousMask = LiveOut;
     771             :     }
     772             : 
     773     2208995 :     if (NewMask.none()) {
     774             :       // Add a 0 entry to LiveUses as a marker that the complete vreg has become
     775             :       // dead.
     776     2074173 :       if (TrackLaneMasks && LiveUses != nullptr)
     777      157467 :         setRegZero(*LiveUses, Reg);
     778             :     }
     779             : 
     780     2208995 :     decreaseRegPressure(Reg, PreviousMask, NewMask);
     781             :   }
     782             : 
     783     2990182 :   SlotIndex SlotIdx;
     784     2990182 :   if (RequireIntervals)
     785     8970546 :     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
     786             : 
     787             :   // Generate liveness for uses.
     788    10136388 :   for (const RegisterMaskPair &Use : RegOpers.Uses) {
     789     3573103 :     unsigned Reg = Use.RegUnit;
     790             :     assert(Use.LaneMask.any());
     791     3573103 :     LaneBitmask PreviousMask = LiveRegs.insert(Use);
     792             :     LaneBitmask NewMask = PreviousMask | Use.LaneMask;
     793     3573103 :     if (NewMask == PreviousMask)
     794             :       continue;
     795             : 
     796             :     // Did the register just become live?
     797     2468975 :     if (PreviousMask.none()) {
     798     2393742 :       if (LiveUses != nullptr) {
     799     1106443 :         if (!TrackLaneMasks) {
     800      922475 :           addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
     801             :         } else {
     802             :           auto I =
     803             :               llvm::find_if(*LiveUses, [Reg](const RegisterMaskPair Other) {
     804             :                 return Other.RegUnit == Reg;
     805             :               });
     806             :           bool IsRedef = I != LiveUses->end();
     807      183968 :           if (IsRedef) {
     808             :             // ignore re-defs here...
     809             :             assert(I->LaneMask.none());
     810        2177 :             removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
     811             :           } else {
     812      181791 :             addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
     813             :           }
     814             :         }
     815             :       }
     816             : 
     817             :       // Discover live outs if this may be the first occurance of this register.
     818     2393742 :       if (RequireIntervals) {
     819     2393742 :         LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx);
     820     2393742 :         if (LiveOut.any())
     821      102613 :           discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
     822             :       }
     823             :     }
     824             : 
     825     2468975 :     increaseRegPressure(Reg, PreviousMask, NewMask);
     826             :   }
     827     2990182 :   if (TrackUntiedDefs) {
     828     3813515 :     for (const RegisterMaskPair &Def : RegOpers.Defs) {
     829     1141172 :       unsigned RegUnit = Def.RegUnit;
     830     2107516 :       if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
     831     2107516 :           (LiveRegs.contains(RegUnit) & Def.LaneMask).none())
     832      777553 :         UntiedDefs.insert(RegUnit);
     833             :     }
     834             :   }
     835     2990182 : }
     836             : 
     837     2845851 : void RegPressureTracker::recedeSkipDebugValues() {
     838             :   assert(CurrPos != MBB->begin());
     839     2845851 :   if (!isBottomClosed())
     840      113668 :     closeBottom();
     841             : 
     842             :   // Open the top of the region using block iterators.
     843     2845851 :   if (!RequireIntervals && isTopClosed())
     844           0 :     static_cast<RegionPressure&>(P).openTop(CurrPos);
     845             : 
     846             :   // Find the previous instruction.
     847     5691702 :   CurrPos = skipDebugInstructionsBackward(std::prev(CurrPos), MBB->begin());
     848             : 
     849     2845851 :   SlotIndex SlotIdx;
     850     2845851 :   if (RequireIntervals)
     851     8537553 :     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
     852             : 
     853             :   // Open the top of the region using slot indexes.
     854     2845851 :   if (RequireIntervals && isTopClosed())
     855           0 :     static_cast<IntervalPressure&>(P).openTop(SlotIdx);
     856     2845851 : }
     857             : 
     858      215587 : void RegPressureTracker::recede(SmallVectorImpl<RegisterMaskPair> *LiveUses) {
     859      215587 :   recedeSkipDebugValues();
     860             : 
     861             :   const MachineInstr &MI = *CurrPos;
     862      215587 :   RegisterOperands RegOpers;
     863      215587 :   RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
     864      215587 :   if (TrackLaneMasks) {
     865      112236 :     SlotIndex SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
     866       37412 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
     867      178175 :   } else if (RequireIntervals) {
     868      178175 :     RegOpers.detectDeadDefs(MI, *LIS);
     869             :   }
     870             : 
     871      215587 :   recede(RegOpers, LiveUses);
     872      215587 : }
     873             : 
     874             : /// Advance across the current instruction.
     875       71763 : void RegPressureTracker::advance(const RegisterOperands &RegOpers) {
     876             :   assert(!TrackUntiedDefs && "unsupported mode");
     877             :   assert(CurrPos != MBB->end());
     878       71763 :   if (!isTopClosed())
     879           5 :     closeTop();
     880             : 
     881       71763 :   SlotIndex SlotIdx;
     882       71763 :   if (RequireIntervals)
     883       71763 :     SlotIdx = getCurrSlot();
     884             : 
     885             :   // Open the bottom of the region using slot indexes.
     886       71763 :   if (isBottomClosed()) {
     887           0 :     if (RequireIntervals)
     888           0 :       static_cast<IntervalPressure&>(P).openBottom(SlotIdx);
     889             :     else
     890           0 :       static_cast<RegionPressure&>(P).openBottom(CurrPos);
     891             :   }
     892             : 
     893      256669 :   for (const RegisterMaskPair &Use : RegOpers.Uses) {
     894       92453 :     unsigned Reg = Use.RegUnit;
     895       92453 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
     896             :     LaneBitmask LiveIn = Use.LaneMask & ~LiveMask;
     897       92453 :     if (LiveIn.any()) {
     898          15 :       discoverLiveIn(RegisterMaskPair(Reg, LiveIn));
     899          15 :       increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn);
     900          15 :       LiveRegs.insert(RegisterMaskPair(Reg, LiveIn));
     901             :     }
     902             :     // Kill liveness at last uses.
     903       92453 :     if (RequireIntervals) {
     904       92453 :       LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
     905       92453 :       if (LastUseMask.any()) {
     906       82652 :         LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask));
     907       82652 :         decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask);
     908             :       }
     909             :     }
     910             :   }
     911             : 
     912             :   // Generate liveness for defs.
     913      217661 :   for (const RegisterMaskPair &Def : RegOpers.Defs) {
     914       72949 :     LaneBitmask PreviousMask = LiveRegs.insert(Def);
     915       72949 :     LaneBitmask NewMask = PreviousMask | Def.LaneMask;
     916       72949 :     increaseRegPressure(Def.RegUnit, PreviousMask, NewMask);
     917             :   }
     918             : 
     919             :   // Boost pressure for all dead defs together.
     920       71763 :   bumpDeadDefs(RegOpers.DeadDefs);
     921             : 
     922             :   // Find the next instruction.
     923      143526 :   CurrPos = skipDebugInstructionsForward(std::next(CurrPos), MBB->end());
     924       71763 : }
     925             : 
     926          32 : void RegPressureTracker::advance() {
     927             :   const MachineInstr &MI = *CurrPos;
     928          32 :   RegisterOperands RegOpers;
     929          32 :   RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
     930          32 :   if (TrackLaneMasks) {
     931           0 :     SlotIndex SlotIdx = getCurrSlot();
     932           0 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
     933             :   }
     934          32 :   advance(RegOpers);
     935          32 : }
     936             : 
     937             : /// Find the max change in excess pressure across all sets.
     938      168355 : static void computeExcessPressureDelta(ArrayRef<unsigned> OldPressureVec,
     939             :                                        ArrayRef<unsigned> NewPressureVec,
     940             :                                        RegPressureDelta &Delta,
     941             :                                        const RegisterClassInfo *RCI,
     942             :                                        ArrayRef<unsigned> LiveThruPressureVec) {
     943      168355 :   Delta.Excess = PressureChange();
     944     2334863 :   for (unsigned i = 0, e = OldPressureVec.size(); i < e; ++i) {
     945     4333454 :     unsigned POld = OldPressureVec[i];
     946     4333454 :     unsigned PNew = NewPressureVec[i];
     947     2166727 :     int PDiff = (int)PNew - (int)POld;
     948     2166727 :     if (!PDiff) // No change in this set in the common case.
     949             :       continue;
     950             :     // Only consider change beyond the limit.
     951      376043 :     unsigned Limit = RCI->getRegPressureSetLimit(i);
     952      376043 :     if (!LiveThruPressureVec.empty())
     953      751132 :       Limit += LiveThruPressureVec[i];
     954             : 
     955      376043 :     if (Limit > POld) {
     956      374185 :       if (Limit > PNew)
     957             :         PDiff = 0;            // Under the limit
     958             :       else
     959         282 :         PDiff = PNew - Limit; // Just exceeded limit.
     960        1858 :     } else if (Limit > PNew)
     961        1641 :       PDiff = Limit - POld;   // Just obeyed limit.
     962             : 
     963        2140 :     if (PDiff) {
     964         219 :       Delta.Excess = PressureChange(i);
     965             :       Delta.Excess.setUnitInc(PDiff);
     966             :       break;
     967             :     }
     968             :   }
     969      168355 : }
     970             : 
     971             : /// Find the max change in max pressure that either surpasses a critical PSet
     972             : /// limit or exceeds the current MaxPressureLimit.
     973             : ///
     974             : /// FIXME: comparing each element of the old and new MaxPressure vectors here is
     975             : /// silly. It's done now to demonstrate the concept but will go away with a
     976             : /// RegPressureTracker API change to work with pressure differences.
     977      168355 : static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec,
     978             :                                     ArrayRef<unsigned> NewMaxPressureVec,
     979             :                                     ArrayRef<PressureChange> CriticalPSets,
     980             :                                     ArrayRef<unsigned> MaxPressureLimit,
     981             :                                     RegPressureDelta &Delta) {
     982      168355 :   Delta.CriticalMax = PressureChange();
     983      168355 :   Delta.CurrentMax = PressureChange();
     984             : 
     985      168355 :   unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
     986     2299310 :   for (unsigned i = 0, e = OldMaxPressureVec.size(); i < e; ++i) {
     987     4284704 :     unsigned POld = OldMaxPressureVec[i];
     988     4284704 :     unsigned PNew = NewMaxPressureVec[i];
     989     2142352 :     if (PNew == POld) // No change in this set in the common case.
     990             :       continue;
     991             : 
     992      181052 :     if (!Delta.CriticalMax.isValid()) {
     993      211164 :       while (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() < i)
     994         325 :         ++CritIdx;
     995             : 
     996      210189 :       if (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() == i) {
     997       29936 :         int PDiff = (int)PNew - (int)CriticalPSets[CritIdx].getUnitInc();
     998       14968 :         if (PDiff > 0) {
     999       14215 :           Delta.CriticalMax = PressureChange(i);
    1000             :           Delta.CriticalMax.setUnitInc(PDiff);
    1001             :         }
    1002             :       }
    1003             :     }
    1004             :     // Find the first increase above MaxPressureLimit.
    1005             :     // (Ignores negative MDiff).
    1006      362104 :     if (!Delta.CurrentMax.isValid() && PNew > MaxPressureLimit[i]) {
    1007       11397 :       Delta.CurrentMax = PressureChange(i);
    1008       11397 :       Delta.CurrentMax.setUnitInc(PNew - POld);
    1009       11397 :       if (CritIdx == CritEnd || Delta.CriticalMax.isValid())
    1010             :         break;
    1011             :     }
    1012             :   }
    1013      168355 : }
    1014             : 
    1015             : /// Record the upward impact of a single instruction on current register
    1016             : /// pressure. Unlike the advance/recede pressure tracking interface, this does
    1017             : /// not discover live in/outs.
    1018             : ///
    1019             : /// This is intended for speculative queries. It leaves pressure inconsistent
    1020             : /// with the current position, so must be restored by the caller.
    1021     2668771 : void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
    1022             :   assert(!MI->isDebugInstr() && "Expect a nondebug instruction.");
    1023             : 
    1024     2668771 :   SlotIndex SlotIdx;
    1025     2668771 :   if (RequireIntervals)
    1026     5337542 :     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
    1027             : 
    1028             :   // Account for register pressure similar to RegPressureTracker::recede().
    1029     2668771 :   RegisterOperands RegOpers;
    1030     2668771 :   RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true);
    1031             :   assert(RegOpers.DeadDefs.size() == 0);
    1032     2668771 :   if (TrackLaneMasks)
    1033     2618672 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
    1034       50099 :   else if (RequireIntervals)
    1035       50099 :     RegOpers.detectDeadDefs(*MI, *LIS);
    1036             : 
    1037             :   // Boost max pressure for all dead defs together.
    1038             :   // Since CurrSetPressure and MaxSetPressure
    1039     2668771 :   bumpDeadDefs(RegOpers.DeadDefs);
    1040             : 
    1041             :   // Kill liveness at live defs.
    1042     7293775 :   for (const RegisterMaskPair &P : RegOpers.Defs) {
    1043     2312502 :     unsigned Reg = P.RegUnit;
    1044     2312502 :     LaneBitmask LiveLanes = LiveRegs.contains(Reg);
    1045     2312502 :     LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg);
    1046     2312502 :     LaneBitmask DefLanes = P.LaneMask;
    1047     2312502 :     LaneBitmask LiveAfter = (LiveLanes & ~DefLanes) | UseLanes;
    1048     2312502 :     decreaseRegPressure(Reg, LiveLanes, LiveAfter);
    1049             :   }
    1050             :   // Generate liveness for uses.
    1051     8057491 :   for (const RegisterMaskPair &P : RegOpers.Uses) {
    1052     2694360 :     unsigned Reg = P.RegUnit;
    1053     2694360 :     LaneBitmask LiveLanes = LiveRegs.contains(Reg);
    1054     2694360 :     LaneBitmask LiveAfter = LiveLanes | P.LaneMask;
    1055     2694360 :     increaseRegPressure(Reg, LiveLanes, LiveAfter);
    1056             :   }
    1057     2668771 : }
    1058             : 
    1059             : /// Consider the pressure increase caused by traversing this instruction
    1060             : /// bottom-up. Find the pressure set with the most change beyond its pressure
    1061             : /// limit based on the tracker's current pressure, and return the change in
    1062             : /// number of register units of that pressure set introduced by this
    1063             : /// instruction.
    1064             : ///
    1065             : /// This assumes that the current LiveOut set is sufficient.
    1066             : ///
    1067             : /// This is expensive for an on-the-fly query because it calls
    1068             : /// bumpUpwardPressure to recompute the pressure sets based on current
    1069             : /// liveness. This mainly exists to verify correctness, e.g. with
    1070             : /// -verify-misched. getUpwardPressureDelta is the fast version of this query
    1071             : /// that uses the per-SUnit cache of the PressureDiff.
    1072       50099 : void RegPressureTracker::
    1073             : getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
    1074             :                           RegPressureDelta &Delta,
    1075             :                           ArrayRef<PressureChange> CriticalPSets,
    1076             :                           ArrayRef<unsigned> MaxPressureLimit) {
    1077             :   // Snapshot Pressure.
    1078             :   // FIXME: The snapshot heap space should persist. But I'm planning to
    1079             :   // summarize the pressure effect so we don't need to snapshot at all.
    1080       50099 :   std::vector<unsigned> SavedPressure = CurrSetPressure;
    1081       50099 :   std::vector<unsigned> SavedMaxPressure = P.MaxSetPressure;
    1082             : 
    1083       50099 :   bumpUpwardPressure(MI);
    1084             : 
    1085      100198 :   computeExcessPressureDelta(SavedPressure, CurrSetPressure, Delta, RCI,
    1086             :                              LiveThruPressure);
    1087      100198 :   computeMaxPressureDelta(SavedMaxPressure, P.MaxSetPressure, CriticalPSets,
    1088             :                           MaxPressureLimit, Delta);
    1089             :   assert(Delta.CriticalMax.getUnitInc() >= 0 &&
    1090             :          Delta.CurrentMax.getUnitInc() >= 0 && "cannot decrease max pressure");
    1091             : 
    1092             :   // Restore the tracker's state.
    1093       50099 :   P.MaxSetPressure.swap(SavedMaxPressure);
    1094             :   CurrSetPressure.swap(SavedPressure);
    1095             : 
    1096             : #ifndef NDEBUG
    1097             :   if (!PDiff)
    1098             :     return;
    1099             : 
    1100             :   // Check if the alternate algorithm yields the same result.
    1101             :   RegPressureDelta Delta2;
    1102             :   getUpwardPressureDelta(MI, *PDiff, Delta2, CriticalPSets, MaxPressureLimit);
    1103             :   if (Delta != Delta2) {
    1104             :     dbgs() << "PDiff: ";
    1105             :     PDiff->dump(*TRI);
    1106             :     dbgs() << "DELTA: " << *MI;
    1107             :     if (Delta.Excess.isValid())
    1108             :       dbgs() << "Excess1 " << TRI->getRegPressureSetName(Delta.Excess.getPSet())
    1109             :              << " " << Delta.Excess.getUnitInc() << "\n";
    1110             :     if (Delta.CriticalMax.isValid())
    1111             :       dbgs() << "Critic1 " << TRI->getRegPressureSetName(Delta.CriticalMax.getPSet())
    1112             :              << " " << Delta.CriticalMax.getUnitInc() << "\n";
    1113             :     if (Delta.CurrentMax.isValid())
    1114             :       dbgs() << "CurrMx1 " << TRI->getRegPressureSetName(Delta.CurrentMax.getPSet())
    1115             :              << " " << Delta.CurrentMax.getUnitInc() << "\n";
    1116             :     if (Delta2.Excess.isValid())
    1117             :       dbgs() << "Excess2 " << TRI->getRegPressureSetName(Delta2.Excess.getPSet())
    1118             :              << " " << Delta2.Excess.getUnitInc() << "\n";
    1119             :     if (Delta2.CriticalMax.isValid())
    1120             :       dbgs() << "Critic2 " << TRI->getRegPressureSetName(Delta2.CriticalMax.getPSet())
    1121             :              << " " << Delta2.CriticalMax.getUnitInc() << "\n";
    1122             :     if (Delta2.CurrentMax.isValid())
    1123             :       dbgs() << "CurrMx2 " << TRI->getRegPressureSetName(Delta2.CurrentMax.getPSet())
    1124             :              << " " << Delta2.CurrentMax.getUnitInc() << "\n";
    1125             :     llvm_unreachable("RegP Delta Mismatch");
    1126             :   }
    1127             : #endif
    1128       50099 : }
    1129             : 
    1130             : /// This is the fast version of querying register pressure that does not
    1131             : /// directly depend on current liveness.
    1132             : ///
    1133             : /// @param Delta captures information needed for heuristics.
    1134             : ///
    1135             : /// @param CriticalPSets Are the pressure sets that are known to exceed some
    1136             : /// limit within the region, not necessarily at the current position.
    1137             : ///
    1138             : /// @param MaxPressureLimit Is the max pressure within the region, not
    1139             : /// necessarily at the current position.
    1140     9033771 : void RegPressureTracker::
    1141             : getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff,
    1142             :                        RegPressureDelta &Delta,
    1143             :                        ArrayRef<PressureChange> CriticalPSets,
    1144             :                        ArrayRef<unsigned> MaxPressureLimit) const {
    1145     9033771 :   unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
    1146     7029092 :   for (PressureDiff::const_iterator
    1147             :          PDiffI = PDiff.begin(), PDiffE = PDiff.end();
    1148    16062863 :        PDiffI != PDiffE && PDiffI->isValid(); ++PDiffI) {
    1149             : 
    1150             :     unsigned PSetID = PDiffI->getPSet();
    1151     7029092 :     unsigned Limit = RCI->getRegPressureSetLimit(PSetID);
    1152     7029092 :     if (!LiveThruPressure.empty())
    1153    14058184 :       Limit += LiveThruPressure[PSetID];
    1154             : 
    1155    14058184 :     unsigned POld = CurrSetPressure[PSetID];
    1156    14058184 :     unsigned MOld = P.MaxSetPressure[PSetID];
    1157             :     unsigned MNew = MOld;
    1158             :     // Ignore DeadDefs here because they aren't captured by PressureChange.
    1159     7029092 :     unsigned PNew = POld + PDiffI->getUnitInc();
    1160             :     assert((PDiffI->getUnitInc() >= 0) == (PNew >= POld)
    1161             :            && "PSet overflow/underflow");
    1162     7029092 :     if (PNew > MOld)
    1163             :       MNew = PNew;
    1164             :     // Check if current pressure has exceeded the limit.
    1165     7029092 :     if (!Delta.Excess.isValid()) {
    1166             :       unsigned ExcessInc = 0;
    1167     6875010 :       if (PNew > Limit)
    1168      249087 :         ExcessInc = POld > Limit ? PNew - POld : PNew - Limit;
    1169     6625923 :       else if (POld > Limit)
    1170        5344 :         ExcessInc = Limit - POld;
    1171      254431 :       if (ExcessInc) {
    1172      254431 :         Delta.Excess = PressureChange(PSetID);
    1173             :         Delta.Excess.setUnitInc(ExcessInc);
    1174             :       }
    1175             :     }
    1176             :     // Check if max pressure has exceeded a critical pressure set max.
    1177     7029092 :     if (MNew == MOld)
    1178     3946372 :       continue;
    1179     3082720 :     if (!Delta.CriticalMax.isValid()) {
    1180     3714843 :       while (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() < PSetID)
    1181        8800 :         ++CritIdx;
    1182             : 
    1183     4117915 :       if (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() == PSetID) {
    1184      575308 :         int CritInc = (int)MNew - (int)CriticalPSets[CritIdx].getUnitInc();
    1185      287654 :         if (CritInc > 0 && CritInc <= std::numeric_limits<int16_t>::max()) {
    1186      287654 :           Delta.CriticalMax = PressureChange(PSetID);
    1187             :           Delta.CriticalMax.setUnitInc(CritInc);
    1188             :         }
    1189             :       }
    1190             :     }
    1191             :     // Check if max pressure has exceeded the current max.
    1192     6005000 :     if (!Delta.CurrentMax.isValid() && MNew > MaxPressureLimit[PSetID]) {
    1193      162426 :       Delta.CurrentMax = PressureChange(PSetID);
    1194      162426 :       Delta.CurrentMax.setUnitInc(MNew - MOld);
    1195             :     }
    1196             :   }
    1197     9033771 : }
    1198             : 
    1199             : /// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx).
    1200             : /// The query starts with a lane bitmask which gets lanes/bits removed for every
    1201             : /// use we find.
    1202      314027 : static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
    1203             :                                   SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
    1204             :                                   const MachineRegisterInfo &MRI,
    1205             :                                   const LiveIntervals *LIS) {
    1206      314027 :   const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
    1207      710070 :   for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
    1208      421667 :     if (MO.isUndef())
    1209             :       continue;
    1210      421639 :     const MachineInstr *MI = MO.getParent();
    1211      421639 :     SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
    1212      774122 :     if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) {
    1213             :       unsigned SubRegIdx = MO.getSubReg();
    1214       90813 :       LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
    1215             :       LastUseMask &= ~UseMask;
    1216       90813 :       if (LastUseMask.none())
    1217       25624 :         return LaneBitmask::getNone();
    1218             :     }
    1219             :   }
    1220      288403 :   return LastUseMask;
    1221             : }
    1222             : 
    1223           0 : LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit,
    1224             :                                                SlotIndex Pos) const {
    1225             :   assert(RequireIntervals);
    1226           0 :   return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
    1227             :                               LaneBitmask::getAll(),
    1228           0 :       [](const LiveRange &LR, SlotIndex Pos) {
    1229             :         return LR.liveAt(Pos);
    1230           0 :       });
    1231             : }
    1232             : 
    1233      516791 : LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit,
    1234             :                                                  SlotIndex Pos) const {
    1235             :   assert(RequireIntervals);
    1236     1033582 :   return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit,
    1237             :                               Pos.getBaseIndex(), LaneBitmask::getNone(),
    1238     1778684 :       [](const LiveRange &LR, SlotIndex Pos) {
    1239             :         const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
    1240     1454396 :         return S != nullptr && S->end == Pos.getRegSlot();
    1241     2439715 :       });
    1242             : }
    1243             : 
    1244     2393742 : LaneBitmask RegPressureTracker::getLiveThroughAt(unsigned RegUnit,
    1245             :                                                  SlotIndex Pos) const {
    1246             :   assert(RequireIntervals);
    1247     4787484 :   return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
    1248             :                               LaneBitmask::getNone(),
    1249     4874538 :       [](const LiveRange &LR, SlotIndex Pos) {
    1250             :         const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
    1251     1089325 :         return S != nullptr && S->start < Pos.getRegSlot(true) &&
    1252     2437269 :                S->end != Pos.getDeadSlot();
    1253     9618495 :       });
    1254             : }
    1255             : 
    1256             : /// Record the downward impact of a single instruction on current register
    1257             : /// pressure. Unlike the advance/recede pressure tracking interface, this does
    1258             : /// not discover live in/outs.
    1259             : ///
    1260             : /// This is intended for speculative queries. It leaves pressure inconsistent
    1261             : /// with the current position, so must be restored by the caller.
    1262      552294 : void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
    1263             :   assert(!MI->isDebugInstr() && "Expect a nondebug instruction.");
    1264             : 
    1265      552294 :   SlotIndex SlotIdx;
    1266      552294 :   if (RequireIntervals)
    1267     1104588 :     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
    1268             : 
    1269             :   // Account for register pressure similar to RegPressureTracker::recede().
    1270      552294 :   RegisterOperands RegOpers;
    1271      552294 :   RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, false);
    1272      552294 :   if (TrackLaneMasks)
    1273      434013 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
    1274             : 
    1275      552294 :   if (RequireIntervals) {
    1276     1400970 :     for (const RegisterMaskPair &Use : RegOpers.Uses) {
    1277      424338 :       unsigned Reg = Use.RegUnit;
    1278      424338 :       LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
    1279      424338 :       if (LastUseMask.none())
    1280      246246 :         continue;
    1281             :       // The LastUseMask is queried from the liveness information of instruction
    1282             :       // which may be further down the schedule. Some lanes may actually not be
    1283             :       // last uses for the current position.
    1284             :       // FIXME: allow the caller to pass in the list of vreg uses that remain
    1285             :       // to be bottom-scheduled to avoid searching uses at each query.
    1286      314027 :       SlotIndex CurrIdx = getCurrSlot();
    1287             :       LastUseMask
    1288      314027 :         = findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
    1289      314027 :       if (LastUseMask.none())
    1290       25624 :         continue;
    1291             : 
    1292      288403 :       LaneBitmask LiveMask = LiveRegs.contains(Reg);
    1293      288403 :       LaneBitmask NewMask = LiveMask & ~LastUseMask;
    1294      288403 :       decreaseRegPressure(Reg, LiveMask, NewMask);
    1295             :     }
    1296             :   }
    1297             : 
    1298             :   // Generate liveness for defs.
    1299     1634836 :   for (const RegisterMaskPair &Def : RegOpers.Defs) {
    1300      541271 :     unsigned Reg = Def.RegUnit;
    1301      541271 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
    1302      541271 :     LaneBitmask NewMask = LiveMask | Def.LaneMask;
    1303      541271 :     increaseRegPressure(Reg, LiveMask, NewMask);
    1304             :   }
    1305             : 
    1306             :   // Boost pressure for all dead defs together.
    1307      552294 :   bumpDeadDefs(RegOpers.DeadDefs);
    1308      552294 : }
    1309             : 
    1310             : /// Consider the pressure increase caused by traversing this instruction
    1311             : /// top-down. Find the register class with the most change in its pressure limit
    1312             : /// based on the tracker's current pressure, and return the number of excess
    1313             : /// register units of that pressure set introduced by this instruction.
    1314             : ///
    1315             : /// This assumes that the current LiveIn set is sufficient.
    1316             : ///
    1317             : /// This is expensive for an on-the-fly query because it calls
    1318             : /// bumpDownwardPressure to recompute the pressure sets based on current
    1319             : /// liveness. We don't yet have a fast version of downward pressure tracking
    1320             : /// analogous to getUpwardPressureDelta.
    1321      118256 : void RegPressureTracker::
    1322             : getMaxDownwardPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta,
    1323             :                             ArrayRef<PressureChange> CriticalPSets,
    1324             :                             ArrayRef<unsigned> MaxPressureLimit) {
    1325             :   // Snapshot Pressure.
    1326      118256 :   std::vector<unsigned> SavedPressure = CurrSetPressure;
    1327      118256 :   std::vector<unsigned> SavedMaxPressure = P.MaxSetPressure;
    1328             : 
    1329      118256 :   bumpDownwardPressure(MI);
    1330             : 
    1331      236512 :   computeExcessPressureDelta(SavedPressure, CurrSetPressure, Delta, RCI,
    1332             :                              LiveThruPressure);
    1333      236512 :   computeMaxPressureDelta(SavedMaxPressure, P.MaxSetPressure, CriticalPSets,
    1334             :                           MaxPressureLimit, Delta);
    1335             :   assert(Delta.CriticalMax.getUnitInc() >= 0 &&
    1336             :          Delta.CurrentMax.getUnitInc() >= 0 && "cannot decrease max pressure");
    1337             : 
    1338             :   // Restore the tracker's state.
    1339      118256 :   P.MaxSetPressure.swap(SavedMaxPressure);
    1340             :   CurrSetPressure.swap(SavedPressure);
    1341      118256 : }
    1342             : 
    1343             : /// Get the pressure of each PSet after traversing this instruction bottom-up.
    1344     2618672 : void RegPressureTracker::
    1345             : getUpwardPressure(const MachineInstr *MI,
    1346             :                   std::vector<unsigned> &PressureResult,
    1347             :                   std::vector<unsigned> &MaxPressureResult) {
    1348             :   // Snapshot pressure.
    1349     2618672 :   PressureResult = CurrSetPressure;
    1350     2618672 :   MaxPressureResult = P.MaxSetPressure;
    1351             : 
    1352     2618672 :   bumpUpwardPressure(MI);
    1353             : 
    1354             :   // Current pressure becomes the result. Restore current pressure.
    1355     2618672 :   P.MaxSetPressure.swap(MaxPressureResult);
    1356             :   CurrSetPressure.swap(PressureResult);
    1357     2618672 : }
    1358             : 
    1359             : /// Get the pressure of each PSet after traversing this instruction top-down.
    1360      434038 : void RegPressureTracker::
    1361             : getDownwardPressure(const MachineInstr *MI,
    1362             :                     std::vector<unsigned> &PressureResult,
    1363             :                     std::vector<unsigned> &MaxPressureResult) {
    1364             :   // Snapshot pressure.
    1365      434038 :   PressureResult = CurrSetPressure;
    1366      434038 :   MaxPressureResult = P.MaxSetPressure;
    1367             : 
    1368      434038 :   bumpDownwardPressure(MI);
    1369             : 
    1370             :   // Current pressure becomes the result. Restore current pressure.
    1371      434038 :   P.MaxSetPressure.swap(MaxPressureResult);
    1372             :   CurrSetPressure.swap(PressureResult);
    1373      434038 : }

Generated by: LCOV version 1.13