LCOV - code coverage report
Current view: top level - lib/CodeGen - RegisterPressure.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 615 661 93.0 %
Date: 2017-09-14 15:23:50 Functions: 60 67 89.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- RegisterPressure.cpp - Dynamic Register Pressure -------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the RegisterPressure class which can be used to track
      11             : // MachineInstr level register pressure.
      12             : //
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "llvm/CodeGen/RegisterPressure.h"
      16             : #include "llvm/ADT/ArrayRef.h"
      17             : #include "llvm/ADT/STLExtras.h"
      18             : #include "llvm/ADT/SmallVector.h"
      19             : #include "llvm/CodeGen/LiveInterval.h"
      20             : #include "llvm/CodeGen/LiveIntervalAnalysis.h"
      21             : #include "llvm/CodeGen/MachineBasicBlock.h"
      22             : #include "llvm/CodeGen/MachineFunction.h"
      23             : #include "llvm/CodeGen/MachineInstr.h"
      24             : #include "llvm/CodeGen/MachineInstrBundle.h"
      25             : #include "llvm/CodeGen/MachineOperand.h"
      26             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      27             : #include "llvm/CodeGen/RegisterClassInfo.h"
      28             : #include "llvm/CodeGen/SlotIndexes.h"
      29             : #include "llvm/MC/LaneBitmask.h"
      30             : #include "llvm/MC/MCRegisterInfo.h"
      31             : #include "llvm/Support/Compiler.h"
      32             : #include "llvm/Support/Debug.h"
      33             : #include "llvm/Support/ErrorHandling.h"
      34             : #include "llvm/Support/raw_ostream.h"
      35             : #include "llvm/Target/TargetRegisterInfo.h"
      36             : #include "llvm/Target/TargetSubtargetInfo.h"
      37             : #include <algorithm>
      38             : #include <cassert>
      39             : #include <cstdint>
      40             : #include <cstdlib>
      41             : #include <cstring>
      42             : #include <iterator>
      43             : #include <limits>
      44             : #include <utility>
      45             : #include <vector>
      46             : 
      47             : using namespace llvm;
      48             : 
      49             : /// Increase pressure for each pressure set provided by TargetRegisterInfo.
      50      271120 : static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure,
      51             :                                 const MachineRegisterInfo &MRI, unsigned Reg,
      52             :                                 LaneBitmask PrevMask, LaneBitmask NewMask) {
      53             :   assert((PrevMask & ~NewMask).none() && "Must not remove bits");
      54      541424 :   if (PrevMask.any() || NewMask.none())
      55         816 :     return;
      56             : 
      57      270304 :   PSetIterator PSetI = MRI.getPressureSets(Reg);
      58      270304 :   unsigned Weight = PSetI.getWeight();
      59     1086244 :   for (; PSetI.isValid(); ++PSetI)
      60     1223910 :     CurrSetPressure[*PSetI] += Weight;
      61             : }
      62             : 
      63             : /// Decrease pressure for each pressure set provided by TargetRegisterInfo.
      64     4229249 : static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
      65             :                                 const MachineRegisterInfo &MRI, unsigned Reg,
      66             :                                 LaneBitmask PrevMask, LaneBitmask NewMask) {
      67             :   //assert((NewMask & !PrevMask) == 0 && "Must not add bits");
      68     7705489 :   if (NewMask.any() || PrevMask.none())
      69      753046 :     return;
      70             : 
      71     3476203 :   PSetIterator PSetI = MRI.getPressureSets(Reg);
      72     3476203 :   unsigned Weight = PSetI.getWeight();
      73    18345551 :   for (; PSetI.isValid(); ++PSetI) {
      74             :     assert(CurrSetPressure[*PSetI] >= Weight && "register pressure underflow");
      75    22304022 :     CurrSetPressure[*PSetI] -= Weight;
      76             :   }
      77             : }
      78             : 
      79             : #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
      80             : LLVM_DUMP_METHOD
      81             : void llvm::dumpRegSetPressure(ArrayRef<unsigned> SetPressure,
      82             :                               const TargetRegisterInfo *TRI) {
      83             :   bool Empty = true;
      84             :   for (unsigned i = 0, e = SetPressure.size(); i < e; ++i) {
      85             :     if (SetPressure[i] != 0) {
      86             :       dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << '\n';
      87             :       Empty = false;
      88             :     }
      89             :   }
      90             :   if (Empty)
      91             :     dbgs() << "\n";
      92             : }
      93             : 
      94             : LLVM_DUMP_METHOD
      95             : void RegisterPressure::dump(const TargetRegisterInfo *TRI) const {
      96             :   dbgs() << "Max Pressure: ";
      97             :   dumpRegSetPressure(MaxSetPressure, TRI);
      98             :   dbgs() << "Live In: ";
      99             :   for (const RegisterMaskPair &P : LiveInRegs) {
     100             :     dbgs() << PrintVRegOrUnit(P.RegUnit, TRI);
     101             :     if (!P.LaneMask.all())
     102             :       dbgs() << ':' << PrintLaneMask(P.LaneMask);
     103             :     dbgs() << ' ';
     104             :   }
     105             :   dbgs() << '\n';
     106             :   dbgs() << "Live Out: ";
     107             :   for (const RegisterMaskPair &P : LiveOutRegs) {
     108             :     dbgs() << PrintVRegOrUnit(P.RegUnit, TRI);
     109             :     if (!P.LaneMask.all())
     110             :       dbgs() << ':' << PrintLaneMask(P.LaneMask);
     111             :     dbgs() << ' ';
     112             :   }
     113             :   dbgs() << '\n';
     114             : }
     115             : 
     116             : LLVM_DUMP_METHOD
     117             : void RegPressureTracker::dump() const {
     118             :   if (!isTopClosed() || !isBottomClosed()) {
     119             :     dbgs() << "Curr Pressure: ";
     120             :     dumpRegSetPressure(CurrSetPressure, TRI);
     121             :   }
     122             :   P.dump(TRI);
     123             : }
     124             : 
     125             : LLVM_DUMP_METHOD
     126             : void PressureDiff::dump(const TargetRegisterInfo &TRI) const {
     127             :   const char *sep = "";
     128             :   for (const PressureChange &Change : *this) {
     129             :     if (!Change.isValid())
     130             :       break;
     131             :     dbgs() << sep << TRI.getRegPressureSetName(Change.getPSet())
     132             :            << " " << Change.getUnitInc();
     133             :     sep = "    ";
     134             :   }
     135             :   dbgs() << '\n';
     136             : }
     137             : #endif
     138             : 
     139     5240304 : void RegPressureTracker::increaseRegPressure(unsigned RegUnit,
     140             :                                              LaneBitmask PreviousMask,
     141             :                                              LaneBitmask NewMask) {
     142     9626402 :   if (PreviousMask.any() || NewMask.none())
     143      854229 :     return;
     144             : 
     145     8772150 :   PSetIterator PSetI = MRI->getPressureSets(RegUnit);
     146     4386075 :   unsigned Weight = PSetI.getWeight();
     147    24959513 :   for (; PSetI.isValid(); ++PSetI) {
     148    30860157 :     CurrSetPressure[*PSetI] += Weight;
     149    20573438 :     P.MaxSetPressure[*PSetI] =
     150    61720314 :         std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]);
     151             :   }
     152             : }
     153             : 
     154     4229249 : void RegPressureTracker::decreaseRegPressure(unsigned RegUnit,
     155             :                                              LaneBitmask PreviousMask,
     156             :                                              LaneBitmask NewMask) {
     157     4229249 :   decreaseSetPressure(CurrSetPressure, *MRI, RegUnit, PreviousMask, NewMask);
     158     4229249 : }
     159             : 
     160             : /// Clear the result so it can be used for another round of pressure tracking.
     161      531653 : void IntervalPressure::reset() {
     162      531653 :   TopIdx = BottomIdx = SlotIndex();
     163     1063306 :   MaxSetPressure.clear();
     164     1063306 :   LiveInRegs.clear();
     165     1063306 :   LiveOutRegs.clear();
     166      531653 : }
     167             : 
     168             : /// Clear the result so it can be used for another round of pressure tracking.
     169           0 : void RegionPressure::reset() {
     170           0 :   TopPos = BottomPos = MachineBasicBlock::const_iterator();
     171           0 :   MaxSetPressure.clear();
     172           0 :   LiveInRegs.clear();
     173           0 :   LiveOutRegs.clear();
     174           0 : }
     175             : 
     176             : /// If the current top is not less than or equal to the next index, open it.
     177             : /// We happen to need the SlotIndex for the next top for pressure update.
     178           0 : void IntervalPressure::openTop(SlotIndex NextTop) {
     179           0 :   if (TopIdx <= NextTop)
     180             :     return;
     181           0 :   TopIdx = SlotIndex();
     182           0 :   LiveInRegs.clear();
     183             : }
     184             : 
     185             : /// If the current top is the previous instruction (before receding), open it.
     186           0 : void RegionPressure::openTop(MachineBasicBlock::const_iterator PrevTop) {
     187           0 :   if (TopPos != PrevTop)
     188             :     return;
     189           0 :   TopPos = MachineBasicBlock::const_iterator();
     190           0 :   LiveInRegs.clear();
     191             : }
     192             : 
     193             : /// If the current bottom is not greater than the previous index, open it.
     194           0 : void IntervalPressure::openBottom(SlotIndex PrevBottom) {
     195           0 :   if (BottomIdx > PrevBottom)
     196             :     return;
     197           0 :   BottomIdx = SlotIndex();
     198           0 :   LiveInRegs.clear();
     199             : }
     200             : 
     201             : /// If the current bottom is the previous instr (before advancing), open it.
     202           0 : void RegionPressure::openBottom(MachineBasicBlock::const_iterator PrevBottom) {
     203           0 :   if (BottomPos != PrevBottom)
     204             :     return;
     205           0 :   BottomPos = MachineBasicBlock::const_iterator();
     206           0 :   LiveInRegs.clear();
     207             : }
     208             : 
     209      276571 : void LiveRegSet::init(const MachineRegisterInfo &MRI) {
     210      553142 :   const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
     211      276571 :   unsigned NumRegUnits = TRI.getNumRegs();
     212      276571 :   unsigned NumVirtRegs = MRI.getNumVirtRegs();
     213      276571 :   Regs.setUniverse(NumRegUnits + NumVirtRegs);
     214      276571 :   this->NumRegUnits = NumRegUnits;
     215      276571 : }
     216             : 
     217      531653 : void LiveRegSet::clear() {
     218     1063306 :   Regs.clear();
     219      531653 : }
     220             : 
     221             : static const LiveRange *getLiveRange(const LiveIntervals &LIS, unsigned Reg) {
     222      702559 :   if (TargetRegisterInfo::isVirtualRegister(Reg))
     223      611125 :     return &LIS.getInterval(Reg);
     224       91434 :   return LIS.getCachedRegUnit(Reg);
     225             : }
     226             : 
     227      531653 : void RegPressureTracker::reset() {
     228      531653 :   MBB = nullptr;
     229      531653 :   LIS = nullptr;
     230             : 
     231     1063306 :   CurrSetPressure.clear();
     232     1063306 :   LiveThruPressure.clear();
     233     1063306 :   P.MaxSetPressure.clear();
     234             : 
     235      531653 :   if (RequireIntervals)
     236      531653 :     static_cast<IntervalPressure&>(P).reset();
     237             :   else
     238           0 :     static_cast<RegionPressure&>(P).reset();
     239             : 
     240      531653 :   LiveRegs.clear();
     241     1063306 :   UntiedDefs.clear();
     242      531653 : }
     243             : 
     244             : /// Setup the RegPressureTracker.
     245             : ///
     246             : /// TODO: Add support for pressure without LiveIntervals.
     247      276571 : void RegPressureTracker::init(const MachineFunction *mf,
     248             :                               const RegisterClassInfo *rci,
     249             :                               const LiveIntervals *lis,
     250             :                               const MachineBasicBlock *mbb,
     251             :                               MachineBasicBlock::const_iterator pos,
     252             :                               bool TrackLaneMasks, bool TrackUntiedDefs) {
     253      276571 :   reset();
     254             : 
     255      276571 :   MF = mf;
     256      276571 :   TRI = MF->getSubtarget().getRegisterInfo();
     257      276571 :   RCI = rci;
     258      276571 :   MRI = &MF->getRegInfo();
     259      276571 :   MBB = mbb;
     260      276571 :   this->TrackUntiedDefs = TrackUntiedDefs;
     261      276571 :   this->TrackLaneMasks = TrackLaneMasks;
     262             : 
     263      276571 :   if (RequireIntervals) {
     264             :     assert(lis && "IntervalPressure requires LiveIntervals");
     265      276571 :     LIS = lis;
     266             :   }
     267             : 
     268      276571 :   CurrPos = pos;
     269      553142 :   CurrSetPressure.assign(TRI->getNumRegPressureSets(), 0);
     270             : 
     271      276571 :   P.MaxSetPressure = CurrSetPressure;
     272             : 
     273      276571 :   LiveRegs.init(*MRI);
     274      276571 :   if (TrackUntiedDefs)
     275      184404 :     UntiedDefs.setUniverse(MRI->getNumVirtRegs());
     276      276571 : }
     277             : 
     278             : /// Does this pressure result have a valid top position and live ins.
     279     2882956 : bool RegPressureTracker::isTopClosed() const {
     280     2882956 :   if (RequireIntervals)
     281     5765912 :     return static_cast<IntervalPressure&>(P).TopIdx.isValid();
     282           0 :   return (static_cast<RegionPressure&>(P).TopPos ==
     283           0 :           MachineBasicBlock::const_iterator());
     284             : }
     285             : 
     286             : /// Does this pressure result have a valid bottom position and live outs.
     287     2829237 : bool RegPressureTracker::isBottomClosed() const {
     288     2829237 :   if (RequireIntervals)
     289     5658474 :     return static_cast<IntervalPressure&>(P).BottomIdx.isValid();
     290           0 :   return (static_cast<RegionPressure&>(P).BottomPos ==
     291           0 :           MachineBasicBlock::const_iterator());
     292             : }
     293             : 
     294      656487 : SlotIndex RegPressureTracker::getCurrSlot() const {
     295             :   MachineBasicBlock::const_iterator IdxPos =
     296     1969461 :     skipDebugInstructionsForward(CurrPos, MBB->end());
     297     1969461 :   if (IdxPos == MBB->end())
     298       92489 :     return LIS->getMBBEndIdx(MBB);
     299     2255992 :   return LIS->getInstructionIndex(*IdxPos).getRegSlot();
     300             : }
     301             : 
     302             : /// Set the boundary for the top of the region and summarize live ins.
     303      184364 : void RegPressureTracker::closeTop() {
     304      184364 :   if (RequireIntervals)
     305      184364 :     static_cast<IntervalPressure&>(P).TopIdx = getCurrSlot();
     306             :   else
     307           0 :     static_cast<RegionPressure&>(P).TopPos = CurrPos;
     308             : 
     309             :   assert(P.LiveInRegs.empty() && "inconsistent max pressure result");
     310      368728 :   P.LiveInRegs.reserve(LiveRegs.size());
     311      184364 :   LiveRegs.appendTo(P.LiveInRegs);
     312      184364 : }
     313             : 
     314             : /// Set the boundary for the bottom of the region and summarize live outs.
     315      184384 : void RegPressureTracker::closeBottom() {
     316      184384 :   if (RequireIntervals)
     317      184384 :     static_cast<IntervalPressure&>(P).BottomIdx = getCurrSlot();
     318             :   else
     319           0 :     static_cast<RegionPressure&>(P).BottomPos = CurrPos;
     320             : 
     321             :   assert(P.LiveOutRegs.empty() && "inconsistent max pressure result");
     322      368768 :   P.LiveOutRegs.reserve(LiveRegs.size());
     323      184384 :   LiveRegs.appendTo(P.LiveOutRegs);
     324      184384 : }
     325             : 
     326             : /// Finalize the region boundaries and record live ins and live outs.
     327       92182 : void RegPressureTracker::closeRegion() {
     328       92182 :   if (!isTopClosed() && !isBottomClosed()) {
     329             :     assert(LiveRegs.size() == 0 && "no region boundary");
     330             :     return;
     331             :   }
     332       92182 :   if (!isBottomClosed())
     333           5 :     closeBottom();
     334       92177 :   else if (!isTopClosed())
     335       92177 :     closeTop();
     336             :   // If both top and bottom are closed, do nothing.
     337             : }
     338             : 
     339             : /// The register tracker is unaware of global liveness so ignores normal
     340             : /// live-thru ranges. However, two-address or coalesced chains can also lead
     341             : /// to live ranges with no holes. Count these to inform heuristics that we
     342             : /// can never drop below this pressure.
     343       92177 : void RegPressureTracker::initLiveThru(const RegPressureTracker &RPTracker) {
     344      184354 :   LiveThruPressure.assign(TRI->getNumRegPressureSets(), 0);
     345             :   assert(isBottomClosed() && "need bottom-up tracking to intialize.");
     346      424929 :   for (const RegisterMaskPair &Pair : P.LiveOutRegs) {
     347      148398 :     unsigned RegUnit = Pair.RegUnit;
     348      148398 :     if (TargetRegisterInfo::isVirtualRegister(RegUnit)
     349      293377 :         && !RPTracker.hasUntiedDef(RegUnit))
     350       67508 :       increaseSetPressure(LiveThruPressure, *MRI, RegUnit,
     351             :                           LaneBitmask::getNone(), Pair.LaneMask);
     352             :   }
     353       92177 : }
     354             : 
     355     2111188 : static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits,
     356             :                                unsigned RegUnit) {
     357     2111188 :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     358             :     return Other.RegUnit == RegUnit;
     359     2111188 :   });
     360     2111188 :   if (I == RegUnits.end())
     361             :     return LaneBitmask::getNone();
     362       61143 :   return I->LaneMask;
     363             : }
     364             : 
     365    11223790 : static void addRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits,
     366             :                         RegisterMaskPair Pair) {
     367    11223790 :   unsigned RegUnit = Pair.RegUnit;
     368             :   assert(Pair.LaneMask.any());
     369    11223790 :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     370             :     return Other.RegUnit == RegUnit;
     371    11223790 :   });
     372    11223790 :   if (I == RegUnits.end()) {
     373    11140108 :     RegUnits.push_back(Pair);
     374             :   } else {
     375       83682 :     I->LaneMask |= Pair.LaneMask;
     376             :   }
     377    11223790 : }
     378             : 
     379      144980 : static void setRegZero(SmallVectorImpl<RegisterMaskPair> &RegUnits,
     380             :                        unsigned RegUnit) {
     381      144980 :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     382             :     return Other.RegUnit == RegUnit;
     383      144980 :   });
     384      144980 :   if (I == RegUnits.end()) {
     385      434940 :     RegUnits.push_back(RegisterMaskPair(RegUnit, LaneBitmask::getNone()));
     386             :   } else {
     387           0 :     I->LaneMask = LaneBitmask::getNone();
     388             :   }
     389      144980 : }
     390             : 
     391     4483768 : static void removeRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits,
     392             :                            RegisterMaskPair Pair) {
     393     4483768 :   unsigned RegUnit = Pair.RegUnit;
     394             :   assert(Pair.LaneMask.any());
     395     4483768 :   auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
     396             :     return Other.RegUnit == RegUnit;
     397     4483768 :   });
     398     4483768 :   if (I != RegUnits.end()) {
     399        9646 :     I->LaneMask &= ~Pair.LaneMask;
     400        4823 :     if (I->LaneMask.none())
     401             :       RegUnits.erase(I);
     402             :   }
     403     4483768 : }
     404             : 
     405     8636848 : static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS,
     406             :     const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit,
     407             :     SlotIndex Pos, LaneBitmask SafeDefault,
     408             :     bool(*Property)(const LiveRange &LR, SlotIndex Pos)) {
     409     8636848 :   if (TargetRegisterInfo::isVirtualRegister(RegUnit)) {
     410     7796968 :     const LiveInterval &LI = LIS.getInterval(RegUnit);
     411     7796968 :     LaneBitmask Result;
     412     7796968 :     if (TrackLaneMasks && LI.hasSubRanges()) {
     413    13685415 :         for (const LiveInterval::SubRange &SR : LI.subranges()) {
     414    11000082 :           if (Property(SR, Pos))
     415             :             Result |= SR.LaneMask;
     416             :         }
     417     5111635 :     } else if (Property(LI, Pos)) {
     418     3679813 :       Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit)
     419             :                               : LaneBitmask::getAll();
     420             :     }
     421             : 
     422     7796968 :     return Result;
     423             :   } else {
     424      839880 :     const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
     425             :     // Be prepared for missing liveranges: We usually do not compute liveranges
     426             :     // for physical registers on targets with many registers (GPUs).
     427      839880 :     if (LR == nullptr)
     428      224813 :       return SafeDefault;
     429      615067 :     return Property(*LR, Pos) ? LaneBitmask::getAll() : LaneBitmask::getNone();
     430             :   }
     431             : }
     432             : 
     433             : static LaneBitmask getLiveLanesAt(const LiveIntervals &LIS,
     434             :                                   const MachineRegisterInfo &MRI,
     435             :                                   bool TrackLaneMasks, unsigned RegUnit,
     436             :                                   SlotIndex Pos) {
     437             :   return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos,
     438             :                               LaneBitmask::getAll(),
     439    13990291 :                               [](const LiveRange &LR, SlotIndex Pos) {
     440             :                                 return LR.liveAt(Pos);
     441    20318784 :                               });
     442             : }
     443             : 
     444             : 
     445             : namespace {
     446             : 
     447             : /// Collect this instruction's unique uses and defs into SmallVectors for
     448             : /// processing defs and uses in order.
     449             : ///
     450             : /// FIXME: always ignore tied opers
     451             : class RegisterOperandsCollector {
     452             :   friend class llvm::RegisterOperands;
     453             : 
     454             :   RegisterOperands &RegOpers;
     455             :   const TargetRegisterInfo &TRI;
     456             :   const MachineRegisterInfo &MRI;
     457             :   bool IgnoreDead;
     458             : 
     459             :   RegisterOperandsCollector(RegisterOperands &RegOpers,
     460             :                             const TargetRegisterInfo &TRI,
     461             :                             const MachineRegisterInfo &MRI, bool IgnoreDead)
     462     5584156 :     : RegOpers(RegOpers), TRI(TRI), MRI(MRI), IgnoreDead(IgnoreDead) {}
     463             : 
     464     2185739 :   void collectInstr(const MachineInstr &MI) const {
     465    12639055 :     for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI)
     466    10453316 :       collectOperand(*OperI);
     467             : 
     468             :     // Remove redundant physreg dead defs.
     469     8026261 :     for (const RegisterMaskPair &P : RegOpers.Defs)
     470     1469044 :       removeRegLanes(RegOpers.DeadDefs, P);
     471     2185739 :   }
     472             : 
     473     3398417 :   void collectInstrLanes(const MachineInstr &MI) const {
     474    17595875 :     for (ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI)
     475    14197458 :       collectOperandLanes(*OperI);
     476             : 
     477             :     // Remove redundant physreg dead defs.
     478    13208444 :     for (const RegisterMaskPair &P : RegOpers.Defs)
     479     3013193 :       removeRegLanes(RegOpers.DeadDefs, P);
     480     3398417 :   }
     481             : 
     482             :   /// Push this operand's register onto the correct vectors.
     483    10453316 :   void collectOperand(const MachineOperand &MO) const {
     484    10453316 :     if (!MO.isReg() || !MO.getReg())
     485             :       return;
     486     4919566 :     unsigned Reg = MO.getReg();
     487     4919566 :     if (MO.isUse()) {
     488     6079918 :       if (!MO.isUndef() && !MO.isInternalRead())
     489     3036237 :         pushReg(Reg, RegOpers.Uses);
     490             :     } else {
     491             :       assert(MO.isDef());
     492             :       // Subregister definitions may imply a register read.
     493       59923 :       if (MO.readsReg())
     494       59923 :         pushReg(Reg, RegOpers.Uses);
     495             : 
     496     1875885 :       if (MO.isDead()) {
     497      284039 :         if (!IgnoreDead)
     498      283193 :           pushReg(Reg, RegOpers.DeadDefs);
     499             :       } else
     500     1591846 :         pushReg(Reg, RegOpers.Defs);
     501             :     }
     502             :   }
     503             : 
     504     4971199 :   void pushReg(unsigned Reg,
     505             :                SmallVectorImpl<RegisterMaskPair> &RegUnits) const {
     506     4971199 :     if (TargetRegisterInfo::isVirtualRegister(Reg)) {
     507     6913524 :       addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneBitmask::getAll()));
     508     1514437 :     } else if (MRI.isAllocatable(Reg)) {
     509     1258580 :       for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
     510     1539240 :         addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
     511             :     }
     512     4971199 :   }
     513             : 
     514    14197458 :   void collectOperandLanes(const MachineOperand &MO) const {
     515    14197458 :     if (!MO.isReg() || !MO.getReg())
     516             :       return;
     517     9089206 :     unsigned Reg = MO.getReg();
     518     9089206 :     unsigned SubRegIdx = MO.getSubReg();
     519     9089206 :     if (MO.isUse()) {
     520    11521019 :       if (!MO.isUndef() && !MO.isInternalRead())
     521     5757303 :         pushRegLanes(Reg, SubRegIdx, RegOpers.Uses);
     522             :     } else {
     523             :       assert(MO.isDef());
     524             :       // Treat read-undef subreg defs as definitions of the whole register.
     525     3325490 :       if (MO.isUndef())
     526       33269 :         SubRegIdx = 0;
     527             : 
     528     3325490 :       if (MO.isDead()) {
     529      318681 :         if (!IgnoreDead)
     530       59650 :           pushRegLanes(Reg, SubRegIdx, RegOpers.DeadDefs);
     531             :       } else
     532     3006809 :         pushRegLanes(Reg, SubRegIdx, RegOpers.Defs);
     533             :     }
     534             :   }
     535             : 
     536     8823762 :   void pushRegLanes(unsigned Reg, unsigned SubRegIdx,
     537             :                     SmallVectorImpl<RegisterMaskPair> &RegUnits) const {
     538     8823762 :     if (TargetRegisterInfo::isVirtualRegister(Reg)) {
     539             :       LaneBitmask LaneMask = SubRegIdx != 0
     540     1929060 :                              ? TRI.getSubRegIndexLaneMask(SubRegIdx)
     541     9877802 :                              : MRI.getMaxLaneMaskForVReg(Reg);
     542     6019682 :       addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneMask));
     543     2804080 :     } else if (MRI.isAllocatable(Reg)) {
     544      863881 :       for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
     545     1116597 :         addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
     546             :     }
     547     8823762 :   }
     548             : };
     549             : 
     550             : } // end anonymous namespace
     551             : 
     552     5584156 : void RegisterOperands::collect(const MachineInstr &MI,
     553             :                                const TargetRegisterInfo &TRI,
     554             :                                const MachineRegisterInfo &MRI,
     555             :                                bool TrackLaneMasks, bool IgnoreDead) {
     556    11168312 :   RegisterOperandsCollector Collector(*this, TRI, MRI, IgnoreDead);
     557     5584156 :   if (TrackLaneMasks)
     558     3398417 :     Collector.collectInstrLanes(MI);
     559             :   else
     560     2185739 :     Collector.collectInstr(MI);
     561     5584156 : }
     562             : 
     563     1131947 : void RegisterOperands::detectDeadDefs(const MachineInstr &MI,
     564             :                                       const LiveIntervals &LIS) {
     565     2263894 :   SlotIndex SlotIdx = LIS.getInstructionIndex(MI);
     566     4098400 :   for (auto RI = Defs.begin(); RI != Defs.end(); /*empty*/) {
     567      702559 :     unsigned Reg = RI->RegUnit;
     568      702559 :     const LiveRange *LR = getLiveRange(LIS, Reg);
     569      702559 :     if (LR != nullptr) {
     570      651667 :       LiveQueryResult LRQ = LR->Query(SlotIdx);
     571      651667 :       if (LRQ.isDeadDef()) {
     572             :         // LiveIntervals knows this is a dead even though it's MachineOperand is
     573             :         // not flagged as such.
     574           0 :         DeadDefs.push_back(*RI);
     575           0 :         RI = Defs.erase(RI);
     576           0 :         continue;
     577             :       }
     578             :     }
     579      702559 :     ++RI;
     580             :   }
     581     1131947 : }
     582             : 
     583     3398417 : void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS,
     584             :                                           const MachineRegisterInfo &MRI,
     585             :                                           SlotIndex Pos,
     586             :                                           MachineInstr *AddFlagsMI) {
     587    13208444 :   for (auto I = Defs.begin(); I != Defs.end(); ) {
     588             :     LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
     589     6026386 :                                            Pos.getDeadSlot());
     590             :     // If the the def is all that is live after the instruction, then in case
     591             :     // of a subregister def we need a read-undef flag.
     592     3013193 :     unsigned RegUnit = I->RegUnit;
     593     5936776 :     if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
     594     3718922 :         AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask).none())
     595      175203 :       AddFlagsMI->setRegisterDefReadUndef(RegUnit);
     596             : 
     597     6026386 :     LaneBitmask ActualDef = I->LaneMask & LiveAfter;
     598     3013193 :     if (ActualDef.none()) {
     599           0 :       I = Defs.erase(I);
     600             :     } else {
     601     3013193 :       I->LaneMask = ActualDef;
     602     3013193 :       ++I;
     603             :     }
     604             :   }
     605    13510055 :   for (auto I = Uses.begin(); I != Uses.end(); ) {
     606             :     LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
     607     6629608 :                                             Pos.getBaseIndex());
     608     6629608 :     LaneBitmask LaneMask = I->LaneMask & LiveBefore;
     609     3314804 :     if (LaneMask.none()) {
     610           0 :       I = Uses.erase(I);
     611             :     } else {
     612     3314804 :       I->LaneMask = LaneMask;
     613     3314804 :       ++I;
     614             :     }
     615             :   }
     616     3398417 :   if (AddFlagsMI != nullptr) {
     617      836154 :     for (const RegisterMaskPair &P : DeadDefs) {
     618       15909 :       unsigned RegUnit = P.RegUnit;
     619       15909 :       if (!TargetRegisterInfo::isVirtualRegister(RegUnit))
     620       15413 :         continue;
     621             :       LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit,
     622         992 :                                              Pos.getDeadSlot());
     623         496 :       if (LiveAfter.none())
     624         496 :         AddFlagsMI->setRegisterDefReadUndef(RegUnit);
     625             :     }
     626             :   }
     627     3398417 : }
     628             : 
     629             : /// Initialize an array of N PressureDiffs.
     630       92177 : void PressureDiffs::init(unsigned N) {
     631       92177 :   Size = N;
     632       92177 :   if (N <= Max) {
     633       47165 :     memset(PDiffArray, 0, N * sizeof(PressureDiff));
     634       47165 :     return;
     635             :   }
     636       45012 :   Max = Size;
     637       45012 :   free(PDiffArray);
     638       45012 :   PDiffArray = reinterpret_cast<PressureDiff*>(calloc(N, sizeof(PressureDiff)));
     639             : }
     640             : 
     641     1235727 : void PressureDiffs::addInstruction(unsigned Idx,
     642             :                                    const RegisterOperands &RegOpers,
     643             :                                    const MachineRegisterInfo &MRI) {
     644     2471454 :   PressureDiff &PDiff = (*this)[Idx];
     645             :   assert(!PDiff.begin()->isValid() && "stale PDiff");
     646     4636956 :   for (const RegisterMaskPair &P : RegOpers.Defs)
     647      929775 :     PDiff.addPressureChange(P.RegUnit, true, &MRI);
     648             : 
     649     5128107 :   for (const RegisterMaskPair &P : RegOpers.Uses)
     650     1420926 :     PDiff.addPressureChange(P.RegUnit, false, &MRI);
     651     1235727 : }
     652             : 
     653             : /// Add a change in pressure to the pressure diff of a given instruction.
     654     3634229 : void PressureDiff::addPressureChange(unsigned RegUnit, bool IsDec,
     655             :                                      const MachineRegisterInfo *MRI) {
     656     3634229 :   PSetIterator PSetI = MRI->getPressureSets(RegUnit);
     657     3634229 :   int Weight = IsDec ? -PSetI.getWeight() : PSetI.getWeight();
     658    19795433 :   for (; PSetI.isValid(); ++PSetI) {
     659             :     // Find an existing entry in the pressure diff for this PSet.
     660    16366138 :     PressureDiff::iterator I = nonconst_begin(), E = nonconst_end();
     661    44376007 :     for (; I != E && I->isValid(); ++I) {
     662    66886590 :       if (I->getPSet() >= *PSetI)
     663             :         break;
     664             :     }
     665             :     // If all pressure sets are more constrained, skip the remaining PSets.
     666     8183069 :     if (I == E)
     667             :       break;
     668             :     // Insert this PressureChange.
     669    16478724 :     if (!I->isValid() || I->getPSet() != *PSetI) {
     670    14450931 :       PressureChange PTmp = PressureChange(*PSetI);
     671    11177520 :       for (PressureDiff::iterator J = I; J != E && PTmp.isValid(); ++J)
     672     6360543 :         std::swap(*J, PTmp);
     673             :     }
     674             :     // Update the units for this pressure set.
     675    16161204 :     unsigned NewUnitInc = I->getUnitInc() + Weight;
     676     8080602 :     if (NewUnitInc != 0) {
     677     5535808 :       I->setUnitInc(NewUnitInc);
     678             :     } else {
     679             :       // Remove entry
     680             :       PressureDiff::iterator J;
     681     6225984 :       for (J = std::next(I); J != E && J->isValid(); ++J, ++I)
     682     3681190 :         *I = *J;
     683     2544794 :       if (J != E)
     684     2515470 :         *I = *J;
     685             :     }
     686             :   }
     687     3634229 : }
     688             : 
     689             : /// Force liveness of registers.
     690      184389 : void RegPressureTracker::addLiveRegs(ArrayRef<RegisterMaskPair> Regs) {
     691      713684 :   for (const RegisterMaskPair &P : Regs) {
     692      344906 :     LaneBitmask PrevMask = LiveRegs.insert(P);
     693      344906 :     LaneBitmask NewMask = PrevMask | P.LaneMask;
     694      344906 :     increaseRegPressure(P.RegUnit, PrevMask, NewMask);
     695             :   }
     696      184389 : }
     697             : 
     698      149311 : void RegPressureTracker::discoverLiveInOrOut(RegisterMaskPair Pair,
     699             :     SmallVectorImpl<RegisterMaskPair> &LiveInOrOut) {
     700             :   assert(Pair.LaneMask.any());
     701             : 
     702      149311 :   unsigned RegUnit = Pair.RegUnit;
     703      149311 :   auto I = llvm::find_if(LiveInOrOut, [RegUnit](const RegisterMaskPair &Other) {
     704             :     return Other.RegUnit == RegUnit;
     705      149311 :   });
     706      149311 :   LaneBitmask PrevMask;
     707      149311 :   LaneBitmask NewMask;
     708      149311 :   if (I == LiveInOrOut.end()) {
     709      148495 :     PrevMask = LaneBitmask::getNone();
     710      148495 :     NewMask = Pair.LaneMask;
     711      148495 :     LiveInOrOut.push_back(Pair);
     712             :   } else {
     713         816 :     PrevMask = I->LaneMask;
     714         816 :     NewMask = PrevMask | Pair.LaneMask;
     715         816 :     I->LaneMask = NewMask;
     716             :   }
     717      149311 :   increaseSetPressure(P.MaxSetPressure, *MRI, RegUnit, PrevMask, NewMask);
     718      149311 : }
     719             : 
     720          15 : void RegPressureTracker::discoverLiveIn(RegisterMaskPair Pair) {
     721          15 :   discoverLiveInOrOut(Pair, P.LiveInRegs);
     722          15 : }
     723             : 
     724      149296 : void RegPressureTracker::discoverLiveOut(RegisterMaskPair Pair) {
     725      149296 :   discoverLiveInOrOut(Pair, P.LiveOutRegs);
     726      149296 : }
     727             : 
     728     5577620 : void RegPressureTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
     729    11208546 :   for (const RegisterMaskPair &P : DeadDefs) {
     730       53306 :     unsigned Reg = P.RegUnit;
     731       53306 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
     732       53306 :     LaneBitmask BumpedMask = LiveMask | P.LaneMask;
     733       53306 :     increaseRegPressure(Reg, LiveMask, BumpedMask);
     734             :   }
     735     5684232 :   for (const RegisterMaskPair &P : DeadDefs) {
     736       53306 :     unsigned Reg = P.RegUnit;
     737       53306 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
     738       53306 :     LaneBitmask BumpedMask = LiveMask | P.LaneMask;
     739       53306 :     decreaseRegPressure(Reg, BumpedMask, LiveMask);
     740             :   }
     741     5577620 : }
     742             : 
     743             : /// Recede across the previous instruction. If LiveUses is provided, record any
     744             : /// RegUnits that are made live by the current instruction's uses. This includes
     745             : /// registers that are both defined and used by the instruction.  If a pressure
     746             : /// difference pointer is provided record the changes is pressure caused by this
     747             : /// instruction independent of liveness.
     748     2586544 : void RegPressureTracker::recede(const RegisterOperands &RegOpers,
     749             :                                 SmallVectorImpl<RegisterMaskPair> *LiveUses) {
     750             :   assert(!CurrPos->isDebugValue());
     751             : 
     752             :   // Boost pressure for all dead defs together.
     753     5173088 :   bumpDeadDefs(RegOpers.DeadDefs);
     754             : 
     755             :   // Kill liveness at live defs.
     756             :   // TODO: consider earlyclobbers?
     757     9564808 :   for (const RegisterMaskPair &Def : RegOpers.Defs) {
     758     1805176 :     unsigned Reg = Def.RegUnit;
     759             : 
     760     1805176 :     LaneBitmask PreviousMask = LiveRegs.erase(Def);
     761     5415528 :     LaneBitmask NewMask = PreviousMask & ~Def.LaneMask;
     762             : 
     763     3610352 :     LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask;
     764     1805176 :     if (LiveOut.any()) {
     765       54301 :       discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
     766             :       // Retroactively model effects on pressure of the live out lanes.
     767       54301 :       increaseSetPressure(CurrSetPressure, *MRI, Reg, LaneBitmask::getNone(),
     768             :                           LiveOut);
     769       54301 :       PreviousMask = LiveOut;
     770             :     }
     771             : 
     772     1805176 :     if (NewMask.none()) {
     773             :       // Add a 0 entry to LiveUses as a marker that the complete vreg has become
     774             :       // dead.
     775     1694545 :       if (TrackLaneMasks && LiveUses != nullptr)
     776      144980 :         setRegZero(*LiveUses, Reg);
     777             :     }
     778             : 
     779     1805176 :     decreaseRegPressure(Reg, PreviousMask, NewMask);
     780             :   }
     781             : 
     782     2586544 :   SlotIndex SlotIdx;
     783     2586544 :   if (RequireIntervals)
     784    10346176 :     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
     785             : 
     786             :   // Generate liveness for uses.
     787    10696694 :   for (const RegisterMaskPair &Use : RegOpers.Uses) {
     788     2937062 :     unsigned Reg = Use.RegUnit;
     789             :     assert(Use.LaneMask.any());
     790     2937062 :     LaneBitmask PreviousMask = LiveRegs.insert(Use);
     791     2937062 :     LaneBitmask NewMask = PreviousMask | Use.LaneMask;
     792     2937062 :     if (NewMask == PreviousMask)
     793     1019134 :       continue;
     794             : 
     795             :     // Did the register just become live?
     796     1917928 :     if (PreviousMask.none()) {
     797     1879863 :       if (LiveUses != nullptr) {
     798      863598 :         if (!TrackLaneMasks) {
     799      694264 :           addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
     800             :         } else {
     801             :           auto I =
     802      169334 :               llvm::find_if(*LiveUses, [Reg](const RegisterMaskPair Other) {
     803             :                 return Other.RegUnit == Reg;
     804      169334 :               });
     805      169334 :           bool IsRedef = I != LiveUses->end();
     806      169334 :           if (IsRedef) {
     807             :             // ignore re-defs here...
     808             :             assert(I->LaneMask.none());
     809        1531 :             removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
     810             :           } else {
     811      167803 :             addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
     812             :           }
     813             :         }
     814             :       }
     815             : 
     816             :       // Discover live outs if this may be the first occurance of this register.
     817     1879863 :       if (RequireIntervals) {
     818     1879863 :         LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx);
     819     1879863 :         if (LiveOut.any())
     820       94995 :           discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
     821             :       }
     822             :     }
     823             : 
     824     1917928 :     increaseRegPressure(Reg, PreviousMask, NewMask);
     825             :   }
     826     2586544 :   if (TrackUntiedDefs) {
     827     4899309 :     for (const RegisterMaskPair &Def : RegOpers.Defs) {
     828      931827 :       unsigned RegUnit = Def.RegUnit;
     829     2690159 :       if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
     830     2584837 :           (LiveRegs.contains(RegUnit) & Def.LaneMask).none())
     831      664275 :         UntiedDefs.insert(RegUnit);
     832             :     }
     833             :   }
     834     2586544 : }
     835             : 
     836     2586544 : void RegPressureTracker::recedeSkipDebugValues() {
     837             :   assert(CurrPos != MBB->begin());
     838     2586544 :   if (!isBottomClosed())
     839       92177 :     closeBottom();
     840             : 
     841             :   // Open the top of the region using block iterators.
     842     2586544 :   if (!RequireIntervals && isTopClosed())
     843           0 :     static_cast<RegionPressure&>(P).openTop(CurrPos);
     844             : 
     845             :   // Find the previous instruction.
     846    10346176 :   CurrPos = skipDebugInstructionsBackward(std::prev(CurrPos), MBB->begin());
     847             : 
     848     2586544 :   SlotIndex SlotIdx;
     849     2586544 :   if (RequireIntervals)
     850    10346176 :     SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
     851             : 
     852             :   // Open the top of the region using slot indexes.
     853     2586544 :   if (RequireIntervals && isTopClosed())
     854           0 :     static_cast<IntervalPressure&>(P).openTop(SlotIdx);
     855     2586544 : }
     856             : 
     857      173392 : void RegPressureTracker::recede(SmallVectorImpl<RegisterMaskPair> *LiveUses) {
     858      173392 :   recedeSkipDebugValues();
     859             : 
     860      346784 :   const MachineInstr &MI = *CurrPos;
     861      346784 :   RegisterOperands RegOpers;
     862      173392 :   RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
     863      173392 :   if (TrackLaneMasks) {
     864      123480 :     SlotIndex SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
     865       30870 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
     866      142522 :   } else if (RequireIntervals) {
     867      142522 :     RegOpers.detectDeadDefs(MI, *LIS);
     868             :   }
     869             : 
     870      173392 :   recede(RegOpers, LiveUses);
     871      173392 : }
     872             : 
     873             : /// Advance across the current instruction.
     874       58334 : void RegPressureTracker::advance(const RegisterOperands &RegOpers) {
     875             :   assert(!TrackUntiedDefs && "unsupported mode");
     876             :   assert(CurrPos != MBB->end());
     877       58334 :   if (!isTopClosed())
     878           5 :     closeTop();
     879             : 
     880       58334 :   SlotIndex SlotIdx;
     881       58334 :   if (RequireIntervals)
     882       58334 :     SlotIdx = getCurrSlot();
     883             : 
     884             :   // Open the bottom of the region using slot indexes.
     885       58334 :   if (isBottomClosed()) {
     886           0 :     if (RequireIntervals)
     887           0 :       static_cast<IntervalPressure&>(P).openBottom(SlotIdx);
     888             :     else
     889           0 :       static_cast<RegionPressure&>(P).openBottom(CurrPos);
     890             :   }
     891             : 
     892      246961 :   for (const RegisterMaskPair &Use : RegOpers.Uses) {
     893       71959 :     unsigned Reg = Use.RegUnit;
     894       71959 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
     895      143918 :     LaneBitmask LiveIn = Use.LaneMask & ~LiveMask;
     896       71959 :     if (LiveIn.any()) {
     897          15 :       discoverLiveIn(RegisterMaskPair(Reg, LiveIn));
     898          15 :       increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn);
     899          30 :       LiveRegs.insert(RegisterMaskPair(Reg, LiveIn));
     900             :     }
     901             :     // Kill liveness at last uses.
     902       71959 :     if (RequireIntervals) {
     903       71959 :       LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
     904       71959 :       if (LastUseMask.any()) {
     905      118194 :         LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask));
     906      118194 :         decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask);
     907             :       }
     908             :     }
     909             :   }
     910             : 
     911             :   // Generate liveness for defs.
     912      233375 :   for (const RegisterMaskPair &Def : RegOpers.Defs) {
     913       58373 :     LaneBitmask PreviousMask = LiveRegs.insert(Def);
     914       58373 :     LaneBitmask NewMask = PreviousMask | Def.LaneMask;
     915       58373 :     increaseRegPressure(Def.RegUnit, PreviousMask, NewMask);
     916             :   }
     917             : 
     918             :   // Boost pressure for all dead defs together.
     919      116668 :   bumpDeadDefs(RegOpers.DeadDefs);
     920             : 
     921             :   // Find the next instruction.
     922      233336 :   CurrPos = skipDebugInstructionsForward(std::next(CurrPos), MBB->end());
     923       58334 : }
     924             : 
     925          32 : void RegPressureTracker::advance() {
     926          64 :   const MachineInstr &MI = *CurrPos;
     927          64 :   RegisterOperands RegOpers;
     928          32 :   RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
     929          32 :   if (TrackLaneMasks) {
     930           0 :     SlotIndex SlotIdx = getCurrSlot();
     931           0 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
     932             :   }
     933          32 :   advance(RegOpers);
     934          32 : }
     935             : 
     936             : /// Find the max change in excess pressure across all sets.
     937      105464 : static void computeExcessPressureDelta(ArrayRef<unsigned> OldPressureVec,
     938             :                                        ArrayRef<unsigned> NewPressureVec,
     939             :                                        RegPressureDelta &Delta,
     940             :                                        const RegisterClassInfo *RCI,
     941             :                                        ArrayRef<unsigned> LiveThruPressureVec) {
     942      105464 :   Delta.Excess = PressureChange();
     943      828890 :   for (unsigned i = 0, e = OldPressureVec.size(); i < e; ++i) {
     944     1464172 :     unsigned POld = OldPressureVec[i];
     945     1464172 :     unsigned PNew = NewPressureVec[i];
     946      732086 :     int PDiff = (int)PNew - (int)POld;
     947      732086 :     if (!PDiff) // No change in this set in the common case.
     948             :       continue;
     949             :     // Only consider change beyond the limit.
     950       85632 :     unsigned Limit = RCI->getRegPressureSetLimit(i);
     951       85632 :     if (!LiveThruPressureVec.empty())
     952      170946 :       Limit += LiveThruPressureVec[i];
     953             : 
     954       85632 :     if (Limit > POld) {
     955       75481 :       if (Limit > PNew)
     956             :         PDiff = 0;            // Under the limit
     957             :       else
     958         525 :         PDiff = PNew - Limit; // Just exceeded limit.
     959       10151 :     } else if (Limit > PNew)
     960        1621 :       PDiff = Limit - POld;   // Just obeyed limit.
     961             : 
     962       10676 :     if (PDiff) {
     963        8660 :       Delta.Excess = PressureChange(i);
     964        8660 :       Delta.Excess.setUnitInc(PDiff);
     965             :       break;
     966             :     }
     967             :   }
     968      105464 : }
     969             : 
     970             : /// Find the max change in max pressure that either surpasses a critical PSet
     971             : /// limit or exceeds the current MaxPressureLimit.
     972             : ///
     973             : /// FIXME: comparing each element of the old and new MaxPressure vectors here is
     974             : /// silly. It's done now to demonstrate the concept but will go away with a
     975             : /// RegPressureTracker API change to work with pressure differences.
     976      105464 : static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec,
     977             :                                     ArrayRef<unsigned> NewMaxPressureVec,
     978             :                                     ArrayRef<PressureChange> CriticalPSets,
     979             :                                     ArrayRef<unsigned> MaxPressureLimit,
     980             :                                     RegPressureDelta &Delta) {
     981      105464 :   Delta.CriticalMax = PressureChange();
     982      105464 :   Delta.CurrentMax = PressureChange();
     983             : 
     984      105464 :   unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
     985      832197 :   for (unsigned i = 0, e = OldMaxPressureVec.size(); i < e; ++i) {
     986     1468308 :     unsigned POld = OldMaxPressureVec[i];
     987     1468308 :     unsigned PNew = NewMaxPressureVec[i];
     988      734154 :     if (PNew == POld) // No change in this set in the common case.
     989             :       continue;
     990             : 
     991       53891 :     if (!Delta.CriticalMax.isValid()) {
     992       85824 :       while (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() < i)
     993          29 :         ++CritIdx;
     994             : 
     995       85737 :       if (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() == i) {
     996       31808 :         int PDiff = (int)PNew - (int)CriticalPSets[CritIdx].getUnitInc();
     997       15904 :         if (PDiff > 0) {
     998        9936 :           Delta.CriticalMax = PressureChange(i);
     999        9936 :           Delta.CriticalMax.setUnitInc(PDiff);
    1000             :         }
    1001             :       }
    1002             :     }
    1003             :     // Find the first increase above MaxPressureLimit.
    1004             :     // (Ignores negative MDiff).
    1005      107782 :     if (!Delta.CurrentMax.isValid() && PNew > MaxPressureLimit[i]) {
    1006        8744 :       Delta.CurrentMax = PressureChange(i);
    1007       17488 :       Delta.CurrentMax.setUnitInc(PNew - POld);
    1008        8744 :       if (CritIdx == CritEnd || Delta.CriticalMax.isValid())
    1009             :         break;
    1010             :     }
    1011             :   }
    1012      105464 : }
    1013             : 
    1014             : /// Record the upward impact of a single instruction on current register
    1015             : /// pressure. Unlike the advance/recede pressure tracking interface, this does
    1016             : /// not discover live in/outs.
    1017             : ///
    1018             : /// This is intended for speculative queries. It leaves pressure inconsistent
    1019             : /// with the current position, so must be restored by the caller.
    1020     2431800 : void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
    1021             :   assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
    1022             : 
    1023     2431800 :   SlotIndex SlotIdx;
    1024     2431800 :   if (RequireIntervals)
    1025     7295400 :     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
    1026             : 
    1027             :   // Account for register pressure similar to RegPressureTracker::recede().
    1028     4863600 :   RegisterOperands RegOpers;
    1029     2431800 :   RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true);
    1030             :   assert(RegOpers.DeadDefs.size() == 0);
    1031     2431800 :   if (TrackLaneMasks)
    1032     2411223 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
    1033       20577 :   else if (RequireIntervals)
    1034       20577 :     RegOpers.detectDeadDefs(*MI, *LIS);
    1035             : 
    1036             :   // Boost max pressure for all dead defs together.
    1037             :   // Since CurrSetPressure and MaxSetPressure
    1038     2431800 :   bumpDeadDefs(RegOpers.DeadDefs);
    1039             : 
    1040             :   // Kill liveness at live defs.
    1041     9406588 :   for (const RegisterMaskPair &P : RegOpers.Defs) {
    1042     2111188 :     unsigned Reg = P.RegUnit;
    1043     2111188 :     LaneBitmask LiveLanes = LiveRegs.contains(Reg);
    1044     2111188 :     LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg);
    1045     2111188 :     LaneBitmask DefLanes = P.LaneMask;
    1046     6333564 :     LaneBitmask LiveAfter = (LiveLanes & ~DefLanes) | UseLanes;
    1047     2111188 :     decreaseRegPressure(Reg, LiveLanes, LiveAfter);
    1048             :   }
    1049             :   // Generate liveness for uses.
    1050     9659777 :   for (const RegisterMaskPair &P : RegOpers.Uses) {
    1051     2364377 :     unsigned Reg = P.RegUnit;
    1052     2364377 :     LaneBitmask LiveLanes = LiveRegs.contains(Reg);
    1053     2364377 :     LaneBitmask LiveAfter = LiveLanes | P.LaneMask;
    1054     2364377 :     increaseRegPressure(Reg, LiveLanes, LiveAfter);
    1055             :   }
    1056     2431800 : }
    1057             : 
    1058             : /// Consider the pressure increase caused by traversing this instruction
    1059             : /// bottom-up. Find the pressure set with the most change beyond its pressure
    1060             : /// limit based on the tracker's current pressure, and return the change in
    1061             : /// number of register units of that pressure set introduced by this
    1062             : /// instruction.
    1063             : ///
    1064             : /// This assumes that the current LiveOut set is sufficient.
    1065             : ///
    1066             : /// This is expensive for an on-the-fly query because it calls
    1067             : /// bumpUpwardPressure to recompute the pressure sets based on current
    1068             : /// liveness. This mainly exists to verify correctness, e.g. with
    1069             : /// -verify-misched. getUpwardPressureDelta is the fast version of this query
    1070             : /// that uses the per-SUnit cache of the PressureDiff.
    1071       20577 : void RegPressureTracker::
    1072             : getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
    1073             :                           RegPressureDelta &Delta,
    1074             :                           ArrayRef<PressureChange> CriticalPSets,
    1075             :                           ArrayRef<unsigned> MaxPressureLimit) {
    1076             :   // Snapshot Pressure.
    1077             :   // FIXME: The snapshot heap space should persist. But I'm planning to
    1078             :   // summarize the pressure effect so we don't need to snapshot at all.
    1079       41154 :   std::vector<unsigned> SavedPressure = CurrSetPressure;
    1080       41154 :   std::vector<unsigned> SavedMaxPressure = P.MaxSetPressure;
    1081             : 
    1082       20577 :   bumpUpwardPressure(MI);
    1083             : 
    1084       82308 :   computeExcessPressureDelta(SavedPressure, CurrSetPressure, Delta, RCI,
    1085             :                              LiveThruPressure);
    1086       61731 :   computeMaxPressureDelta(SavedMaxPressure, P.MaxSetPressure, CriticalPSets,
    1087             :                           MaxPressureLimit, Delta);
    1088             :   assert(Delta.CriticalMax.getUnitInc() >= 0 &&
    1089             :          Delta.CurrentMax.getUnitInc() >= 0 && "cannot decrease max pressure");
    1090             : 
    1091             :   // Restore the tracker's state.
    1092       41154 :   P.MaxSetPressure.swap(SavedMaxPressure);
    1093       41154 :   CurrSetPressure.swap(SavedPressure);
    1094             : 
    1095             : #ifndef NDEBUG
    1096             :   if (!PDiff)
    1097             :     return;
    1098             : 
    1099             :   // Check if the alternate algorithm yields the same result.
    1100             :   RegPressureDelta Delta2;
    1101             :   getUpwardPressureDelta(MI, *PDiff, Delta2, CriticalPSets, MaxPressureLimit);
    1102             :   if (Delta != Delta2) {
    1103             :     dbgs() << "PDiff: ";
    1104             :     PDiff->dump(*TRI);
    1105             :     dbgs() << "DELTA: " << *MI;
    1106             :     if (Delta.Excess.isValid())
    1107             :       dbgs() << "Excess1 " << TRI->getRegPressureSetName(Delta.Excess.getPSet())
    1108             :              << " " << Delta.Excess.getUnitInc() << "\n";
    1109             :     if (Delta.CriticalMax.isValid())
    1110             :       dbgs() << "Critic1 " << TRI->getRegPressureSetName(Delta.CriticalMax.getPSet())
    1111             :              << " " << Delta.CriticalMax.getUnitInc() << "\n";
    1112             :     if (Delta.CurrentMax.isValid())
    1113             :       dbgs() << "CurrMx1 " << TRI->getRegPressureSetName(Delta.CurrentMax.getPSet())
    1114             :              << " " << Delta.CurrentMax.getUnitInc() << "\n";
    1115             :     if (Delta2.Excess.isValid())
    1116             :       dbgs() << "Excess2 " << TRI->getRegPressureSetName(Delta2.Excess.getPSet())
    1117             :              << " " << Delta2.Excess.getUnitInc() << "\n";
    1118             :     if (Delta2.CriticalMax.isValid())
    1119             :       dbgs() << "Critic2 " << TRI->getRegPressureSetName(Delta2.CriticalMax.getPSet())
    1120             :              << " " << Delta2.CriticalMax.getUnitInc() << "\n";
    1121             :     if (Delta2.CurrentMax.isValid())
    1122             :       dbgs() << "CurrMx2 " << TRI->getRegPressureSetName(Delta2.CurrentMax.getPSet())
    1123             :              << " " << Delta2.CurrentMax.getUnitInc() << "\n";
    1124             :     llvm_unreachable("RegP Delta Mismatch");
    1125             :   }
    1126             : #endif
    1127       20577 : }
    1128             : 
    1129             : /// This is the fast version of querying register pressure that does not
    1130             : /// directly depend on current liveness.
    1131             : ///
    1132             : /// @param Delta captures information needed for heuristics.
    1133             : ///
    1134             : /// @param CriticalPSets Are the pressure sets that are known to exceed some
    1135             : /// limit within the region, not necessarily at the current position.
    1136             : ///
    1137             : /// @param MaxPressureLimit Is the max pressure within the region, not
    1138             : /// necessarily at the current position.
    1139     7692087 : void RegPressureTracker::
    1140             : getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff,
    1141             :                        RegPressureDelta &Delta,
    1142             :                        ArrayRef<PressureChange> CriticalPSets,
    1143             :                        ArrayRef<unsigned> MaxPressureLimit) const {
    1144     7692087 :   unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
    1145     5676307 :   for (PressureDiff::const_iterator
    1146    15384174 :          PDiffI = PDiff.begin(), PDiffE = PDiff.end();
    1147    13368394 :        PDiffI != PDiffE && PDiffI->isValid(); ++PDiffI) {
    1148             : 
    1149    11352614 :     unsigned PSetID = PDiffI->getPSet();
    1150     5676307 :     unsigned Limit = RCI->getRegPressureSetLimit(PSetID);
    1151    11352614 :     if (!LiveThruPressure.empty())
    1152    11352614 :       Limit += LiveThruPressure[PSetID];
    1153             : 
    1154    11352614 :     unsigned POld = CurrSetPressure[PSetID];
    1155    11352614 :     unsigned MOld = P.MaxSetPressure[PSetID];
    1156     5676307 :     unsigned MNew = MOld;
    1157             :     // Ignore DeadDefs here because they aren't captured by PressureChange.
    1158     5676307 :     unsigned PNew = POld + PDiffI->getUnitInc();
    1159             :     assert((PDiffI->getUnitInc() >= 0) == (PNew >= POld)
    1160             :            && "PSet overflow/underflow");
    1161     5676307 :     if (PNew > MOld)
    1162     2400298 :       MNew = PNew;
    1163             :     // Check if current pressure has exceeded the limit.
    1164     5676307 :     if (!Delta.Excess.isValid()) {
    1165     5571240 :       unsigned ExcessInc = 0;
    1166     5571240 :       if (PNew > Limit)
    1167      228858 :         ExcessInc = POld > Limit ? PNew - POld : PNew - Limit;
    1168     5342382 :       else if (POld > Limit)
    1169        5034 :         ExcessInc = Limit - POld;
    1170      233892 :       if (ExcessInc) {
    1171      233892 :         Delta.Excess = PressureChange(PSetID);
    1172      233892 :         Delta.Excess.setUnitInc(ExcessInc);
    1173             :       }
    1174             :     }
    1175             :     // Check if max pressure has exceeded a critical pressure set max.
    1176     5676307 :     if (MNew == MOld)
    1177     3276009 :       continue;
    1178     2400298 :     if (!Delta.CriticalMax.isValid()) {
    1179     3129713 :       while (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() < PSetID)
    1180       11940 :         ++CritIdx;
    1181             : 
    1182     3553768 :       if (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() == PSetID) {
    1183      638840 :         int CritInc = (int)MNew - (int)CriticalPSets[CritIdx].getUnitInc();
    1184      319420 :         if (CritInc > 0 && CritInc <= std::numeric_limits<int16_t>::max()) {
    1185      319420 :           Delta.CriticalMax = PressureChange(PSetID);
    1186      319420 :           Delta.CriticalMax.setUnitInc(CritInc);
    1187             :         }
    1188             :       }
    1189             :     }
    1190             :     // Check if max pressure has exceeded the current max.
    1191     4702508 :     if (!Delta.CurrentMax.isValid() && MNew > MaxPressureLimit[PSetID]) {
    1192      128912 :       Delta.CurrentMax = PressureChange(PSetID);
    1193      128912 :       Delta.CurrentMax.setUnitInc(MNew - MOld);
    1194             :     }
    1195             :   }
    1196     7692087 : }
    1197             : 
    1198             : /// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx).
    1199             : /// The query starts with a lane bitmask which gets lanes/bits removed for every
    1200             : /// use we find.
    1201      229405 : static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
    1202             :                                   SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
    1203             :                                   const MachineRegisterInfo &MRI,
    1204             :                                   const LiveIntervals *LIS) {
    1205      458810 :   const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
    1206      714801 :   for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
    1207      284914 :     if (MO.isUndef())
    1208          90 :       continue;
    1209      284824 :     const MachineInstr *MI = MO.getParent();
    1210      854472 :     SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
    1211      537000 :     if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) {
    1212       70307 :       unsigned SubRegIdx = MO.getSubReg();
    1213      140614 :       LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
    1214      140614 :       LastUseMask &= ~UseMask;
    1215       70307 :       if (LastUseMask.none())
    1216       28923 :         return LaneBitmask::getNone();
    1217             :     }
    1218             :   }
    1219      200482 :   return LastUseMask;
    1220             : }
    1221             : 
    1222           0 : LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit,
    1223             :                                                SlotIndex Pos) const {
    1224             :   assert(RequireIntervals);
    1225           0 :   return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
    1226             :                               LaneBitmask::getAll(),
    1227           0 :       [](const LiveRange &LR, SlotIndex Pos) {
    1228             :         return LR.liveAt(Pos);
    1229           0 :       });
    1230             : }
    1231             : 
    1232      428492 : LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit,
    1233             :                                                  SlotIndex Pos) const {
    1234             :   assert(RequireIntervals);
    1235      856984 :   return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit,
    1236             :                               Pos.getBaseIndex(), LaneBitmask::getNone(),
    1237     1610752 :       [](const LiveRange &LR, SlotIndex Pos) {
    1238      683284 :         const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
    1239     2049852 :         return S != nullptr && S->end == Pos.getRegSlot();
    1240     2947836 :       });
    1241             : }
    1242             : 
    1243     1879863 : LaneBitmask RegPressureTracker::getLiveThroughAt(unsigned RegUnit,
    1244             :                                                  SlotIndex Pos) const {
    1245             :   assert(RequireIntervals);
    1246     3759726 :   return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
    1247             :                               LaneBitmask::getNone(),
    1248     3862234 :       [](const LiveRange &LR, SlotIndex Pos) {
    1249      424836 :         const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
    1250     1370089 :         return S != nullptr && S->start < Pos.getRegSlot(true) &&
    1251     2122279 :                S->end != Pos.getDeadSlot();
    1252     7570706 :       });
    1253             : }
    1254             : 
    1255             : /// Record the downward impact of a single instruction on current register
    1256             : /// pressure. Unlike the advance/recede pressure tracking interface, this does
    1257             : /// not discover live in/outs.
    1258             : ///
    1259             : /// This is intended for speculative queries. It leaves pressure inconsistent
    1260             : /// with the current position, so must be restored by the caller.
    1261      500942 : void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
    1262             :   assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
    1263             : 
    1264      500942 :   SlotIndex SlotIdx;
    1265      500942 :   if (RequireIntervals)
    1266     1502826 :     SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
    1267             : 
    1268             :   // Account for register pressure similar to RegPressureTracker::recede().
    1269     1001884 :   RegisterOperands RegOpers;
    1270      500942 :   RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, false);
    1271      500942 :   if (TrackLaneMasks)
    1272      416030 :     RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
    1273             : 
    1274      500942 :   if (RequireIntervals) {
    1275     1859359 :     for (const RegisterMaskPair &Use : RegOpers.Uses) {
    1276      356533 :       unsigned Reg = Use.RegUnit;
    1277      356533 :       LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
    1278      356533 :       if (LastUseMask.none())
    1279      283179 :         continue;
    1280             :       // The LastUseMask is queried from the liveness information of instruction
    1281             :       // which may be further down the schedule. Some lanes may actually not be
    1282             :       // last uses for the current position.
    1283             :       // FIXME: allow the caller to pass in the list of vreg uses that remain
    1284             :       // to be bottom-scheduled to avoid searching uses at each query.
    1285      229405 :       SlotIndex CurrIdx = getCurrSlot();
    1286             :       LastUseMask
    1287      229405 :         = findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
    1288      229405 :       if (LastUseMask.none())
    1289       28923 :         continue;
    1290             : 
    1291      200482 :       LaneBitmask LiveMask = LiveRegs.contains(Reg);
    1292      400964 :       LaneBitmask NewMask = LiveMask & ~LastUseMask;
    1293      200482 :       decreaseRegPressure(Reg, LiveMask, NewMask);
    1294             :     }
    1295             :   }
    1296             : 
    1297             :   // Generate liveness for defs.
    1298     2004225 :   for (const RegisterMaskPair &Def : RegOpers.Defs) {
    1299      501399 :     unsigned Reg = Def.RegUnit;
    1300      501399 :     LaneBitmask LiveMask = LiveRegs.contains(Reg);
    1301      501399 :     LaneBitmask NewMask = LiveMask | Def.LaneMask;
    1302      501399 :     increaseRegPressure(Reg, LiveMask, NewMask);
    1303             :   }
    1304             : 
    1305             :   // Boost pressure for all dead defs together.
    1306      500942 :   bumpDeadDefs(RegOpers.DeadDefs);
    1307      500942 : }
    1308             : 
    1309             : /// Consider the pressure increase caused by traversing this instruction
    1310             : /// top-down. Find the register class with the most change in its pressure limit
    1311             : /// based on the tracker's current pressure, and return the number of excess
    1312             : /// register units of that pressure set introduced by this instruction.
    1313             : ///
    1314             : /// This assumes that the current LiveIn set is sufficient.
    1315             : ///
    1316             : /// This is expensive for an on-the-fly query because it calls
    1317             : /// bumpDownwardPressure to recompute the pressure sets based on current
    1318             : /// liveness. We don't yet have a fast version of downward pressure tracking
    1319             : /// analogous to getUpwardPressureDelta.
    1320       84887 : void RegPressureTracker::
    1321             : getMaxDownwardPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta,
    1322             :                             ArrayRef<PressureChange> CriticalPSets,
    1323             :                             ArrayRef<unsigned> MaxPressureLimit) {
    1324             :   // Snapshot Pressure.
    1325      169774 :   std::vector<unsigned> SavedPressure = CurrSetPressure;
    1326      169774 :   std::vector<unsigned> SavedMaxPressure = P.MaxSetPressure;
    1327             : 
    1328       84887 :   bumpDownwardPressure(MI);
    1329             : 
    1330      339548 :   computeExcessPressureDelta(SavedPressure, CurrSetPressure, Delta, RCI,
    1331             :                              LiveThruPressure);
    1332      254661 :   computeMaxPressureDelta(SavedMaxPressure, P.MaxSetPressure, CriticalPSets,
    1333             :                           MaxPressureLimit, Delta);
    1334             :   assert(Delta.CriticalMax.getUnitInc() >= 0 &&
    1335             :          Delta.CurrentMax.getUnitInc() >= 0 && "cannot decrease max pressure");
    1336             : 
    1337             :   // Restore the tracker's state.
    1338      169774 :   P.MaxSetPressure.swap(SavedMaxPressure);
    1339      169774 :   CurrSetPressure.swap(SavedPressure);
    1340       84887 : }
    1341             : 
    1342             : /// Get the pressure of each PSet after traversing this instruction bottom-up.
    1343     2411223 : void RegPressureTracker::
    1344             : getUpwardPressure(const MachineInstr *MI,
    1345             :                   std::vector<unsigned> &PressureResult,
    1346             :                   std::vector<unsigned> &MaxPressureResult) {
    1347             :   // Snapshot pressure.
    1348     2411223 :   PressureResult = CurrSetPressure;
    1349     2411223 :   MaxPressureResult = P.MaxSetPressure;
    1350             : 
    1351     2411223 :   bumpUpwardPressure(MI);
    1352             : 
    1353             :   // Current pressure becomes the result. Restore current pressure.
    1354     4822446 :   P.MaxSetPressure.swap(MaxPressureResult);
    1355     4822446 :   CurrSetPressure.swap(PressureResult);
    1356     2411223 : }
    1357             : 
    1358             : /// Get the pressure of each PSet after traversing this instruction top-down.
    1359      416055 : void RegPressureTracker::
    1360             : getDownwardPressure(const MachineInstr *MI,
    1361             :                     std::vector<unsigned> &PressureResult,
    1362             :                     std::vector<unsigned> &MaxPressureResult) {
    1363             :   // Snapshot pressure.
    1364      416055 :   PressureResult = CurrSetPressure;
    1365      416055 :   MaxPressureResult = P.MaxSetPressure;
    1366             : 
    1367      416055 :   bumpDownwardPressure(MI);
    1368             : 
    1369             :   // Current pressure becomes the result. Restore current pressure.
    1370      832110 :   P.MaxSetPressure.swap(MaxPressureResult);
    1371      832110 :   CurrSetPressure.swap(PressureResult);
    1372      416055 : }

Generated by: LCOV version 1.13