|
LLVM
4.0.0
|
#include "llvm/ADT/ArrayRef.h"#include "llvm/ADT/BitVector.h"#include "llvm/ADT/DenseMap.h"#include "llvm/ADT/iterator_range.h"#include "llvm/ADT/MapVector.h"#include "llvm/ADT/PriorityQueue.h"#include "llvm/ADT/SetVector.h"#include "llvm/ADT/SmallPtrSet.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/SmallVector.h"#include "llvm/ADT/Statistic.h"#include "llvm/Analysis/AliasAnalysis.h"#include "llvm/Analysis/MemoryLocation.h"#include "llvm/Analysis/ValueTracking.h"#include "llvm/CodeGen/DFAPacketizer.h"#include "llvm/CodeGen/LiveIntervalAnalysis.h"#include "llvm/CodeGen/MachineBasicBlock.h"#include "llvm/CodeGen/MachineDominators.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineFunctionPass.h"#include "llvm/CodeGen/MachineInstr.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineInstrBundle.h"#include "llvm/CodeGen/MachineLoopInfo.h"#include "llvm/CodeGen/MachineMemOperand.h"#include "llvm/CodeGen/MachineOperand.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/RegisterClassInfo.h"#include "llvm/CodeGen/RegisterPressure.h"#include "llvm/CodeGen/ScheduleDAG.h"#include "llvm/CodeGen/ScheduleDAGInstrs.h"#include "llvm/CodeGen/ScheduleDAGMutation.h"#include "llvm/IR/Attributes.h"#include "llvm/IR/DebugLoc.h"#include "llvm/MC/MCInstrItineraries.h"#include "llvm/PassAnalysisSupport.h"#include "llvm/PassRegistry.h"#include "llvm/PassSupport.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Debug.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Target/TargetInstrInfo.h"#include "llvm/Target/TargetRegisterInfo.h"#include "llvm/Target/TargetSubtargetInfo.h"#include <algorithm>#include <cassert>#include <climits>#include <cstdint>#include <deque>#include <functional>#include <iterator>#include <map>#include <tuple>#include <utility>#include <vector>Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "pipeliner" |
Functions | |
| STATISTIC (NumTrytoPipeline,"Number of loops that we attempt to pipeline") | |
| STATISTIC (NumPipelined,"Number of loops software pipelined") | |
| INITIALIZE_PASS_BEGIN (MachinePipeliner,"pipeliner","Modulo Software Pipelining", false, false) INITIALIZE_PASS_END(MachinePipeliner | |
| static void | getPhiRegs (MachineInstr &Phi, MachineBasicBlock *Loop, unsigned &InitVal, unsigned &LoopVal) |
| Return the register values for the operands of a Phi instruction. More... | |
| static unsigned | getInitPhiReg (MachineInstr &Phi, MachineBasicBlock *LoopBB) |
| Return the Phi register value that comes from the incoming block. More... | |
| static unsigned | getLoopPhiReg (MachineInstr &Phi, MachineBasicBlock *LoopBB) |
| Return the Phi register value that comes the the loop block. More... | |
| static bool | isSuccOrder (SUnit *SUa, SUnit *SUb) |
| Return true if SUb can be reached from SUa following the chain edges. More... | |
| static bool | isDependenceBarrier (MachineInstr &MI, AliasAnalysis *AA) |
| Return true if the instruction causes a chain between memory references before and after it. More... | |
| static void | getUnderlyingObjects (MachineInstr *MI, SmallVectorImpl< Value * > &Objs, const DataLayout &DL) |
| Return the underlying objects for the memory references of an instruction. More... | |
| static void | swapAntiDependences (std::vector< SUnit > &SUnits) |
| Swap all the anti dependences in the DAG. More... | |
| static bool | ignoreDependence (const SDep &D, bool isPred) |
| Return true for DAG nodes that we ignore when computing the cost functions. More... | |
| static bool | pred_L (SetVector< SUnit * > &NodeOrder, SmallSetVector< SUnit *, 8 > &Preds, const NodeSet *S=nullptr) |
| Compute the Pred_L(O) set, as defined in the paper. More... | |
| static bool | succ_L (SetVector< SUnit * > &NodeOrder, SmallSetVector< SUnit *, 8 > &Succs, const NodeSet *S=nullptr) |
| Compute the Succ_L(O) set, as defined in the paper. More... | |
| static bool | computePath (SUnit *Cur, SetVector< SUnit * > &Path, SetVector< SUnit * > &DestNodes, SetVector< SUnit * > &Exclude, SmallPtrSet< SUnit *, 8 > &Visited) |
| Return true if there is a path from the specified node to any of the nodes in DestNodes. More... | |
| template<class S1Ty , class S2Ty > | |
| static bool | isSubset (S1Ty &Set1, S2Ty &Set2) |
| Return true if Set1 is a subset of Set2. More... | |
| static void | computeLiveOuts (MachineFunction &MF, RegPressureTracker &RPTracker, NodeSet &NS) |
| Compute the live-out registers for the instructions in a node-set. More... | |
| static bool | isIntersect (SmallSetVector< SUnit *, 8 > &Set1, const NodeSet &Set2, SmallSetVector< SUnit *, 8 > &Result) |
| Return true if Set1 contains elements in Set2. More... | |
| static bool | hasDataDependence (SUnit *Inst1, SUnit *Inst2) |
| Return true if Inst1 defines a value that is used in Inst2. More... | |
| static void | replaceRegUsesAfterLoop (unsigned FromReg, unsigned ToReg, MachineBasicBlock *MBB, MachineRegisterInfo &MRI, LiveIntervals &LIS) |
| Replace all uses of FromReg that appear outside the specified basic block with ToReg. More... | |
| static bool | hasUseAfterLoop (unsigned Reg, MachineBasicBlock *BB, MachineRegisterInfo &MRI) |
| Return true if the register has a use that occurs outside the specified loop. More... | |
| static void | removePhis (MachineBasicBlock *BB, MachineBasicBlock *Incoming) |
| Remove the incoming block from the Phis in a basic block. More... | |
| static SUnit * | multipleIterations (SUnit *SU, SwingSchedulerDAG *DAG) |
| If an instruction has a use that spans multiple iterations, then return true. More... | |
Variables | |
| static cl::opt< bool > | EnableSWP ("enable-pipeliner", cl::Hidden, cl::init(true), cl::ZeroOrMore, cl::desc("Enable Software Pipelining")) |
| A command line option to turn software pipelining on or off. More... | |
| static cl::opt< bool > | EnableSWPOptSize ("enable-pipeliner-opt-size", cl::desc("Enable SWP at Os."), cl::Hidden, cl::init(false)) |
| A command line option to enable SWP at -Os. More... | |
| static cl::opt< int > | SwpMaxMii ("pipeliner-max-mii", cl::desc("Size limit for the the MII."), cl::Hidden, cl::init(27)) |
| A command line argument to limit minimum initial interval for pipelining. More... | |
| static cl::opt< int > | SwpMaxStages ("pipeliner-max-stages", cl::desc("Maximum stages allowed in the generated scheduled."), cl::Hidden, cl::init(3)) |
| A command line argument to limit the number of stages in the pipeline. More... | |
| static cl::opt< bool > | SwpPruneDeps ("pipeliner-prune-deps", cl::desc("Prune dependences between unrelated Phi nodes."), cl::Hidden, cl::init(true)) |
| A command line option to disable the pruning of chain dependences due to an unrelated Phi. More... | |
| static cl::opt< bool > | SwpPruneLoopCarried ("pipeliner-prune-loop-carried", cl::desc("Prune loop carried order dependences."), cl::Hidden, cl::init(true)) |
| A command line option to disable the pruning of loop carried order dependences. More... | |
| static cl::opt< int > | SwpLoopLimit ("pipeliner-max", cl::Hidden, cl::init(-1)) |
| static cl::opt< bool > | SwpIgnoreRecMII ("pipeliner-ignore-recmii", cl::ReallyHidden, cl::init(false), cl::ZeroOrMore, cl::desc("Ignore RecMII")) |
| pipeliner | |
| Modulo Software | Pipelining |
| Modulo Software | false |
| #define DEBUG_TYPE "pipeliner" |
Definition at line 120 of file MachinePipeliner.cpp.
|
static |
Compute the live-out registers for the instructions in a node-set.
The live-out registers are those that are defined in the node-set, but not used. Except for use operands of Phis.
Definition at line 1731 of file MachinePipeliner.cpp.
References llvm::RegPressureTracker::addLiveRegs(), llvm::SmallSet< T, N, C >::count(), llvm::SUnit::getInstr(), llvm::LaneBitmask::getNone(), llvm::MachineFunction::getRegInfo(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::MachineFunction::getSubtarget(), llvm::SmallSet< T, N, C >::insert(), llvm::MachineRegisterInfo::isAllocatable(), llvm::MachineInstr::isPHI(), llvm::MCRegisterInfo::DiffListIterator::isValid(), llvm::TargetRegisterInfo::isVirtualRegister(), MRI, llvm::MachineInstr::operands(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
|
static |
Return true if there is a path from the specified node to any of the nodes in DestNodes.
Keep track and return the nodes in any path.
Definition at line 1696 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SetVector< T, Vector, Set >::count(), llvm::SetVector< T, Vector, Set >::insert(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SUnit::isBoundaryNode(), llvm::SUnit::Preds, and llvm::SUnit::Succs.
|
static |
Return the Phi register value that comes from the incoming block.
Definition at line 970 of file MachinePipeliner.cpp.
References llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and i.
|
static |
Return the Phi register value that comes the the loop block.
Definition at line 978 of file MachinePipeliner.cpp.
References llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and i.
|
static |
Return the register values for the operands of a Phi instruction.
This function assume the instruction is a Phi.
Definition at line 954 of file MachinePipeliner.cpp.
References assert(), llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), i, and llvm::MachineInstr::isPHI().
|
static |
Return the underlying objects for the memory references of an instruction.
This function calls the code in ValueTracking, but first checks that the instruction has a memory operand.
Definition at line 1018 of file MachinePipeliner.cpp.
References llvm::GetUnderlyingObjects(), llvm::MachineMemOperand::getValue(), llvm::MachineInstr::hasOneMemOperand(), and llvm::MachineInstr::memoperands_begin().
Return true if Inst1 defines a value that is used in Inst2.
Definition at line 1994 of file MachinePipeliner.cpp.
References llvm::SDep::Data, and llvm::SUnit::Succs.
|
static |
Return true if the register has a use that occurs outside the specified loop.
Definition at line 2499 of file MachinePipeliner.cpp.
References E, I, llvm::MachineRegisterInfo::use_begin(), and llvm::MachineRegisterInfo::use_end().
Return true for DAG nodes that we ignore when computing the cost functions.
We ignore the back-edge recurrence in order to avoid unbounded recurison in the calculation of the ASAP, ALAP, etc functions.
Definition at line 1549 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SDep::getKind(), and llvm::SDep::isArtificial().
| INITIALIZE_PASS_BEGIN | ( | MachinePipeliner | , |
| "pipeliner" | , | ||
| "Modulo Software Pipelining" | , | ||
| false | , | ||
| false | |||
| ) |
|
static |
Return true if the instruction causes a chain between memory references before and after it.
Definition at line 1009 of file MachinePipeliner.cpp.
References llvm::MachineInstr::hasOrderedMemoryRef(), llvm::MachineInstr::hasUnmodeledSideEffects(), llvm::MachineInstr::isCall(), llvm::MachineInstr::isDereferenceableInvariantLoad(), and llvm::MachineInstr::mayLoad().
|
static |
Return true if Set1 contains elements in Set2.
The elements in common are returned in a different container.
Definition at line 1944 of file MachinePipeliner.cpp.
References llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty(), i, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), and llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size().
|
static |
Return true if Set1 is a subset of Set2.
Definition at line 1721 of file MachinePipeliner.cpp.
References llvm::MapVector< KeyT, ValueT, MapType, VectorType >::begin(), E, and I.
Return true if SUb can be reached from SUa following the chain edges.
Definition at line 986 of file MachinePipeliner.cpp.
References llvm::SmallPtrSetImpl< PtrType >::count(), llvm::SmallVectorBase::empty(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SDep::Order, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), SI, and llvm::SUnit::Succs.
If an instruction has a use that spans multiple iterations, then return true.
These instructions are characterized by having a back-ege to a Phi, which contains a reference to another Phi.
Definition at line 3597 of file MachinePipeliner.cpp.
References llvm::SDep::Order, and llvm::SUnit::Preds.
|
static |
Compute the Pred_L(O) set, as defined in the paper.
The set is defined as the predecessors of the elements of NodeOrder that are not also in NodeOrder.
Definition at line 1632 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SetVector< T, Vector, Set >::begin(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::SetVector< T, Vector, Set >::count(), E, llvm::SetVector< T, Vector, Set >::end(), I, ignoreDependence(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), and llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size().
|
static |
Remove the incoming block from the Phis in a basic block.
Definition at line 2976 of file MachinePipeliner.cpp.
References llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineInstr::isPHI(), and llvm::MachineInstr::RemoveOperand().
|
static |
Replace all uses of FromReg that appear outside the specified basic block with ToReg.
Definition at line 2481 of file MachinePipeliner.cpp.
References llvm::LiveIntervals::createEmptyInterval(), E, llvm::MachineInstr::getParent(), llvm::MachineOperand::getParent(), llvm::LiveIntervals::hasInterval(), I, MBB, llvm::MachineOperand::setReg(), llvm::MachineRegisterInfo::use_begin(), and llvm::MachineRegisterInfo::use_end().
| STATISTIC | ( | NumTrytoPipeline | , |
| "Number of loops that we attempt to pipeline" | |||
| ) |
| STATISTIC | ( | NumPipelined | , |
| "Number of loops software pipelined" | |||
| ) |
|
static |
Compute the Succ_L(O) set, as defined in the paper.
The set is defined as the successors of the elements of NodeOrder that are not also in NodeOrder.
Definition at line 1665 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SetVector< T, Vector, Set >::begin(), llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::SetVector< T, Vector, Set >::count(), E, llvm::SetVector< T, Vector, Set >::end(), I, ignoreDependence(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), and llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size().
|
static |
Swap all the anti dependences in the DAG.
That means it is no longer a DAG, but we do this to find the circuits, and then change them back.
Definition at line 1410 of file MachinePipeliner.cpp.
References llvm::SUnit::addPred(), llvm::SDep::Anti, llvm::SmallVectorTemplateCommon< T >::begin(), D, E, llvm::SmallVectorTemplateCommon< T >::end(), llvm::SDep::getLatency(), llvm::SDep::getReg(), llvm::SDep::getSUnit(), I, i, llvm::SUnit::Preds, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SUnit::removePred(), and llvm::SDep::setLatency().
|
static |
A command line option to turn software pipelining on or off.
|
static |
A command line option to enable SWP at -Os.
| Modulo Software false |
Definition at line 729 of file MachinePipeliner.cpp.
| pipeliner |
Definition at line 729 of file MachinePipeliner.cpp.
| Modulo Software Pipelining |
Definition at line 729 of file MachinePipeliner.cpp.
|
static |
|
static |
|
static |
A command line argument to limit minimum initial interval for pipelining.
|
static |
A command line argument to limit the number of stages in the pipeline.
|
static |
A command line option to disable the pruning of chain dependences due to an unrelated Phi.
1.8.6