36 #define DEBUG_TYPE "pre-RA-sched"
38 STATISTIC(LoadsClustered,
"Number of loads clustered together");
45 cl::desc(
"Roughly estimate the number of cycles that 'long latency'"
46 "instructions take for targets with no itinerary"));
50 InstrItins(mf.getSubtarget().getInstrItineraryData()) {}
70 const SUnit *Addr =
nullptr;
75 assert((Addr ==
nullptr || Addr == &
SUnits[0]) &&
76 "SUnits std::vector reallocated on the fly!");
113 unsigned &PhysReg,
int &Cost) {
143 if (ExtraOper.getNode())
167 if (GlueDestNode == N)
return false;
191 "expected an unused glue value");
202 void ScheduleDAGSDNodes::ClusterNeighboringLoads(
SDNode *Node) {
215 bool Cluster =
false;
219 unsigned UseCount = 0;
221 I != E && UseCount < 100; ++
I, ++UseCount) {
223 if (User == Node || !Visited.
insert(User).second)
225 int64_t Offset1, Offset2;
231 if (O2SMap.
insert(std::make_pair(Offset1, Base)).second)
233 O2SMap.
insert(std::make_pair(Offset2, User));
235 if (Offset2 < Offset1)
246 std::sort(Offsets.
begin(), Offsets.
end());
250 unsigned NumLoads = 0;
251 int64_t BaseOff = Offsets[0];
252 SDNode *BaseLoad = O2SMap[BaseOff];
253 Loads.push_back(BaseLoad);
254 for (
unsigned i = 1, e = Offsets.
size(); i != e; ++i) {
255 int64_t Offset = Offsets[i];
259 Loads.push_back(Load);
270 if (
AddGlue(Lead, InGlue,
true, DAG))
272 for (
unsigned I = 1, E = Loads.size();
I != E; ++
I) {
273 bool OutGlue =
I < E - 1;
278 if (
AddGlue(Load, InGlue, OutGlue, DAG)) {
284 else if (!OutGlue && InGlue.
getNode())
291 void ScheduleDAGSDNodes::ClusterNodes() {
301 ClusterNeighboringLoads(Node);
305 void ScheduleDAGSDNodes::BuildSchedUnits() {
309 unsigned NumNodes = 0;
320 SUnits.reserve(NumNodes * 2);
329 while (!Worklist.
empty()) {
334 if (Visited.
insert(Op.getNode()).second)
354 assert(N->
getNodeId() == -1 &&
"Node already inserted!");
366 bool HasGlueUse =
false;
369 if (GlueVal.isOperandOf(*UI)) {
371 assert(N->
getNodeId() == -1 &&
"Node already inserted!");
378 if (!HasGlueUse)
break;
394 assert(N->
getNodeId() == -1 &&
"Node already inserted!");
405 while (!CallSUnits.
empty()) {
419 void ScheduleDAGSDNodes::AddSchedEdges() {
426 for (
unsigned su = 0, e =
SUnits.size(); su != e; ++su) {
459 assert(OpSU &&
"Node has no SUnit!");
460 if (OpSU == SU)
continue;
463 assert(OpVT !=
MVT::Glue &&
"Glued nodes should be in same sunit!");
466 unsigned PhysReg = 0;
470 assert((PhysReg == 0 || !isChain) &&
471 "Chain dependence via physreg data?");
481 unsigned OpLatency = isChain ? 1 : OpSU->
Latency;
487 : SDep(OpSU, SDep::
Data, PhysReg);
489 if (!isChain && !UnitLatencies) {
524 void ScheduleDAGSDNodes::RegDefIter::InitNodeNumDefs() {
560 : SchedDAG(SD), Node(SU->getNode()), DefIdx(0), NodeNumDefs(0) {
568 for (;DefIdx < NodeNumDefs; ++DefIdx) {
586 assert(SU->
NumRegDefsLeft < USHRT_MAX &&
"overflow is ok but unexpected");
626 unsigned OpIdx, SDep& dep)
const{
646 Latency = (Latency > 1) ? Latency - 1 : 1;
653 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
655 dbgs() <<
"PHYS REG COPY\n";
664 while (!GluedNodes.
empty()) {
673 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
675 for (
unsigned i = 0, e =
Sequence.size(); i != e; i++) {
679 dbgs() <<
"**** NOOP ****\n";
691 for (
unsigned i = 0, e =
Sequence.size(); i != e; ++i)
694 assert(
Sequence.size() - Noops == ScheduledNodes &&
695 "The number of nodes scheduled doesn't match the expected number!");
712 for (
unsigned i = 0, e = DVs.
size(); i != e; ++i) {
713 if (DVs[i]->isInvalidated())
715 unsigned DVOrder = DVs[i]->getOrder();
716 if (!Order || DVOrder == ++Order) {
719 Orders.push_back(std::make_pair(DVOrder, DbgMI));
720 BB->
insert(InsertPos, DbgMI);
722 DVs[i]->setIsInvalidated();
736 if (!Order || !Seen.
insert(Order).second) {
753 Orders.push_back(std::make_pair(Order, std::prev(Emitter.
getInsertPos())));
757 void ScheduleDAGSDNodes::
762 if (
I->isCtrl())
continue;
763 if (
I->getSUnit()->CopyDstRC) {
766 assert(VRI != VRBaseMap.
end() &&
"Node emitted out of order - late");
770 EE = SU->
Succs.end(); II != EE; ++II) {
771 if (II->isCtrl())
continue;
778 .addReg(VRI->second);
781 assert(
I->getReg() &&
"Unknown physical register!");
783 bool isNew = VRBaseMap.
insert(std::make_pair(SU, VRBase)).second;
785 assert(isNew &&
"Node emitted out of order - early");
787 .addReg(
I->getReg());
810 for (; PDI != PDE; ++PDI) {
817 for (
unsigned i = 0, e =
Sequence.size(); i != e; i++) {
829 EmitPhysRegCopy(SU, CopyVRBaseMap, InsertPos);
836 while (!GluedNodes.
empty()) {
865 unsigned LastOrder = 0;
866 for (
unsigned i = 0, e = Orders.
size(); i != e && DI != DE; ++i) {
867 unsigned Order = Orders[i].first;
873 (*DI)->getOrder() >= LastOrder && (*DI)->getOrder() < Order; ++
DI) {
874 if ((*DI)->isInvalidated())
895 if (!(*DI)->isInvalidated())
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
value_iterator value_begin() const
void push_back(const T &Elt)
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
bool isCtrl() const
isCtrl - Shorthand for getKind() != SDep::Data.
void dump() const
Dump this node, for debugging.
void setNode(SDNode *N)
setNode - Assign the representative SDNode for this SUnit.
STATISTIC(NumFunctions,"Total number of functions")
bool hasDebugValues() const
Return true if there are any SDDbgValue nodes associated with this SelectionDAG.
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineSDNodes's memory reference descriptor list.
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
virtual void computeOperandLatency(SDNode *Def, SDNode *Use, unsigned OpIdx, SDep &dep) const
iterator getFirstTerminator()
getFirstTerminator - returns an iterator to the first terminator instruction of this basic block...
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
Describe properties that are true of each instruction in the target description file.
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool getHasDebugValue() const
Get this bit.
static unsigned CountResults(SDNode *Node)
CountResults - The results of target nodes have register or immediate operands first, then an optional chain, and optional flag operands (which do not go into the machine instrs.)
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
mmo_iterator memoperands_end() const
virtual bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, int64_t Offset1, int64_t Offset2, unsigned NumLoads) const
This is a used by the pre-regalloc scheduler to determine (in conjunction with areLoadsFromSameBasePt...
void setNodeId(int Id)
Set unique node id.
const TargetRegisterClass * CopyDstRC
unsigned getResNo() const
get the index which selects a specific result in the SDNode
SmallVector< SDep, 4 > Preds
MachineBasicBlock * getBlock()
getBlock - Return the current basic block.
static void CloneNodeWithValues(SDNode *N, SelectionDAG *DAG, ArrayRef< EVT > VTs, SDValue ExtraOper=SDValue())
COPY - Target-independent register copy.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
SDDbgInfo::DbgIterator DbgBegin()
void InitNumRegDefsLeft(SUnit *SU)
InitNumRegDefsLeft - Determine the # of regs defined by this node.
static void ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter, SmallVectorImpl< std::pair< unsigned, MachineInstr * > > &Orders, DenseMap< SDValue, unsigned > &VRBaseMap, unsigned Order)
ProcessSDDbgValues - Process SDDbgValues associated with this node.
MachineMemOperand - A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
mmo_iterator memoperands_begin() const
const HexagonInstrInfo * TII
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Regular data dependence (aka true-dependence).
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Reg
All possible values of the reg field in the ModR/M byte.
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
void dumpSchedule() const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
ScheduleDAGSDNodes(MachineFunction &mf)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
iterator_range< allnodes_iterator > allnodes()
load Combine Adjacent Loads
static cl::opt< int > HighLatencyCycles("sched-high-latency-cycles", cl::Hidden, cl::init(10), cl::desc("Roughly estimate the number of cycles that 'long latency'""instructions take for targets with no itinerary"))
unsigned getIROrder() const
Return the node ordering.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
static void CheckForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, unsigned &PhysReg, int &Cost)
CheckForPhysRegDependency - Check if the dependency between def and use of a specified operand is a p...
value_iterator value_end() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
virtual void adjustSchedDependency(SUnit *def, SUnit *use, SDep &dep) const
std::string getDAGName() const override
Return the basic block label.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
size_t size() const
size - Get the array size.
virtual unsigned getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr *MI, unsigned *PredCost=nullptr) const
Compute the instruction latency of a given instruction.
SUnit * Clone(SUnit *N)
Clone - Creates a clone of the specified SUnit.
const MachineBasicBlock * getParent() const
void dumpNode(const SUnit *SU) const override
TargetInstrInfo - Interface to description of machine instruction set.
SDDbgInfo::DbgIterator ByvalParmDbgEnd()
SDep - Scheduling dependency.
SDNode * getNode() const
get the SDNode which holds the desired result
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
Patchable call instruction - this instruction represents a call to a constant address, followed by a series of NOPs.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
std::vector< SUnit * > Sequence
The schedule. Null SUnit*'s represent noop instructions.
SUnit * newSUnit(SDNode *N)
NewSUnit - Creates a new SUnit and return a ptr to it.
void clearDAG()
clearDAG - clear the DAG state (between regions).
SDNode * getNode() const
getNode - Return the representative SDNode for this SUnit.
virtual bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1, int64_t &Offset2) const
This is used by the pre-regalloc scheduler to determine if two loads are loading from the same base a...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
const InstrItineraryData * InstrItins
This class provides iterator support for SDUse operands that use a specific SDNode.
virtual void computeLatency(SUnit *SU)
computeLatency - Compute node latency.
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD)
Get the debug values which reference the given SDNode.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
EVT - Extended Value Type.
Sched::Preference SchedulingPref
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
RegDefIter - In place iteration over the values defined by an SUnit.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
An unknown scheduling barrier.
std::string getFullName() const
getFullName - Return a formatted string to identify this block and its parent function.
static void RemoveUnusedGlue(SDNode *N, SelectionDAG *DAG)
TokenFactor - This node takes multiple tokens as input and produces a single token result...
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
virtual void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const
Insert a noop into the instruction stream at the specified point.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
SDDbgInfo::DbgIterator DbgEnd()
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool mayLoad() const
Return true if this instruction could possibly read memory.
Represents one node in the SelectionDAG.
SDDbgInfo::DbgIterator ByvalParmDbgBegin()
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
const TargetRegisterClass * getMinimalPhysRegClass(unsigned Reg, MVT VT=MVT::Other) const
getMinimalPhysRegClass - Returns the Register Class of a physical register of the given type...
op_iterator op_begin() const
static use_iterator use_end()
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
iterator_range< value_op_iterator > op_values() const
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
static bool isPassiveNode(SDNode *Node)
isPassiveNode - Return true if the node is a non-scheduled leaf.
TargetSubtargetInfo - Generic base class for all target subtargets.
int getCopyCost() const
getCopyCost - Return the cost of copying a value between two registers in this class.
unsigned VerifyScheduledDAG(bool isBottomUp)
VerifyScheduledDAG - Verify that all SUnits were scheduled and that their state is consistent...
Representation of each machine instruction.
const TargetRegisterInfo * TRI
void EmitNode(SDNode *Node, bool IsClone, bool IsCloned, DenseMap< SDValue, unsigned > &VRBaseMap)
EmitNode - Generate machine code for a node and needed dependencies.
static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter, DenseMap< SDValue, unsigned > &VRBaseMap, SmallVectorImpl< std::pair< unsigned, MachineInstr * > > &Orders, SmallSet< unsigned, 8 > &Seen)
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
RegDefIter(const SUnit *SU, const ScheduleDAGSDNodes *SD)
void BuildSchedGraph(AliasAnalysis *AA)
BuildSchedGraph - Build the SUnit graph from the selection dag that we are input. ...
void push_back(MachineInstr *MI)
unsigned short NumRegDefsLeft
iterator find(const KeyT &Val)
op_iterator op_end() const
virtual int getOperandLatency(const InstrItineraryData *ItinData, SDNode *DefNode, unsigned DefIdx, SDNode *UseNode, unsigned UseIdx) const
Kind getKind() const
getKind - Return an enum value representing the kind of the dependence.
void Run(SelectionDAG *dag, MachineBasicBlock *bb)
Run - perform scheduling.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const TargetInstrInfo * TII
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
EVT getValueType() const
Return the ValueType of the referenced return value.
const uint16_t * ImplicitDefs
MachineInstr * EmitDbgValue(SDDbgValue *SD, DenseMap< SDValue, unsigned > &VRBaseMap)
EmitDbgValue - Generate machine instruction for a dbg_value node.
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
MachineBasicBlock::iterator getInsertPos()
getInsertPos - Return the current insertion position.
virtual MachineBasicBlock * EmitSchedule(MachineBasicBlock::iterator &InsertPos)
EmitSchedule - Insert MachineInstrs into the MachineBasicBlock according to the order specified in Se...
SmallVector< SDep, 4 > Succs
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
BasicBlockListType::iterator iterator
const TargetLowering & getTargetLoweringInfo() const
virtual bool forceUnitLatencies() const
ForceUnitLatencies - Return true if all scheduling edges should be given a latency value of one...
MachineRegisterInfo & MRI
std::vector< SUnit > SUnits
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
bool isEmpty() const
Returns true if there are no itineraries.
virtual void Schedule()=0
Schedule - Order nodes according to selected style, filling in the Sequence member.
Function object to check whether the first component of a std::pair compares less than the first comp...
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
virtual bool isHighLatencyDef(int opc) const
Return true if this opcode has high latency to its result.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
static bool AddGlue(SDNode *N, SDValue Glue, bool AddGlue, SelectionDAG *DAG)
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
This file describes how to lower LLVM code to machine code.
void VerifyScheduledSequence(bool isBottomUp)
VerifyScheduledSequence - Verify that all SUnits are scheduled and consistent with the Sequence of sc...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.