34 #define DEBUG_TYPE "statepoint-lowering"
36 STATISTIC(NumSlotsAllocatedForStatepoints,
37 "Number of stack slots allocated for statepoints");
38 STATISTIC(NumOfStatepoints,
"Number of statepoint nodes encountered");
40 "Maximum number of stack slots required for a singe statepoint");
52 assert(PendingGCRelocateCalls.empty() &&
53 "Trying to visit statepoint before finished processing previous one");
55 NextSlotToAllocate = 0;
59 AllocatedStackSlots.
clear();
65 AllocatedStackSlots.
clear();
66 assert(PendingGCRelocateCalls.empty() &&
67 "cleared before statepoint sequence completed");
73 NumSlotsAllocatedForStatepoints++;
83 const size_t NumSlots = AllocatedStackSlots.
size();
84 assert(NextSlotToAllocate <= NumSlots &&
"Broken invariant");
90 for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
91 if (!AllocatedStackSlots.
test(NextSlotToAllocate)) {
94 AllocatedStackSlots.
set(NextSlotToAllocate);
104 const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
108 AllocatedStackSlots.
resize(AllocatedStackSlots.
size()+1,
true);
113 StatepointMaxSlotsRequired = std::max<unsigned long>(
126 if (LookUpDepth <= 0)
130 if (
const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
131 const auto &SpillMap =
134 auto It = SpillMap.find(Relocate->getDerivedPtr());
135 if (It == SpillMap.end())
142 if (
const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
148 if (
const PHINode *Phi = dyn_cast<PHINode>(Val)) {
151 for (
auto &IncomingValue : Phi->incoming_values()) {
157 if (MergedResult.
hasValue() && *MergedResult != *SpillSlot)
160 MergedResult = SpillSlot;
206 if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
217 const int LookUpDepth = 6;
225 auto SlotIt =
find(StatepointSlots, *Index);
226 assert(SlotIt != StatepointSlots.end() &&
227 "Value spilled to the unknown stack slot");
230 const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
263 for (
size_t i = 0, e = Ptrs.
size();
i < e;
i++) {
265 auto SeenIt = Seen.
find(SD);
267 if (SeenIt == Seen.
end()) {
295 SDValue ReturnValue, CallEndVal;
296 std::tie(ReturnValue, CallEndVal) =
335 static std::pair<SDValue, SDValue>
344 int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
360 "Bad spill: stack slot does not match!");
371 return std::make_pair(Loc, Chain);
396 }
else if (LiveInOnly) {
433 if (
auto *GFI = Builder.
GFI) {
442 if (Opt.hasValue()) {
444 "non gc managed base pointer found in statepoint");
449 if (Opt.hasValue()) {
451 "non gc managed derived pointer found in statepoint");
456 assert(SI.
Bases.empty() &&
"No gc specified, so cannot relocate pointers!");
457 assert(SI.
Ptrs.empty() &&
"No gc specified, so cannot relocate pointers!");
471 const bool LiveInDeopt =
474 auto isGCValue =[&](
const Value *V) {
484 if (!LiveInDeopt || isGCValue(V))
487 for (
unsigned i = 0;
i < SI.
Bases.size(); ++
i) {
502 const bool LiveInValue = LiveInDeopt && !isGCValue(V);
511 for (
unsigned i = 0;
i < SI.
Bases.size(); ++
i) {
547 SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
555 SpillMap.SlotMap[V] =
None;
607 std::tie(ReturnVal, CallNode) =
618 if (CallHasIncomingGlue) {
630 const bool IsGCTransition =
633 if (IsGCTransition) {
647 if (CallHasIncomingGlue)
655 Chain = GCTransitionStart.
getValue(0);
656 Glue = GCTransitionStart.
getValue(1);
671 unsigned NumCallRegArgs =
682 if (CallHasIncomingGlue)
683 RegMaskIt = CallNode->
op_end() - 2;
685 RegMaskIt = CallNode->
op_end() - 1;
694 "Unknown flag used");
698 Ops.
insert(Ops.
end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
714 SDNode *StatepointMCNode =
717 SDNode *SinkNode = StatepointMCNode;
723 if (IsGCTransition) {
744 SinkNode = GCTransitionStart.
getNode();
768 "anyregcc is not supported on statepoints!");
778 "GCStrategy does not expect to encounter statepoints");
807 SI.
Bases.push_back(Relocate->getBasePtr());
808 SI.
Ptrs.push_back(Relocate->getDerivedPtr());
826 if (!RetTy->
isVoidTy() && GCResult) {
841 RFV.getCopyToRegs(ReturnValue,
DAG,
getCurSDLoc(), Chain,
nullptr);
842 PendingExports.push_back(Chain);
859 bool VarArgDisallowed,
bool ForceVoidReturnTy) {
866 if (!VarArgDisallowed)
874 SI.
ID = SD.StatepointID.getValueOr(DefaultID);
898 void SelectionDAGBuilder::visitGCResult(
const GCResultInst &CI) {
913 cast<FunctionType>(CalleeType->
getElementType())->getReturnType();
923 void SelectionDAGBuilder::visitGCRelocate(
const GCRelocateInst &Relocate) {
934 assert(*IsManaged &&
"Non gc managed pointer relocated!");
941 auto SlotIt = SpillMap.find(DerivedPtr);
942 assert(SlotIt != SpillMap.end() &&
"Relocating not lowered gc value");
947 if (!DerivedPtrLocation) {
963 *DerivedPtrLocation));
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
const Instruction * getStatepoint() const
The statepoint with which this gc.relocate is associated.
void push_back(const T &Elt)
Represents calls to the gc.result intrinsic.
SDValue getValue(unsigned R) const
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
LLVMContext * getContext() const
STATISTIC(NumFunctions,"Total number of functions")
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block...
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it's base class for read only access to a gc.statepoint.
const TargetMachine & getTarget() const
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
BBTy * getParent() const
Get the basic block containing the call site.
void LowerStatepoint(ImmutableStatepoint Statepoint, const BasicBlock *EHPadBB=nullptr)
This class represents a function call, abstracting a target machine's calling convention.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
void LowerDeoptimizingReturn()
void setValue(const Value *V, SDValue NewN)
CallSiteTy::arg_iterator vm_state_begin() const
unsigned getNumOperands() const
Return the number of values used by this operation.
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Type * getElementType() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
SDValue getExternalSymbol(const char *Sym, EVT VT)
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
Value * getDerivedPtr() const
struct fuzzer::@269 Flags
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
MachineFunction & getMachineFunction() const
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Reg
All possible values of the reg field in the ModR/M byte.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
CallSiteTy getCallSite() const
Return the underlying CallSite.
void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - get or set the calling convention of the call.
bool useStatepoints() const
Returns true if this strategy is expecting the use of gc.statepoints, and false otherwise.
InstructionTy * getInstruction() const
SDValue getTargetFrameIndex(int FI, EVT VT)
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Type * getScalarType() const LLVM_READONLY
If this is a vector type, return the element type, otherwise return 'this'.
const DataLayout & getDataLayout() const
Class to represent pointers.
static const uint64_t DeoptBundleStatepointID
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
SDNode * getNode() const
get the SDNode which holds the desired result
CallSiteTy::arg_iterator gc_args_begin() const
void reserveStackSlot(int Offset)
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
FunctionType * getFunctionType() const
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
static std::pair< SDValue, SDValue > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
CallSiteTy::arg_iterator gc_args_end() const
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
bool isStackSlotAllocated(int Offset)
GCStrategy & getStrategy()
getStrategy - Return the GC strategy for the function.
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
static Type * getVoidTy(LLVMContext &C)
void clear()
Clear the memory usage of this object.
void verify()
Asserts if this statepoint is malformed.
TRAP - Trapping instruction.
SDLoc getCurSDLoc() const
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
EVT - Extended Value Type.
Type * getType() const
getType - Return the type of the instruction that generated this call site
bool isPointerTy() const
True if this is an instance of PointerType.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeSet AS)
Parse out statepoint directives from the function attributes present in AS.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
void setLocation(SDValue Val, SDValue Location)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
AttributeSet getAttributes() const
getAttributes/setAttributes - get or set the parameter attributes of the call.
void clear()
Clear all bits.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
InstrTy * getInstruction() const
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SLI)
Lower SLI into a STATEPOINT instruction.
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
CallSiteTy::arg_iterator vm_state_end() const
RegsForValue - This struct represents the registers (physical or virtual) that a particular set of va...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
uint64_t getID() const
Return the ID associated with this statepoint.
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
size_t size() const
Returns the number of bits in this bitvector.
GCFunctionInfo * GFI
GFI - Garbage collection metadata for the function.
unsigned getNumArgOperands() const
op_iterator op_begin() const
iterator insert(iterator I, T &&Elt)
bool test(unsigned Idx) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Represents a use of a SDNode.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
GCStrategy describes a garbage collector algorithm's code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
ImmutableCallSite - establish a view to a call site for examination.
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
iterator find(const KeyT &Val)
op_iterator op_end() const
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
getCopyFromRegs - If there was virtual register allocated for the value V emit CopyFromReg of the spe...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
FunctionLoweringInfo & FuncInfo
FuncInfo - Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
void scheduleRelocCall(const CallInst &RelocCall)
Record the fact that we expect to encounter a given gc_relocate before the next statepoint.
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
const TargetLowering & getTargetLoweringInfo() const
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
void relocCallVisited(const CallInst &RelocCall)
Remove this gc_relocate from the list we're expecting to see before the next statepoint.
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value * > &Bases, SmallVectorImpl< const Value * > &Ptrs, SmallVectorImpl< const GCRelocateInst * > &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
uint64_t getFlags() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
const BasicBlock * getParent() const
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
SmallVector< const Value *, 16 > Ptrs
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
Return true if this is 'void'.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.