18#define GET_SDNODE_DESC
19#include "ARMGenSDNodeInfo.inc"
23#define DEBUG_TYPE "arm-selectiondag-info"
27 cl::desc(
"Control conversion of memcpy to "
28 "Tail predicated loops (WLSTP)"),
31 "Don't convert memcpy to TP loop."),
33 "Always convert memcpy to TP loop."),
35 "Allow (may be subject to certain conditions) "
36 "conversion of memcpy to TP loop.")));
96 switch (
N->getOpcode()) {
99 case ARMISD::WIN__DBZCHK:
101 case ARMISD::WIN__CHKSTK:
103 case ARMISD::COPY_STRUCT_BYVAL:
107 case ARMISD::VMOVRRD:
109 case ARMISD::VMOVIMM:
123 bool CanTailCall = IsTailCall;
147 AEABILibcall = AEABI_MEMCPY;
154 AEABILibcall = AEABI_MEMMOVE;
161 AEABILibcall = AEABI_MEMSET;
163 AEABILibcall = AEABI_MEMCLR;
175 if ((
Align & 7) == 0)
176 AlignVariant = ALIGN8;
177 else if ((
Align & 3) == 0)
178 AlignVariant = ALIGN4;
180 AlignVariant = ALIGN1;
185 if (AEABILibcall == AEABI_MEMCLR) {
187 }
else if (AEABILibcall == AEABI_MEMSET) {
194 if (Src.getValueType().bitsGT(MVT::i32))
196 else if (Src.getValueType().bitsLT(MVT::i32))
201 Entry.IsSExt =
false;
202 Args.push_back(Entry);
208 static const RTLIB::Libcall FunctionImpls[4][3] = {
209 {RTLIB::AEABI_MEMCPY, RTLIB::AEABI_MEMCPY4, RTLIB::AEABI_MEMCPY8},
210 {RTLIB::AEABI_MEMMOVE, RTLIB::AEABI_MEMMOVE4, RTLIB::AEABI_MEMMOVE8},
211 {RTLIB::AEABI_MEMSET, RTLIB::AEABI_MEMSET4, RTLIB::AEABI_MEMSET8},
212 {RTLIB::AEABI_MEMCLR, RTLIB::AEABI_MEMCLR4, RTLIB::AEABI_MEMCLR8}};
214 RTLIB::Libcall NewLC = FunctionImpls[AEABILibcall][AlignVariant];
216 if (LCImpl == RTLIB::Unsupported)
229 std::pair<SDValue,SDValue> CallResult =
TLI->LowerCallTo(CLI);
231 return CallResult.second;
237 Align Alignment,
bool IsMemcpy) {
245 if (
F.hasOptNone() ||
F.hasOptSize())
251 if (!ConstantSize && Alignment >=
Align(4))
256 Subtarget.getMaxMemcpyTPInlineSizeThreshold())
266 Align Alignment = std::min(DstAlign, SrcAlign);
273 return DAG.
getNode(ARMISD::MEMCPYLOOP, dl, MVT::Other, Chain, Dst, Src,
278 if (Alignment <
Align(4)) {
285 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
292 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
293 uint64_t SizeVal = ConstantSize->getZExtValue();
294 if (!AlwaysInline && SizeVal >
Subtarget.getMaxInlineSizeThreshold())
296 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
298 unsigned BytesLeft = SizeVal & 3;
299 unsigned NumMemOps = SizeVal >> 2;
300 unsigned EmittedNumMemOps = 0;
305 const unsigned MaxLoadsInLDM =
Subtarget.isThumb1Only() ? 4 : 6;
308 uint64_t SrcOff = 0, DstOff = 0;
319 unsigned NumMEMCPYs = (NumMemOps + MaxLoadsInLDM - 1) / MaxLoadsInLDM;
323 if (NumMEMCPYs > 1 &&
Subtarget.hasMinSize()) {
325 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
330 for (
unsigned I = 0;
I != NumMEMCPYs; ++
I) {
333 unsigned NextEmittedNumMemOps = NumMemOps * (
I + 1) / NumMEMCPYs;
334 unsigned NumRegs = NextEmittedNumMemOps - EmittedNumMemOps;
336 Dst = DAG.
getNode(ARMISD::MEMCPY, dl, VTs, Chain, Dst, Src,
344 EmittedNumMemOps = NextEmittedNumMemOps;
351 auto getRemainingValueType = [](
unsigned BytesLeft) {
352 return (BytesLeft >= 2) ? MVT::i16 : MVT::i8;
354 auto getRemainingSize = [](
unsigned BytesLeft) {
355 return (BytesLeft >= 2) ? 2 : 1;
358 unsigned BytesLeftSave = BytesLeft;
361 VT = getRemainingValueType(BytesLeft);
362 VTSize = getRemainingSize(BytesLeft);
363 Loads[i] = DAG.
getLoad(VT, dl, Chain,
375 BytesLeft = BytesLeftSave;
377 VT = getRemainingValueType(BytesLeft);
378 VTSize = getRemainingSize(BytesLeft);
379 TFOps[i] = DAG.
getStore(Chain, dl, Loads[i],
395 Align Alignment = std::min(DstAlign, SrcAlign);
397 Alignment.value(), RTLIB::MEMMOVE, IsTailCall);
416 return DAG.
getNode(ARMISD::MEMSETLOOP, dl, MVT::Other, Chain, Dst, Src,
422 Alignment.value(), RTLIB::MEMSET, IsTailCall);
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
static bool shouldGenerateInlineTPLoop(const ARMSubtarget &Subtarget, const SelectionDAG &DAG, ConstantSDNode *ConstantSize, Align Alignment, bool IsMemcpy)
static cl::opt< TPLoop::MemTransfer > EnableMemtransferTPLoop("arm-memtransfer-tploop", cl::Hidden, cl::desc("Control conversion of memcpy to " "Tail predicated loops (WLSTP)"), cl::init(TPLoop::ForceDisabled), cl::values(clEnumValN(TPLoop::ForceDisabled, "force-disabled", "Don't convert memcpy to TP loop."), clEnumValN(TPLoop::ForceEnabled, "force-enabled", "Always convert memcpy to TP loop."), clEnumValN(TPLoop::Allow, "allow", "Allow (may be subject to certain conditions) " "conversion of memcpy to TP loop.")))
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
const char * getTargetNodeName(unsigned Opcode) const override
Returns the name of the given target-specific opcode, suitable for debug printing.
SDValue EmitSpecializedLibcall(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, RTLIB::Libcall LC, bool &IsTailCall) const
SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile, bool &IsTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memmove.
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile, bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override
Checks that the given target-specific node is valid. Aborts if it is not.
bool isTargetMemoryOpcode(unsigned Opcode) const override
Returns true if a node with the given target-specific opcode has a memory operand.
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
uint64_t getZExtValue() const
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDValue getValue(unsigned R) const
const char * getTargetNodeName(unsigned Opcode) const override
Returns the name of the given target-specific opcode, suitable for debug printing.
SelectionDAGGenTargetInfo(const SDNodeInfo &GenNodeInfo)
bool isTargetMemoryOpcode(unsigned Opcode) const override
Returns true if a node with the given target-specific opcode has a memory operand.
void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override
Checks that the given target-specific node is valid. Aborts if it is not.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
std::vector< ArgListEntry > ArgListTy
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
@ ADD
Simple integer binary arithmetic operators.
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
ArrayRef(const T &OneElt) -> ArrayRef< T >
This struct is a compact representation of a valid (non-zero power of two) alignment.
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)