16#define DEBUG_TYPE "aarch64-selectiondag-info"
20 cl::desc(
"Enable AArch64 SME memory operations "
21 "to lower to librt functions"),
28 Align Alignment,
bool isVolatile,
34 if (
auto *
C = dyn_cast<ConstantSDNode>(
Size))
35 ConstSize =
C->getZExtValue();
40 const auto MachineOpcode = [&]() {
43 return AArch64::MOPSMemorySetPseudo;
45 return AArch64::MOPSMemorySetTaggingPseudo;
47 return AArch64::MOPSMemoryCopyPseudo;
49 return AArch64::MOPSMemoryMovePseudo;
68 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
74 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
96 Args.push_back(DstEntry);
100 case RTLIB::MEMCPY: {
105 Args.push_back(Entry);
108 case RTLIB::MEMMOVE: {
113 Args.push_back(Entry);
116 case RTLIB::MEMSET: {
122 Args.push_back(Entry);
132 Args.push_back(SizeEntry);
134 "Function name is not set");
139 TLI->getLibcallCallingConv(LC),
RetTy, Symbol, std::move(Args));
140 return TLI->LowerCallTo(CLI).second;
152 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
188 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
204 unsigned ObjSizeScaled = ObjSize / 16;
208 int FI = cast<FrameIndexSDNode>(
Ptr)->getIndex();
219 unsigned OffsetScaled = 0;
220 while (OffsetScaled < ObjSizeScaled) {
221 if (ObjSizeScaled - OffsetScaled >= 2) {
226 {Chain, TagSrc, AddrNode},
234 if (ObjSizeScaled - OffsetScaled > 0) {
239 {Chain, TagSrc, AddrNode},
255 assert(ObjSize % 16 == 0);
261 bool UseSetTagRangeLoop =
263 if (!UseSetTagRangeLoop)
267 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
271 int FI = cast<FrameIndexSDNode>(
Addr)->getIndex();
273 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
275 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Ptr, uint64_t ObjSize, const MachineMemOperand *BaseMemOperand, bool ZeroData)
static cl::opt< bool > LowerToSMERoutines("aarch64-lower-to-sme-routines", cl::Hidden, cl::desc("Enable AArch64 SME memory operations " "to lower to librt functions"), cl::init(true))
static const int kSetTagLoopThreshold
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memmove.
SDValue EmitMOPS(AArch64ISD::NodeType SDOpcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
SDValue EmitStreamingCompatibleMemLibCall(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, RTLIB::Libcall LC) const
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo DstPtrInfo, bool ZeroData) const override
const AArch64TargetLowering * getTargetLowering() const override
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Class to represent pointers.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
EVT getValueType() const
Return the ValueType of the referenced return value.
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getExternalSymbol(const char *Sym, EVT VT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
MachineFunction & getMachineFunction() const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
std::vector< ArgListEntry > ArgListTy
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static IntegerType * getInt32Ty(LLVMContext &C)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
This struct is a compact representation of a valid (non-zero power of two) alignment.
This class contains a discriminated union of information about pointers in memory operands,...
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setChain(SDValue InChain)