LLVM 17.0.0git
AArch64SelectionDAGInfo.cpp
Go to the documentation of this file.
1//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64SelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
14using namespace llvm;
15
16#define DEBUG_TYPE "aarch64-selectiondag-info"
17
19 SelectionDAG &DAG, const SDLoc &DL,
20 SDValue Chain, SDValue Dst,
21 SDValue SrcOrValue, SDValue Size,
22 Align Alignment, bool isVolatile,
23 MachinePointerInfo DstPtrInfo,
24 MachinePointerInfo SrcPtrInfo) const {
25
26 // Get the constant size of the copy/set.
27 uint64_t ConstSize = 0;
28 if (auto *C = dyn_cast<ConstantSDNode>(Size))
29 ConstSize = C->getZExtValue();
30
31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
33
34 const auto MachineOpcode = [&]() {
35 switch (SDOpcode) {
37 return AArch64::MOPSMemorySetPseudo;
39 return AArch64::MOPSMemorySetTaggingPseudo;
41 return AArch64::MOPSMemoryCopyPseudo;
43 return AArch64::MOPSMemoryMovePseudo;
44 default:
45 llvm_unreachable("Unhandled MOPS ISD Opcode");
46 }
47 }();
48
50 if (isVolatile)
52 if (!IsSet)
54
56
57 auto *DstOp =
58 MF.getMachineMemOperand(DstPtrInfo, Flags, ConstSize, Alignment);
59 auto *SrcOp =
60 MF.getMachineMemOperand(SrcPtrInfo, Flags, ConstSize, Alignment);
61
62 if (IsSet) {
63 // Extend value to i64 if required
64 if (SrcOrValue.getValueType() != MVT::i64)
65 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
66 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
67 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
68 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
69 DAG.setNodeMemRefs(Node, {DstOp});
70 return SDValue(Node, 2);
71 } else {
72 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
73 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
74 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
75 DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
76 return SDValue(Node, 3);
77 }
78}
79
81 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
82 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
83 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
84 const AArch64Subtarget &STI =
86 if (STI.hasMOPS())
87 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
88 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
89 return SDValue();
90}
91
93 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
94 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
95 MachinePointerInfo DstPtrInfo) const {
96 const AArch64Subtarget &STI =
98
99 if (STI.hasMOPS()) {
100 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
101 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
102 }
103 return SDValue();
104}
105
107 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
108 SDValue Size, Align Alignment, bool isVolatile,
109 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
110 const AArch64Subtarget &STI =
112 if (STI.hasMOPS()) {
113 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
114 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
115 }
116 return SDValue();
117}
118
119static const int kSetTagLoopThreshold = 176;
120
122 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
123 const MachineMemOperand *BaseMemOperand,
124 bool ZeroData) {
126 unsigned ObjSizeScaled = ObjSize / 16;
127
128 SDValue TagSrc = Ptr;
129 if (Ptr.getOpcode() == ISD::FrameIndex) {
130 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
131 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
132 // A frame index operand may end up as [SP + offset] => it is fine to use SP
133 // register as the tag source.
134 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
135 }
136
137 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
138 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
139
140 SmallVector<SDValue, 8> OutChains;
141 unsigned OffsetScaled = 0;
142 while (OffsetScaled < ObjSizeScaled) {
143 if (ObjSizeScaled - OffsetScaled >= 2) {
144 SDValue AddrNode =
145 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
147 OpCode2, dl, DAG.getVTList(MVT::Other),
148 {Chain, TagSrc, AddrNode},
149 MVT::v4i64,
150 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
151 OffsetScaled += 2;
152 OutChains.push_back(St);
153 continue;
154 }
155
156 if (ObjSizeScaled - OffsetScaled > 0) {
157 SDValue AddrNode =
158 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
160 OpCode1, dl, DAG.getVTList(MVT::Other),
161 {Chain, TagSrc, AddrNode},
162 MVT::v2i64,
163 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
164 OffsetScaled += 1;
165 OutChains.push_back(St);
166 }
167 }
168
169 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
170 return Res;
171}
172
174 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
175 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
176 uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
177 assert(ObjSize % 16 == 0);
178
180 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
181 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
182
183 bool UseSetTagRangeLoop =
184 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
185 if (!UseSetTagRangeLoop)
186 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
187 ZeroData);
188
189 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
190
191 unsigned Opcode;
192 if (Addr.getOpcode() == ISD::FrameIndex) {
193 int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
194 Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
195 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
196 } else {
197 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
198 }
199 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
200 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
201
202 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
203 return SDValue(St, 2);
204}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Ptr, uint64_t ObjSize, const MachineMemOperand *BaseMemOperand, bool ZeroData)
static const int kSetTagLoopThreshold
uint64_t Addr
uint64_t Size
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
@ Flags
Definition: TextStubV5.cpp:93
SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memmove.
SDValue EmitMOPS(AArch64ISD::NodeType SDOpcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo DstPtrInfo, bool ZeroData) const override
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
EVT getValueType() const
Return the ValueType of the referenced return value.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:726
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:675
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
static constexpr TypeSize Fixed(ScalarTy ExactSize)
Definition: TypeSize.h:331
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
@ FrameIndex
Definition: ISDOpcodes.h:80
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
This class contains a discriminated union of information about pointers in memory operands,...