LLVM  15.0.0git
AArch64SelectionDAGInfo.cpp
Go to the documentation of this file.
1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64TargetMachine.h"
14 using namespace llvm;
15 
16 #define DEBUG_TYPE "aarch64-selectiondag-info"
17 
19  SelectionDAG &DAG, const SDLoc &DL,
20  SDValue Chain, SDValue Dst,
21  SDValue SrcOrValue, SDValue Size,
22  Align Alignment, bool isVolatile,
23  MachinePointerInfo DstPtrInfo,
24  MachinePointerInfo SrcPtrInfo) const {
25 
26  // Get the constant size of the copy/set.
27  uint64_t ConstSize = 0;
28  if (auto *C = dyn_cast<ConstantSDNode>(Size))
29  ConstSize = C->getZExtValue();
30 
31  const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
33 
34  const auto MachineOpcode = [&]() {
35  switch (SDOpcode) {
37  return AArch64::MOPSMemorySetPseudo;
39  return AArch64::MOPSMemorySetTaggingPseudo;
41  return AArch64::MOPSMemoryCopyPseudo;
43  return AArch64::MOPSMemoryMovePseudo;
44  default:
45  llvm_unreachable("Unhandled MOPS ISD Opcode");
46  }
47  }();
48 
50  if (isVolatile)
52  if (!IsSet)
54 
56 
57  auto *DstOp =
58  MF.getMachineMemOperand(DstPtrInfo, Flags, ConstSize, Alignment);
59  auto *SrcOp =
60  MF.getMachineMemOperand(SrcPtrInfo, Flags, ConstSize, Alignment);
61 
62  if (IsSet) {
63  // Extend value to i64 if required
64  if (SrcOrValue.getValueType() != MVT::i64)
65  SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
66  SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
67  const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
68  MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
69  DAG.setNodeMemRefs(Node, {DstOp});
70  return SDValue(Node, 2);
71  } else {
72  SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
73  const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
74  MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
75  DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
76  return SDValue(Node, 3);
77  }
78 }
79 
81  SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
82  SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
83  MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
84  const AArch64Subtarget &STI =
86  if (STI.hasMOPS())
87  return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
88  Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
89  return SDValue();
90 }
91 
93  SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
94  SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
95  MachinePointerInfo DstPtrInfo) const {
96  const AArch64Subtarget &STI =
98 
99  if (STI.hasMOPS()) {
100  return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
101  Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
102  }
103  return SDValue();
104 }
105 
107  SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
108  SDValue Size, Align Alignment, bool isVolatile,
109  MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
110  const AArch64Subtarget &STI =
112  if (STI.hasMOPS()) {
113  return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
114  Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
115  }
116  return SDValue();
117 }
118 
119 static const int kSetTagLoopThreshold = 176;
120 
122  SDValue Chain, SDValue Ptr, uint64_t ObjSize,
123  const MachineMemOperand *BaseMemOperand,
124  bool ZeroData) {
126  unsigned ObjSizeScaled = ObjSize / 16;
127 
128  SDValue TagSrc = Ptr;
129  if (Ptr.getOpcode() == ISD::FrameIndex) {
130  int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
131  Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
132  // A frame index operand may end up as [SP + offset] => it is fine to use SP
133  // register as the tag source.
134  TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
135  }
136 
137  const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
138  const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
139 
140  SmallVector<SDValue, 8> OutChains;
141  unsigned OffsetScaled = 0;
142  while (OffsetScaled < ObjSizeScaled) {
143  if (ObjSizeScaled - OffsetScaled >= 2) {
144  SDValue AddrNode =
145  DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
146  SDValue St = DAG.getMemIntrinsicNode(
147  OpCode2, dl, DAG.getVTList(MVT::Other),
148  {Chain, TagSrc, AddrNode},
149  MVT::v4i64,
150  MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
151  OffsetScaled += 2;
152  OutChains.push_back(St);
153  continue;
154  }
155 
156  if (ObjSizeScaled - OffsetScaled > 0) {
157  SDValue AddrNode =
158  DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
159  SDValue St = DAG.getMemIntrinsicNode(
160  OpCode1, dl, DAG.getVTList(MVT::Other),
161  {Chain, TagSrc, AddrNode},
162  MVT::v2i64,
163  MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
164  OffsetScaled += 1;
165  OutChains.push_back(St);
166  }
167  }
168 
169  SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
170  return Res;
171 }
172 
174  SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
175  SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
176  uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
177  assert(ObjSize % 16 == 0);
178 
180  MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
181  DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
182 
183  bool UseSetTagRangeLoop =
184  kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
185  if (!UseSetTagRangeLoop)
186  return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
187  ZeroData);
188 
189  const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
190 
191  unsigned Opcode;
192  if (Addr.getOpcode() == ISD::FrameIndex) {
193  int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
194  Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
195  Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
196  } else {
197  Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
198  }
199  SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
200  SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
201 
202  DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
203  return SDValue(St, 2);
204 }
llvm::AArch64ISD::MOPS_MEMSET_TAGGING
@ MOPS_MEMSET_TAGGING
Definition: AArch64ISelLowering.h:458
llvm::AArch64ISD::NodeType
NodeType
Definition: AArch64ISelLowering.h:49
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1090
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9121
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:454
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2867
llvm::AArch64ISD::STG
@ STG
Definition: AArch64ISelLowering.h:447
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:454
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:6621
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:127
llvm::AArch64ISD::MOPS_MEMSET
@ MOPS_MEMSET
Definition: AArch64ISelLowering.h:457
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2061
EmitUnrolledSetTag
static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Ptr, uint64_t ObjSize, const MachineMemOperand *BaseMemOperand, bool ZeroData)
Definition: AArch64SelectionDAGInfo.cpp:121
llvm::SelectionDAG::getTargetFrameIndex
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:703
AArch64TargetMachine.h
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1125
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
llvm::AArch64ISD::MOPS_MEMMOVE
@ MOPS_MEMMOVE
Definition: AArch64ISelLowering.h:460
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
int
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
Definition: README.txt:536
llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemmove
SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memmove.
Definition: AArch64SelectionDAGInfo.cpp:106
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:441
llvm::AArch64ISD::STZ2G
@ STZ2G
Definition: AArch64ISelLowering.h:450
llvm::SelectionDAG::getMemIntrinsicNode
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
Definition: SelectionDAG.cpp:7542
Align
uint64_t Align
Definition: ELFObjHandler.cpp:81
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::AArch64SelectionDAGInfo::EmitTargetCodeForSetTag
SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo DstPtrInfo, bool ZeroData) const override
Definition: AArch64SelectionDAGInfo.cpp:173
llvm::MVT::v4i64
@ v4i64
Definition: MachineValueType.h:126
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:656
llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemset
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
Definition: AArch64SelectionDAGInfo.cpp:92
llvm::MVT::v2i64
@ v2i64
Definition: MachineValueType.h:124
uint64_t
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:39
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8851
llvm::MachineMemOperand::Flags
Flags
Flags values. These may be or'd together.
Definition: MachineMemOperand.h:130
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9559
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:9327
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
Definition: AArch64SelectionDAGInfo.cpp:80
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition: MachineMemOperand.h:138
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
llvm::ISD::FrameIndex
@ FrameIndex
Definition: ISDOpcodes.h:80
kSetTagLoopThreshold
static const int kSetTagLoopThreshold
Definition: AArch64SelectionDAGInfo.cpp:119
llvm::AArch64ISD::STZG
@ STZG
Definition: AArch64ISelLowering.h:448
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:137
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:136
llvm::AArch64ISD::ST2G
@ ST2G
Definition: AArch64ISelLowering.h:449
llvm::DstOp
Definition: MachineIRBuilder.h:65
llvm::SDValue::getOpcode
unsigned getOpcode() const
Definition: SelectionDAGNodes.h:1121
llvm::SelectionDAG::getTargetConstant
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:652
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:449
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
llvm::AArch64ISD::MOPS_MEMCOPY
@ MOPS_MEMCOPY
Definition: AArch64ISelLowering.h:459
llvm::AArch64SelectionDAGInfo::EmitMOPS
SDValue EmitMOPS(AArch64ISD::NodeType SDOpcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Definition: AArch64SelectionDAGInfo.cpp:18
llvm::SrcOp
Definition: MachineIRBuilder.h:126
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52