LLVM 24.0.0git
ARMSelectionDAGInfo.cpp
Go to the documentation of this file.
1//===-- ARMSelectionDAGInfo.cpp - ARM SelectionDAG Info -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the ARMSelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ARMSelectionDAGInfo.h"
17
18#define GET_SDNODE_DESC
19#include "ARMGenSDNodeInfo.inc"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "arm-selectiondag-info"
24
26 "arm-memtransfer-tploop", cl::Hidden,
27 cl::desc("Control conversion of memcpy to "
28 "Tail predicated loops (WLSTP)"),
31 "Don't convert memcpy to TP loop."),
32 clEnumValN(TPLoop::ForceEnabled, "force-enabled",
33 "Always convert memcpy to TP loop."),
35 "Allow (may be subject to certain conditions) "
36 "conversion of memcpy to TP loop.")));
37
40
84
85bool ARMSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
86 // These nodes don't have corresponding entries in *.td files yet.
87 if (Opcode >= ARMISD::FIRST_MEMORY_OPCODE &&
89 return true;
90
92}
93
95 const SDNode *N) const {
96 switch (N->getOpcode()) {
97 default:
98 break;
99 case ARMISD::WIN__DBZCHK:
100 // invalid number of results; expected 2, got 1
101 case ARMISD::WIN__CHKSTK:
102 // invalid number of results; expected 1, got 2
103 case ARMISD::COPY_STRUCT_BYVAL:
104 // invalid number of operands; expected 6, got 5
105 case ARMISD::MEMCPY:
106 // invalid number of operands; expected 5, got 4
107 case ARMISD::VMOVRRD:
108 // operand #0 must have type f64, but has type v1i64/v4f16/v8i8
109 case ARMISD::VMOVIMM:
110 // operand #0 must have type i32, but has type i16
111 return;
112 }
113
115}
116
117// Emit, if possible, a specialized version of the given Libcall. Typically this
118// means selecting the appropriately aligned version, but we also convert memset
119// of 0 into memclr.
121 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
122 SDValue Size, unsigned Align, RTLIB::Libcall LC, bool &IsTailCall) const {
123 bool CanTailCall = IsTailCall;
124 IsTailCall = false;
125
126 const ARMSubtarget &Subtarget =
128 const ARMTargetLowering *TLI = Subtarget.getTargetLowering();
129
130 // Only use the AEABI functions if they are available.
131 //
132 // Translate RTLIB::Libcall to AEABILibcall. We only do this in order to be
133 // able to translate memset to memclr and use the value to index the function
134 // name array.
135 enum {
136 AEABI_MEMCPY = 0,
137 AEABI_MEMMOVE,
138 AEABI_MEMSET,
139 AEABI_MEMCLR
140 } AEABILibcall;
141 switch (LC) {
142 case RTLIB::MEMCPY:
143 if (DAG.getLibcalls().getLibcallImpl(RTLIB::AEABI_MEMCPY) ==
144 RTLIB::Unsupported)
145 return SDValue();
146
147 AEABILibcall = AEABI_MEMCPY;
148 break;
149 case RTLIB::MEMMOVE:
150 if (DAG.getLibcalls().getLibcallImpl(RTLIB::AEABI_MEMMOVE) ==
151 RTLIB::Unsupported)
152 return SDValue();
153
154 AEABILibcall = AEABI_MEMMOVE;
155 break;
156 case RTLIB::MEMSET:
157 if (DAG.getLibcalls().getLibcallImpl(RTLIB::AEABI_MEMSET) ==
158 RTLIB::Unsupported)
159 return SDValue();
160
161 AEABILibcall = AEABI_MEMSET;
162 if (isNullConstant(Src))
163 AEABILibcall = AEABI_MEMCLR;
164 break;
165 default:
166 return SDValue();
167 }
168
169 // Choose the most-aligned libcall variant that we can
170 enum {
171 ALIGN1 = 0,
172 ALIGN4,
173 ALIGN8
174 } AlignVariant;
175 if ((Align & 7) == 0)
176 AlignVariant = ALIGN8;
177 else if ((Align & 3) == 0)
178 AlignVariant = ALIGN4;
179 else
180 AlignVariant = ALIGN1;
181
184 Args.emplace_back(Dst, IntPtrTy);
185 if (AEABILibcall == AEABI_MEMCLR) {
186 Args.emplace_back(Size, IntPtrTy);
187 } else if (AEABILibcall == AEABI_MEMSET) {
188 // Adjust parameters for memset, EABI uses format (ptr, size, value),
189 // GNU library uses (ptr, value, size)
190 // See RTABI section 4.3.4
191 Args.emplace_back(Size, IntPtrTy);
192
193 // Extend or truncate the argument to be an i32 value for the call.
194 if (Src.getValueType().bitsGT(MVT::i32))
195 Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
196 else if (Src.getValueType().bitsLT(MVT::i32))
197 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
198
201 Entry.IsSExt = false;
202 Args.push_back(Entry);
203 } else {
204 Args.emplace_back(Src, IntPtrTy);
205 Args.emplace_back(Size, IntPtrTy);
206 }
207
208 static const RTLIB::Libcall FunctionImpls[4][3] = {
209 {RTLIB::AEABI_MEMCPY, RTLIB::AEABI_MEMCPY4, RTLIB::AEABI_MEMCPY8},
210 {RTLIB::AEABI_MEMMOVE, RTLIB::AEABI_MEMMOVE4, RTLIB::AEABI_MEMMOVE8},
211 {RTLIB::AEABI_MEMSET, RTLIB::AEABI_MEMSET4, RTLIB::AEABI_MEMSET8},
212 {RTLIB::AEABI_MEMCLR, RTLIB::AEABI_MEMCLR4, RTLIB::AEABI_MEMCLR8}};
213
214 RTLIB::Libcall NewLC = FunctionImpls[AEABILibcall][AlignVariant];
215 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(NewLC);
216 if (LCImpl == RTLIB::Unsupported)
217 return SDValue();
218
220 CLI.setDebugLoc(dl)
221 .setChain(Chain)
225 DAG.getExternalSymbol(LCImpl, TLI->getPointerTy(DAG.getDataLayout())),
226 std::move(Args))
228 .setTailCall(CanTailCall);
229 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
230 IsTailCall = CLI.IsTailCall;
231 return CallResult.second;
232}
233
235 const SelectionDAG &DAG,
236 ConstantSDNode *ConstantSize,
237 Align Alignment, bool IsMemcpy) {
238 auto &F = DAG.getMachineFunction().getFunction();
240 return false;
242 return true;
243 // Do not generate inline TP loop if optimizations is disabled,
244 // or if optimization for size (-Os or -Oz) is on.
245 if (F.hasOptNone() || F.hasOptSize())
246 return false;
247 // If cli option is unset, for memset always generate inline TP.
248 // For memcpy, check some conditions
249 if (!IsMemcpy)
250 return true;
251 if (!ConstantSize && Alignment >= Align(4))
252 return true;
253 if (ConstantSize &&
254 ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold() &&
255 ConstantSize->getZExtValue() <
256 Subtarget.getMaxMemcpyTPInlineSizeThreshold())
257 return true;
258 return false;
259}
260
262 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
263 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
264 bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo,
265 MachinePointerInfo SrcPtrInfo) const {
266 Align Alignment = std::min(DstAlign, SrcAlign);
267 const ARMSubtarget &Subtarget =
270
271 if (Subtarget.hasMVEIntegerOps() &&
272 shouldGenerateInlineTPLoop(Subtarget, DAG, ConstantSize, Alignment, true))
273 return DAG.getNode(ARMISD::MEMCPYLOOP, dl, MVT::Other, Chain, Dst, Src,
274 DAG.getZExtOrTrunc(Size, dl, MVT::i32));
275
276 // Do repeated 4-byte loads and stores. To be improved.
277 // This requires 4-byte alignment.
278 if (Alignment < Align(4)) {
279 if (AlwaysInline) {
280 IsTailCall = false;
281 return SDValue();
282 }
283
284 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
285 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
286 }
287
288 // This requires the copy size to be a constant, preferably
289 // within a subtarget-specific limit.
290 if (!ConstantSize)
291 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
292 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
293 uint64_t SizeVal = ConstantSize->getZExtValue();
294 if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
295 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
296 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
297
298 unsigned BytesLeft = SizeVal & 3;
299 unsigned NumMemOps = SizeVal >> 2;
300 unsigned EmittedNumMemOps = 0;
301 EVT VT = MVT::i32;
302 unsigned VTSize = 4;
303 unsigned i = 0;
304 // Emit a maximum of 4 loads in Thumb1 since we have fewer registers
305 const unsigned MaxLoadsInLDM = Subtarget.isThumb1Only() ? 4 : 6;
306 SDValue TFOps[6];
307 SDValue Loads[6];
308 uint64_t SrcOff = 0, DstOff = 0;
309
310 // FIXME: We should invent a VMEMCPY pseudo-instruction that lowers to
311 // VLDM/VSTM and make this code emit it when appropriate. This would reduce
312 // pressure on the general purpose registers. However this seems harder to map
313 // onto the register allocator's view of the world.
314
315 // The number of MEMCPY pseudo-instructions to emit. We use up to
316 // MaxLoadsInLDM registers per mcopy, which will get lowered into ldm/stm
317 // later on. This is a lower bound on the number of MEMCPY operations we must
318 // emit.
319 unsigned NumMEMCPYs = (NumMemOps + MaxLoadsInLDM - 1) / MaxLoadsInLDM;
320
321 // Code size optimisation: do not inline memcpy if expansion results in
322 // more instructions than the library call.
323 if (NumMEMCPYs > 1 && Subtarget.hasMinSize()) {
324 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
325 Alignment.value(), RTLIB::MEMCPY, IsTailCall);
326 }
327
328 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other, MVT::Glue);
329
330 for (unsigned I = 0; I != NumMEMCPYs; ++I) {
331 // Evenly distribute registers among MEMCPY operations to reduce register
332 // pressure.
333 unsigned NextEmittedNumMemOps = NumMemOps * (I + 1) / NumMEMCPYs;
334 unsigned NumRegs = NextEmittedNumMemOps - EmittedNumMemOps;
335
336 Dst = DAG.getNode(ARMISD::MEMCPY, dl, VTs, Chain, Dst, Src,
337 DAG.getConstant(NumRegs, dl, MVT::i32));
338 Src = Dst.getValue(1);
339 Chain = Dst.getValue(2);
340
341 DstPtrInfo = DstPtrInfo.getWithOffset(NumRegs * VTSize);
342 SrcPtrInfo = SrcPtrInfo.getWithOffset(NumRegs * VTSize);
343
344 EmittedNumMemOps = NextEmittedNumMemOps;
345 }
346
347 if (BytesLeft == 0)
348 return Chain;
349
350 // Issue loads / stores for the trailing (1 - 3) bytes.
351 auto getRemainingValueType = [](unsigned BytesLeft) {
352 return (BytesLeft >= 2) ? MVT::i16 : MVT::i8;
353 };
354 auto getRemainingSize = [](unsigned BytesLeft) {
355 return (BytesLeft >= 2) ? 2 : 1;
356 };
357
358 unsigned BytesLeftSave = BytesLeft;
359 i = 0;
360 while (BytesLeft) {
361 VT = getRemainingValueType(BytesLeft);
362 VTSize = getRemainingSize(BytesLeft);
363 Loads[i] = DAG.getLoad(VT, dl, Chain,
364 DAG.getNode(ISD::ADD, dl, MVT::i32, Src,
365 DAG.getConstant(SrcOff, dl, MVT::i32)),
366 SrcPtrInfo.getWithOffset(SrcOff));
367 TFOps[i] = Loads[i].getValue(1);
368 ++i;
369 SrcOff += VTSize;
370 BytesLeft -= VTSize;
371 }
372 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ArrayRef(TFOps, i));
373
374 i = 0;
375 BytesLeft = BytesLeftSave;
376 while (BytesLeft) {
377 VT = getRemainingValueType(BytesLeft);
378 VTSize = getRemainingSize(BytesLeft);
379 TFOps[i] = DAG.getStore(Chain, dl, Loads[i],
380 DAG.getNode(ISD::ADD, dl, MVT::i32, Dst,
381 DAG.getConstant(DstOff, dl, MVT::i32)),
382 DstPtrInfo.getWithOffset(DstOff));
383 ++i;
384 DstOff += VTSize;
385 BytesLeft -= VTSize;
386 }
387 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ArrayRef(TFOps, i));
388}
389
391 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
392 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
393 bool &IsTailCall, MachinePointerInfo DstPtrInfo,
394 MachinePointerInfo SrcPtrInfo) const {
395 Align Alignment = std::min(DstAlign, SrcAlign);
396 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
397 Alignment.value(), RTLIB::MEMMOVE, IsTailCall);
398}
399
401 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
402 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
403 bool &IsTailCall, MachinePointerInfo DstPtrInfo) const {
404
405 const ARMSubtarget &Subtarget =
407
409
410 // Generate TP loop for llvm.memset
411 if (Subtarget.hasMVEIntegerOps() &&
412 shouldGenerateInlineTPLoop(Subtarget, DAG, ConstantSize, Alignment,
413 false)) {
414 Src = DAG.getSplatBuildVector(MVT::v16i8, dl,
415 DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src));
416 return DAG.getNode(ARMISD::MEMSETLOOP, dl, MVT::Other, Chain, Dst, Src,
417 DAG.getZExtOrTrunc(Size, dl, MVT::i32));
418 }
419
420 if (!AlwaysInline)
421 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
422 Alignment.value(), RTLIB::MEMSET, IsTailCall);
423
424 IsTailCall = false;
425 return SDValue();
426}
return SDValue()
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
static bool shouldGenerateInlineTPLoop(const ARMSubtarget &Subtarget, const SelectionDAG &DAG, ConstantSDNode *ConstantSize, Align Alignment, bool IsMemcpy)
static cl::opt< TPLoop::MemTransfer > EnableMemtransferTPLoop("arm-memtransfer-tploop", cl::Hidden, cl::desc("Control conversion of memcpy to " "Tail predicated loops (WLSTP)"), cl::init(TPLoop::ForceDisabled), cl::values(clEnumValN(TPLoop::ForceDisabled, "force-disabled", "Don't convert memcpy to TP loop."), clEnumValN(TPLoop::ForceEnabled, "force-enabled", "Always convert memcpy to TP loop."), clEnumValN(TPLoop::Allow, "allow", "Allow (may be subject to certain conditions) " "conversion of memcpy to TP loop.")))
This file a TargetTransformInfoImplBase conforming object specific to the ARM target machine.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define MAKE_CASE(V)
const char * getTargetNodeName(unsigned Opcode) const override
Returns the name of the given target-specific opcode, suitable for debug printing.
SDValue EmitSpecializedLibcall(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, RTLIB::Libcall LC, bool &IsTailCall) const
SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile, bool &IsTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memmove.
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile, bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override
Checks that the given target-specific node is valid. Aborts if it is not.
bool isTargetMemoryOpcode(unsigned Opcode) const override
Returns true if a node with the given target-specific opcode has a memory operand.
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
uint64_t getZExtValue() const
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDValue getValue(unsigned R) const
const char * getTargetNodeName(unsigned Opcode) const override
Returns the name of the given target-specific opcode, suitable for debug printing.
SelectionDAGGenTargetInfo(const SDNodeInfo &GenNodeInfo)
bool isTargetMemoryOpcode(unsigned Opcode) const override
Returns true if a node with the given target-specific opcode has a memory operand.
void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override
Checks that the given target-specific node is valid. Aborts if it is not.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
std::vector< ArgListEntry > ArgListTy
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:267
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
IntPtrTy
Definition InstrProf.h:82
ArrayRef(const T &OneElt) -> ArrayRef< T >
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)