LLVM  12.0.0git
X86SelectionDAGInfo.cpp
Go to the documentation of this file.
1 //===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "X86SelectionDAGInfo.h"
14 #include "X86ISelLowering.h"
15 #include "X86InstrInfo.h"
16 #include "X86RegisterInfo.h"
17 #include "X86Subtarget.h"
21 #include "llvm/IR/DerivedTypes.h"
22 
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "x86-selectiondag-info"
26 
27 bool X86SelectionDAGInfo::isBaseRegConflictPossible(
28  SelectionDAG &DAG, ArrayRef<MCPhysReg> ClobberSet) const {
29  // We cannot use TRI->hasBasePointer() until *after* we select all basic
30  // blocks. Legalization may introduce new stack temporaries with large
31  // alignment requirements. Fall back to generic code if there are any
32  // dynamic stack adjustments (hopefully rare) and the base pointer would
33  // conflict if we had to use it.
35  if (!MFI.hasVarSizedObjects() && !MFI.hasOpaqueSPAdjustment())
36  return false;
37 
38  const X86RegisterInfo *TRI = static_cast<const X86RegisterInfo *>(
40  Register BaseReg = TRI->getBaseRegister();
41  for (unsigned R : ClobberSet)
42  if (BaseReg == R)
43  return true;
44  return false;
45 }
46 
48  SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
49  SDValue Size, Align Alignment, bool isVolatile,
50  MachinePointerInfo DstPtrInfo) const {
51  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
52  const X86Subtarget &Subtarget =
54 
55 #ifndef NDEBUG
56  // If the base register might conflict with our physical registers, bail out.
57  const MCPhysReg ClobberSet[] = {X86::RCX, X86::RAX, X86::RDI,
59  assert(!isBaseRegConflictPossible(DAG, ClobberSet));
60 #endif
61 
62  // If to a segment-relative address space, use the default lowering.
63  if (DstPtrInfo.getAddrSpace() >= 256)
64  return SDValue();
65 
66  // If not DWORD aligned or size is more than the threshold, call the library.
67  // The libc version is likely to be faster for these cases. It can use the
68  // address value and run time information about the CPU.
69  if (Alignment < Align(4) || !ConstantSize ||
70  ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold()) {
71  // Check to see if there is a specialized entry-point for memory zeroing.
73 
74  if (const char *bzeroName = (ValC && ValC->isNullValue())
75  ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO)
76  : nullptr) {
77  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
78  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
79  Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
82  Entry.Node = Dst;
83  Entry.Ty = IntPtrTy;
84  Args.push_back(Entry);
85  Entry.Node = Size;
86  Args.push_back(Entry);
87 
89  CLI.setDebugLoc(dl)
90  .setChain(Chain)
92  DAG.getExternalSymbol(bzeroName, IntPtr),
93  std::move(Args))
94  .setDiscardResult();
95 
96  std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
97  return CallResult.second;
98  }
99 
100  // Otherwise have the target-independent code call memset.
101  return SDValue();
102  }
103 
104  uint64_t SizeVal = ConstantSize->getZExtValue();
105  SDValue InFlag;
106  EVT AVT;
107  SDValue Count;
108  ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Val);
109  unsigned BytesLeft = 0;
110  if (ValC) {
111  unsigned ValReg;
112  uint64_t Val = ValC->getZExtValue() & 255;
113 
114  // If the value is a constant, then we can potentially use larger sets.
115  if (Alignment > Align(2)) {
116  // DWORD aligned
117  AVT = MVT::i32;
118  ValReg = X86::EAX;
119  Val = (Val << 8) | Val;
120  Val = (Val << 16) | Val;
121  if (Subtarget.is64Bit() && Alignment > Align(8)) { // QWORD aligned
122  AVT = MVT::i64;
123  ValReg = X86::RAX;
124  Val = (Val << 32) | Val;
125  }
126  } else if (Alignment == Align(2)) {
127  // WORD aligned
128  AVT = MVT::i16;
129  ValReg = X86::AX;
130  Val = (Val << 8) | Val;
131  } else {
132  // Byte aligned
133  AVT = MVT::i8;
134  ValReg = X86::AL;
135  Count = DAG.getIntPtrConstant(SizeVal, dl);
136  }
137 
138  if (AVT.bitsGT(MVT::i8)) {
139  unsigned UBytes = AVT.getSizeInBits() / 8;
140  Count = DAG.getIntPtrConstant(SizeVal / UBytes, dl);
141  BytesLeft = SizeVal % UBytes;
142  }
143 
144  Chain = DAG.getCopyToReg(Chain, dl, ValReg, DAG.getConstant(Val, dl, AVT),
145  InFlag);
146  InFlag = Chain.getValue(1);
147  } else {
148  AVT = MVT::i8;
149  Count = DAG.getIntPtrConstant(SizeVal, dl);
150  Chain = DAG.getCopyToReg(Chain, dl, X86::AL, Val, InFlag);
151  InFlag = Chain.getValue(1);
152  }
153 
154  bool Use64BitRegs = Subtarget.isTarget64BitLP64();
155  Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RCX : X86::ECX,
156  Count, InFlag);
157  InFlag = Chain.getValue(1);
158  Chain = DAG.getCopyToReg(Chain, dl, Use64BitRegs ? X86::RDI : X86::EDI,
159  Dst, InFlag);
160  InFlag = Chain.getValue(1);
161 
163  SDValue Ops[] = { Chain, DAG.getValueType(AVT), InFlag };
164  Chain = DAG.getNode(X86ISD::REP_STOS, dl, Tys, Ops);
165 
166  if (BytesLeft) {
167  // Handle the last 1 - 7 bytes.
168  unsigned Offset = SizeVal - BytesLeft;
169  EVT AddrVT = Dst.getValueType();
170  EVT SizeVT = Size.getValueType();
171 
172  Chain =
173  DAG.getMemset(Chain, dl,
174  DAG.getNode(ISD::ADD, dl, AddrVT, Dst,
175  DAG.getConstant(Offset, dl, AddrVT)),
176  Val, DAG.getConstant(BytesLeft, dl, SizeVT), Alignment,
177  isVolatile, false, DstPtrInfo.getWithOffset(Offset));
178  }
179 
180  // TODO: Use a Tokenfactor, as in memcpy, instead of a single chain.
181  return Chain;
182 }
183 
184 /// Emit a single REP MOVS{B,W,D,Q} instruction.
185 static SDValue emitRepmovs(const X86Subtarget &Subtarget, SelectionDAG &DAG,
186  const SDLoc &dl, SDValue Chain, SDValue Dst,
187  SDValue Src, SDValue Size, MVT AVT) {
188  const bool Use64BitRegs = Subtarget.isTarget64BitLP64();
189  const unsigned CX = Use64BitRegs ? X86::RCX : X86::ECX;
190  const unsigned DI = Use64BitRegs ? X86::RDI : X86::EDI;
191  const unsigned SI = Use64BitRegs ? X86::RSI : X86::ESI;
192 
193  SDValue InFlag;
194  Chain = DAG.getCopyToReg(Chain, dl, CX, Size, InFlag);
195  InFlag = Chain.getValue(1);
196  Chain = DAG.getCopyToReg(Chain, dl, DI, Dst, InFlag);
197  InFlag = Chain.getValue(1);
198  Chain = DAG.getCopyToReg(Chain, dl, SI, Src, InFlag);
199  InFlag = Chain.getValue(1);
200 
202  SDValue Ops[] = {Chain, DAG.getValueType(AVT), InFlag};
203  return DAG.getNode(X86ISD::REP_MOVS, dl, Tys, Ops);
204 }
205 
206 /// Emit a single REP MOVSB instruction for a particular constant size.
207 static SDValue emitRepmovsB(const X86Subtarget &Subtarget, SelectionDAG &DAG,
208  const SDLoc &dl, SDValue Chain, SDValue Dst,
209  SDValue Src, uint64_t Size) {
210  return emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src,
211  DAG.getIntPtrConstant(Size, dl), MVT::i8);
212 }
213 
214 /// Returns the best type to use with repmovs depending on alignment.
215 static MVT getOptimalRepmovsType(const X86Subtarget &Subtarget,
216  uint64_t Align) {
217  assert((Align != 0) && "Align is normalized");
218  assert(isPowerOf2_64(Align) && "Align is a power of 2");
219  switch (Align) {
220  case 1:
221  return MVT::i8;
222  case 2:
223  return MVT::i16;
224  case 4:
225  return MVT::i32;
226  default:
227  return Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
228  }
229 }
230 
231 /// Returns a REP MOVS instruction, possibly with a few load/stores to implement
232 /// a constant size memory copy. In some cases where we know REP MOVS is
233 /// inefficient we return an empty SDValue so the calling code can either
234 /// generate a load/store sequence or call the runtime memcpy function.
236  SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl,
237  SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
238  unsigned Align, bool isVolatile, bool AlwaysInline,
239  MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) {
240 
241  /// TODO: Revisit next line: big copy with ERMSB on march >= haswell are very
242  /// efficient.
243  if (!AlwaysInline && Size > Subtarget.getMaxInlineSizeThreshold())
244  return SDValue();
245 
246  /// If we have enhanced repmovs we use it.
247  if (Subtarget.hasERMSB())
248  return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
249 
250  assert(!Subtarget.hasERMSB() && "No efficient RepMovs");
251  /// We assume runtime memcpy will do a better job for unaligned copies when
252  /// ERMS is not present.
253  if (!AlwaysInline && (Align & 3) != 0)
254  return SDValue();
255 
256  const MVT BlockType = getOptimalRepmovsType(Subtarget, Align);
257  const uint64_t BlockBytes = BlockType.getSizeInBits() / 8;
258  const uint64_t BlockCount = Size / BlockBytes;
259  const uint64_t BytesLeft = Size % BlockBytes;
260  SDValue RepMovs =
261  emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src,
262  DAG.getIntPtrConstant(BlockCount, dl), BlockType);
263 
264  /// RepMov can process the whole length.
265  if (BytesLeft == 0)
266  return RepMovs;
267 
268  assert(BytesLeft && "We have leftover at this point");
269 
270  /// In case we optimize for size we use repmovsb even if it's less efficient
271  /// so we can save the loads/stores of the leftover.
273  return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
274 
275  // Handle the last 1 - 7 bytes.
277  Results.push_back(RepMovs);
278  unsigned Offset = Size - BytesLeft;
279  EVT DstVT = Dst.getValueType();
280  EVT SrcVT = Src.getValueType();
281  Results.push_back(DAG.getMemcpy(
282  Chain, dl,
283  DAG.getNode(ISD::ADD, dl, DstVT, Dst, DAG.getConstant(Offset, dl, DstVT)),
284  DAG.getNode(ISD::ADD, dl, SrcVT, Src, DAG.getConstant(Offset, dl, SrcVT)),
285  DAG.getConstant(BytesLeft, dl, SizeVT), llvm::Align(Align), isVolatile,
286  /*AlwaysInline*/ true, /*isTailCall*/ false,
287  DstPtrInfo.getWithOffset(Offset), SrcPtrInfo.getWithOffset(Offset)));
288  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
289 }
290 
292  SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
293  SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
294  MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
295  // If to a segment-relative address space, use the default lowering.
296  if (DstPtrInfo.getAddrSpace() >= 256 || SrcPtrInfo.getAddrSpace() >= 256)
297  return SDValue();
298 
299  // If the base registers conflict with our physical registers, use the default
300  // lowering.
301  const MCPhysReg ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
303  if (isBaseRegConflictPossible(DAG, ClobberSet))
304  return SDValue();
305 
306  const X86Subtarget &Subtarget =
308 
309  /// Handle constant sizes,
310  if (ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size))
311  return emitConstantSizeRepmov(
312  DAG, Subtarget, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
313  Size.getValueType(), Alignment.value(), isVolatile, AlwaysInline,
314  DstPtrInfo, SrcPtrInfo);
315 
316  return SDValue();
317 }
static SDValue emitConstantSizeRepmov(SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT, unsigned Align, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
Returns a REP MOVS instruction, possibly with a few load/stores to implement a constant size memory c...
bool is64Bit() const
Is this x86_64? (disregarding specific ABI / programming model)
Definition: X86Subtarget.h:579
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
EVT getValueType() const
Return the ValueType of the referenced return value.
LLVM_NODISCARD std::enable_if_t< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type > dyn_cast(const Y &Val)
Definition: Casting.h:334
Repeat move, corresponds to X86::REP_MOVSx.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Function Alias Analysis Results
unsigned const TargetRegisterInfo * TRI
static SDValue emitRepmovs(const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, MVT AVT)
Emit a single REP MOVS{B,W,D,Q} instruction.
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
static MVT getOptimalRepmovsType(const X86Subtarget &Subtarget, uint64_t Align)
Returns the best type to use with repmovs depending on alignment.
SDValue getExternalSymbol(const char *Sym, EVT VT)
Function & getFunction()
Return the LLVM function that this machine code represents.
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
bool isTarget64BitLP64() const
Is this x86_64 with the LP64 programming model (standard AMD64, no x32)?
Definition: X86Subtarget.h:598
CallLoweringInfo & setChain(SDValue InChain)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:424
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:421
SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override
Emit target-specific code that performs a memcpy.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:223
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:236
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:789
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Repeat fill, corresponds to X86::REP_STOSx.
bool hasERMSB() const
Definition: X86Subtarget.h:708
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:497
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:170
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::vector< ArgListEntry > ArgListTy
Extended Value Type.
Definition: ValueTypes.h:35
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:315
This structure contains all information that is necessary for lowering calls.
This class contains a discriminated union of information about pointers in memory operands...
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
unsigned getMaxInlineSizeThreshold() const
Returns the maximum memset / memcpy size that still makes it profitable to inline the call...
Definition: X86Subtarget.h:559
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:52
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:427
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:223
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:883
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:718
MachinePointerInfo getWithOffset(int64_t O) const
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo) const override
Emit target-specific code that performs a memset.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo)
BlockType
Used as immediate MachineOperands for block signatures.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:426
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:644
uint32_t Size
Definition: Profile.cpp:46
SDValue getValue(unsigned R) const
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getValueType(EVT)
static SDValue emitRepmovsB(const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size)
Emit a single REP MOVSB instruction for a particular constant size.
static bool isVolatile(Instruction *Inst)
uint64_t getZExtValue() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
LLVMContext * getContext() const
Definition: SelectionDAG.h:431
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
This file describes how to lower LLVM code to machine code.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Register getBaseRegister() const