LLVM  8.0.0svn
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the TargetLowering class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/BitVector.h"
16 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/GlobalVariable.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCExpr.h"
32 #include "llvm/Support/KnownBits.h"
36 #include <cctype>
37 using namespace llvm;
38 
39 /// NOTE: The TargetMachine owns TLOF.
41  : TargetLoweringBase(tm) {}
42 
43 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
44  return nullptr;
45 }
46 
49 }
50 
51 /// Check whether a given call node is in tail position within its function. If
52 /// so, it sets Chain to the input chain of the tail call.
54  SDValue &Chain) const {
55  const Function &F = DAG.getMachineFunction().getFunction();
56 
57  // Conservatively require the attributes of the call to match those of
58  // the return. Ignore noalias because it doesn't affect the call sequence.
59  AttributeList CallerAttrs = F.getAttributes();
60  if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
61  .removeAttribute(Attribute::NoAlias)
62  .hasAttributes())
63  return false;
64 
65  // It's not safe to eliminate the sign / zero extension of the return value.
66  if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
67  CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
68  return false;
69 
70  // Check if the only use is a function return node.
71  return isUsedByReturnOnly(Node, Chain);
72 }
73 
75  const uint32_t *CallerPreservedMask,
76  const SmallVectorImpl<CCValAssign> &ArgLocs,
77  const SmallVectorImpl<SDValue> &OutVals) const {
78  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
79  const CCValAssign &ArgLoc = ArgLocs[I];
80  if (!ArgLoc.isRegLoc())
81  continue;
82  unsigned Reg = ArgLoc.getLocReg();
83  // Only look at callee saved registers.
84  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
85  continue;
86  // Check that we pass the value used for the caller.
87  // (We look for a CopyFromReg reading a virtual register that is used
88  // for the function live-in value of register Reg)
89  SDValue Value = OutVals[I];
90  if (Value->getOpcode() != ISD::CopyFromReg)
91  return false;
92  unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
93  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
94  return false;
95  }
96  return true;
97 }
98 
99 /// Set CallLoweringInfo attribute flags based on a call instruction
100 /// and called function attributes.
102  unsigned ArgIdx) {
103  IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt);
104  IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt);
105  IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg);
106  IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet);
107  IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest);
108  IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal);
109  IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca);
110  IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned);
111  IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
112  IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError);
113  Alignment = CS->getParamAlignment(ArgIdx);
114 }
115 
116 /// Generate a libcall taking the given operands as arguments and returning a
117 /// result of type RetVT.
118 std::pair<SDValue, SDValue>
120  ArrayRef<SDValue> Ops, bool isSigned,
121  const SDLoc &dl, bool doesNotReturn,
122  bool isReturnValueUsed) const {
124  Args.reserve(Ops.size());
125 
127  for (SDValue Op : Ops) {
128  Entry.Node = Op;
129  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
130  Entry.IsSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
131  Entry.IsZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
132  Args.push_back(Entry);
133  }
134 
135  if (LC == RTLIB::UNKNOWN_LIBCALL)
136  report_fatal_error("Unsupported library call operation!");
138  getPointerTy(DAG.getDataLayout()));
139 
140  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
142  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, isSigned);
143  CLI.setDebugLoc(dl)
144  .setChain(DAG.getEntryNode())
145  .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
146  .setNoReturn(doesNotReturn)
147  .setDiscardResult(!isReturnValueUsed)
148  .setSExtResult(signExtend)
149  .setZExtResult(!signExtend);
150  return LowerCallTo(CLI);
151 }
152 
153 /// Soften the operands of a comparison. This code is shared among BR_CC,
154 /// SELECT_CC, and SETCC handlers.
156  SDValue &NewLHS, SDValue &NewRHS,
157  ISD::CondCode &CCCode,
158  const SDLoc &dl) const {
159  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
160  && "Unsupported setcc type!");
161 
162  // Expand into one or more soft-fp libcall(s).
163  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
164  bool ShouldInvertCC = false;
165  switch (CCCode) {
166  case ISD::SETEQ:
167  case ISD::SETOEQ:
168  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
169  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
170  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
171  break;
172  case ISD::SETNE:
173  case ISD::SETUNE:
174  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
175  (VT == MVT::f64) ? RTLIB::UNE_F64 :
176  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
177  break;
178  case ISD::SETGE:
179  case ISD::SETOGE:
180  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
181  (VT == MVT::f64) ? RTLIB::OGE_F64 :
182  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
183  break;
184  case ISD::SETLT:
185  case ISD::SETOLT:
186  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
187  (VT == MVT::f64) ? RTLIB::OLT_F64 :
188  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
189  break;
190  case ISD::SETLE:
191  case ISD::SETOLE:
192  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
193  (VT == MVT::f64) ? RTLIB::OLE_F64 :
194  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
195  break;
196  case ISD::SETGT:
197  case ISD::SETOGT:
198  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
199  (VT == MVT::f64) ? RTLIB::OGT_F64 :
200  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
201  break;
202  case ISD::SETUO:
203  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
204  (VT == MVT::f64) ? RTLIB::UO_F64 :
205  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
206  break;
207  case ISD::SETO:
208  LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
209  (VT == MVT::f64) ? RTLIB::O_F64 :
210  (VT == MVT::f128) ? RTLIB::O_F128 : RTLIB::O_PPCF128;
211  break;
212  case ISD::SETONE:
213  // SETONE = SETOLT | SETOGT
214  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
215  (VT == MVT::f64) ? RTLIB::OLT_F64 :
216  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
217  LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
218  (VT == MVT::f64) ? RTLIB::OGT_F64 :
219  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
220  break;
221  case ISD::SETUEQ:
222  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
223  (VT == MVT::f64) ? RTLIB::UO_F64 :
224  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
225  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
226  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
227  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
228  break;
229  default:
230  // Invert CC for unordered comparisons
231  ShouldInvertCC = true;
232  switch (CCCode) {
233  case ISD::SETULT:
234  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
235  (VT == MVT::f64) ? RTLIB::OGE_F64 :
236  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
237  break;
238  case ISD::SETULE:
239  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
240  (VT == MVT::f64) ? RTLIB::OGT_F64 :
241  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
242  break;
243  case ISD::SETUGT:
244  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
245  (VT == MVT::f64) ? RTLIB::OLE_F64 :
246  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
247  break;
248  case ISD::SETUGE:
249  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
250  (VT == MVT::f64) ? RTLIB::OLT_F64 :
251  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
252  break;
253  default: llvm_unreachable("Do not know how to soften this setcc!");
254  }
255  }
256 
257  // Use the target specific return value for comparions lib calls.
258  EVT RetVT = getCmpLibcallReturnType();
259  SDValue Ops[2] = {NewLHS, NewRHS};
260  NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, false /*sign irrelevant*/,
261  dl).first;
262  NewRHS = DAG.getConstant(0, dl, RetVT);
263 
264  CCCode = getCmpLibcallCC(LC1);
265  if (ShouldInvertCC)
266  CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);
267 
268  if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
269  SDValue Tmp = DAG.getNode(
270  ISD::SETCC, dl,
271  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
272  NewLHS, NewRHS, DAG.getCondCode(CCCode));
273  NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, false/*sign irrelevant*/,
274  dl).first;
275  NewLHS = DAG.getNode(
276  ISD::SETCC, dl,
277  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
278  NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
279  NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
280  NewRHS = SDValue();
281  }
282 }
283 
284 /// Return the entry encoding for a jump table in the current function. The
285 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
287  // In non-pic modes, just use the address of a block.
288  if (!isPositionIndependent())
290 
291  // In PIC mode, if the target supports a GPRel32 directive, use it.
292  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
294 
295  // Otherwise, use a label difference.
297 }
298 
300  SelectionDAG &DAG) const {
301  // If our PIC model is GP relative, use the global offset table as the base.
302  unsigned JTEncoding = getJumpTableEncoding();
303 
304  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
307 
308  return Table;
309 }
310 
311 /// This returns the relocation base for the given PIC jumptable, the same as
312 /// getPICJumpTableRelocBase, but as an MCExpr.
313 const MCExpr *
315  unsigned JTI,MCContext &Ctx) const{
316  // The normal PIC reloc base is the label at the start of the jump table.
317  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
318 }
319 
320 bool
322  const TargetMachine &TM = getTargetMachine();
323  const GlobalValue *GV = GA->getGlobal();
324 
325  // If the address is not even local to this DSO we will have to load it from
326  // a got and then add the offset.
327  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
328  return false;
329 
330  // If the code is position independent we will have to add a base register.
331  if (isPositionIndependent())
332  return false;
333 
334  // Otherwise we can do it.
335  return true;
336 }
337 
338 //===----------------------------------------------------------------------===//
339 // Optimization Methods
340 //===----------------------------------------------------------------------===//
341 
342 /// If the specified instruction has a constant integer operand and there are
343 /// bits set in that constant that are not demanded, then clear those bits and
344 /// return true.
346  TargetLoweringOpt &TLO) const {
347  SelectionDAG &DAG = TLO.DAG;
348  SDLoc DL(Op);
349  unsigned Opcode = Op.getOpcode();
350 
351  // Do target-specific constant optimization.
352  if (targetShrinkDemandedConstant(Op, Demanded, TLO))
353  return TLO.New.getNode();
354 
355  // FIXME: ISD::SELECT, ISD::SELECT_CC
356  switch (Opcode) {
357  default:
358  break;
359  case ISD::XOR:
360  case ISD::AND:
361  case ISD::OR: {
362  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
363  if (!Op1C)
364  return false;
365 
366  // If this is a 'not' op, don't touch it because that's a canonical form.
367  const APInt &C = Op1C->getAPIntValue();
368  if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
369  return false;
370 
371  if (!C.isSubsetOf(Demanded)) {
372  EVT VT = Op.getValueType();
373  SDValue NewC = DAG.getConstant(Demanded & C, DL, VT);
374  SDValue NewOp = DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
375  return TLO.CombineTo(Op, NewOp);
376  }
377 
378  break;
379  }
380  }
381 
382  return false;
383 }
384 
385 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
386 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
387 /// generalized for targets with other types of implicit widening casts.
389  const APInt &Demanded,
390  TargetLoweringOpt &TLO) const {
391  assert(Op.getNumOperands() == 2 &&
392  "ShrinkDemandedOp only supports binary operators!");
393  assert(Op.getNode()->getNumValues() == 1 &&
394  "ShrinkDemandedOp only supports nodes with one result!");
395 
396  SelectionDAG &DAG = TLO.DAG;
397  SDLoc dl(Op);
398 
399  // Early return, as this function cannot handle vector types.
400  if (Op.getValueType().isVector())
401  return false;
402 
403  // Don't do this if the node has another user, which may require the
404  // full value.
405  if (!Op.getNode()->hasOneUse())
406  return false;
407 
408  // Search for the smallest integer type with free casts to and from
409  // Op's type. For expedience, just check power-of-2 integer types.
410  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
411  unsigned DemandedSize = Demanded.getActiveBits();
412  unsigned SmallVTBits = DemandedSize;
413  if (!isPowerOf2_32(SmallVTBits))
414  SmallVTBits = NextPowerOf2(SmallVTBits);
415  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
416  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
417  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
418  TLI.isZExtFree(SmallVT, Op.getValueType())) {
419  // We found a type with free casts.
420  SDValue X = DAG.getNode(
421  Op.getOpcode(), dl, SmallVT,
422  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
423  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
424  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
425  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
426  return TLO.CombineTo(Op, Z);
427  }
428  }
429  return false;
430 }
431 
432 bool
434  const APInt &Demanded,
435  DAGCombinerInfo &DCI,
436  TargetLoweringOpt &TLO) const {
437  SDValue Op = User->getOperand(OpIdx);
438  KnownBits Known;
439 
440  if (!SimplifyDemandedBits(Op, Demanded, Known, TLO, 0, true))
441  return false;
442 
443 
444  // Old will not always be the same as Op. For example:
445  //
446  // Demanded = 0xffffff
447  // Op = i64 truncate (i32 and x, 0xffffff)
448  // In this case simplify demand bits will want to replace the 'and' node
449  // with the value 'x', which will give us:
450  // Old = i32 and x, 0xffffff
451  // New = x
452  if (TLO.Old.hasOneUse()) {
453  // For the one use case, we just commit the change.
454  DCI.CommitTargetLoweringOpt(TLO);
455  return true;
456  }
457 
458  // If Old has more than one use then it must be Op, because the
459  // AssumeSingleUse flag is not propogated to recursive calls of
460  // SimplifyDemanded bits, so the only node with multiple use that
461  // it will attempt to combine will be Op.
462  assert(TLO.Old == Op);
463 
465  for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
466  if (i == OpIdx) {
467  NewOps.push_back(TLO.New);
468  continue;
469  }
470  NewOps.push_back(User->getOperand(i));
471  }
472  User = TLO.DAG.UpdateNodeOperands(User, NewOps);
473  // Op has less users now, so we may be able to perform additional combines
474  // with it.
475  DCI.AddToWorklist(Op.getNode());
476  // User's operands have been updated, so we may be able to do new combines
477  // with it.
478  DCI.AddToWorklist(User);
479  return true;
480 }
481 
483  DAGCombinerInfo &DCI) const {
484 
485  SelectionDAG &DAG = DCI.DAG;
486  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
487  !DCI.isBeforeLegalizeOps());
488  KnownBits Known;
489 
490  bool Simplified = SimplifyDemandedBits(Op, DemandedMask, Known, TLO);
491  if (Simplified)
492  DCI.CommitTargetLoweringOpt(TLO);
493  return Simplified;
494 }
495 
496 /// Look at Op. At this point, we know that only the DemandedMask bits of the
497 /// result of Op are ever used downstream. If we can use this information to
498 /// simplify Op, create a new simplified DAG node and return true, returning the
499 /// original and new nodes in Old and New. Otherwise, analyze the expression and
500 /// return a mask of Known bits for the expression (used to simplify the
501 /// caller). The Known bits may only be accurate for those bits in the
502 /// DemandedMask.
504  const APInt &DemandedMask,
505  KnownBits &Known,
506  TargetLoweringOpt &TLO,
507  unsigned Depth,
508  bool AssumeSingleUse) const {
509  unsigned BitWidth = DemandedMask.getBitWidth();
510  assert(Op.getScalarValueSizeInBits() == BitWidth &&
511  "Mask size mismatches value type size!");
512  APInt NewMask = DemandedMask;
513  SDLoc dl(Op);
514  auto &DL = TLO.DAG.getDataLayout();
515 
516  // Don't know anything.
517  Known = KnownBits(BitWidth);
518 
519  if (Op.getOpcode() == ISD::Constant) {
520  // We know all of the bits for a constant!
521  Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
522  Known.Zero = ~Known.One;
523  return false;
524  }
525 
526  // Other users may use these bits.
527  EVT VT = Op.getValueType();
528  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
529  if (Depth != 0) {
530  // If not at the root, Just compute the Known bits to
531  // simplify things downstream.
532  TLO.DAG.computeKnownBits(Op, Known, Depth);
533  return false;
534  }
535  // If this is the root being simplified, allow it to have multiple uses,
536  // just set the NewMask to all bits.
537  NewMask = APInt::getAllOnesValue(BitWidth);
538  } else if (DemandedMask == 0) {
539  // Not demanding any bits from Op.
540  if (!Op.isUndef())
541  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
542  return false;
543  } else if (Depth == 6) { // Limit search depth.
544  return false;
545  }
546 
547  KnownBits Known2, KnownOut;
548  switch (Op.getOpcode()) {
549  case ISD::BUILD_VECTOR:
550  // Collect the known bits that are shared by every constant vector element.
551  Known.Zero.setAllBits(); Known.One.setAllBits();
552  for (SDValue SrcOp : Op->ops()) {
553  if (!isa<ConstantSDNode>(SrcOp)) {
554  // We can only handle all constant values - bail out with no known bits.
555  Known = KnownBits(BitWidth);
556  return false;
557  }
558  Known2.One = cast<ConstantSDNode>(SrcOp)->getAPIntValue();
559  Known2.Zero = ~Known2.One;
560 
561  // BUILD_VECTOR can implicitly truncate sources, we must handle this.
562  if (Known2.One.getBitWidth() != BitWidth) {
563  assert(Known2.getBitWidth() > BitWidth &&
564  "Expected BUILD_VECTOR implicit truncation");
565  Known2 = Known2.trunc(BitWidth);
566  }
567 
568  // Known bits are the values that are shared by every element.
569  // TODO: support per-element known bits.
570  Known.One &= Known2.One;
571  Known.Zero &= Known2.Zero;
572  }
573  return false; // Don't fall through, will infinitely loop.
574  case ISD::AND:
575  // If the RHS is a constant, check to see if the LHS would be zero without
576  // using the bits from the RHS. Below, we use knowledge about the RHS to
577  // simplify the LHS, here we're using information from the LHS to simplify
578  // the RHS.
579  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op.getOperand(1))) {
580  SDValue Op0 = Op.getOperand(0);
581  KnownBits LHSKnown;
582  // Do not increment Depth here; that can cause an infinite loop.
583  TLO.DAG.computeKnownBits(Op0, LHSKnown, Depth);
584  // If the LHS already has zeros where RHSC does, this 'and' is dead.
585  if ((LHSKnown.Zero & NewMask) == (~RHSC->getAPIntValue() & NewMask))
586  return TLO.CombineTo(Op, Op0);
587 
588  // If any of the set bits in the RHS are known zero on the LHS, shrink
589  // the constant.
590  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & NewMask, TLO))
591  return true;
592 
593  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
594  // constant, but if this 'and' is only clearing bits that were just set by
595  // the xor, then this 'and' can be eliminated by shrinking the mask of
596  // the xor. For example, for a 32-bit X:
597  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
598  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
599  LHSKnown.One == ~RHSC->getAPIntValue()) {
600  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0),
601  Op.getOperand(1));
602  return TLO.CombineTo(Op, Xor);
603  }
604  }
605 
606  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known, TLO, Depth+1))
607  return true;
608  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
609  if (SimplifyDemandedBits(Op.getOperand(0), ~Known.Zero & NewMask,
610  Known2, TLO, Depth+1))
611  return true;
612  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
613 
614  // If all of the demanded bits are known one on one side, return the other.
615  // These bits cannot contribute to the result of the 'and'.
616  if (NewMask.isSubsetOf(Known2.Zero | Known.One))
617  return TLO.CombineTo(Op, Op.getOperand(0));
618  if (NewMask.isSubsetOf(Known.Zero | Known2.One))
619  return TLO.CombineTo(Op, Op.getOperand(1));
620  // If all of the demanded bits in the inputs are known zeros, return zero.
621  if (NewMask.isSubsetOf(Known.Zero | Known2.Zero))
622  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
623  // If the RHS is a constant, see if we can simplify it.
624  if (ShrinkDemandedConstant(Op, ~Known2.Zero & NewMask, TLO))
625  return true;
626  // If the operation can be done in a smaller type, do so.
627  if (ShrinkDemandedOp(Op, BitWidth, NewMask, TLO))
628  return true;
629 
630  // Output known-1 bits are only known if set in both the LHS & RHS.
631  Known.One &= Known2.One;
632  // Output known-0 are known to be clear if zero in either the LHS | RHS.
633  Known.Zero |= Known2.Zero;
634  break;
635  case ISD::OR:
636  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known, TLO, Depth+1))
637  return true;
638  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
639  if (SimplifyDemandedBits(Op.getOperand(0), ~Known.One & NewMask,
640  Known2, TLO, Depth+1))
641  return true;
642  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
643 
644  // If all of the demanded bits are known zero on one side, return the other.
645  // These bits cannot contribute to the result of the 'or'.
646  if (NewMask.isSubsetOf(Known2.One | Known.Zero))
647  return TLO.CombineTo(Op, Op.getOperand(0));
648  if (NewMask.isSubsetOf(Known.One | Known2.Zero))
649  return TLO.CombineTo(Op, Op.getOperand(1));
650  // If the RHS is a constant, see if we can simplify it.
651  if (ShrinkDemandedConstant(Op, NewMask, TLO))
652  return true;
653  // If the operation can be done in a smaller type, do so.
654  if (ShrinkDemandedOp(Op, BitWidth, NewMask, TLO))
655  return true;
656 
657  // Output known-0 bits are only known if clear in both the LHS & RHS.
658  Known.Zero &= Known2.Zero;
659  // Output known-1 are known to be set if set in either the LHS | RHS.
660  Known.One |= Known2.One;
661  break;
662  case ISD::XOR: {
663  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known, TLO, Depth+1))
664  return true;
665  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
666  if (SimplifyDemandedBits(Op.getOperand(0), NewMask, Known2, TLO, Depth+1))
667  return true;
668  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
669 
670  // If all of the demanded bits are known zero on one side, return the other.
671  // These bits cannot contribute to the result of the 'xor'.
672  if (NewMask.isSubsetOf(Known.Zero))
673  return TLO.CombineTo(Op, Op.getOperand(0));
674  if (NewMask.isSubsetOf(Known2.Zero))
675  return TLO.CombineTo(Op, Op.getOperand(1));
676  // If the operation can be done in a smaller type, do so.
677  if (ShrinkDemandedOp(Op, BitWidth, NewMask, TLO))
678  return true;
679 
680  // If all of the unknown bits are known to be zero on one side or the other
681  // (but not both) turn this into an *inclusive* or.
682  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
683  if (NewMask.isSubsetOf(Known.Zero|Known2.Zero))
684  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT,
685  Op.getOperand(0),
686  Op.getOperand(1)));
687 
688  // Output known-0 bits are known if clear or set in both the LHS & RHS.
689  KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
690  // Output known-1 are known to be set if set in only one of the LHS, RHS.
691  KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
692 
694  // If one side is a constant, and all of the known set bits on the other
695  // side are also set in the constant, turn this into an AND, as we know
696  // the bits will be cleared.
697  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
698  // NB: it is okay if more bits are known than are requested
699  if (C->getAPIntValue() == Known2.One) {
700  SDValue ANDC = TLO.DAG.getConstant(~C->getAPIntValue() & NewMask,
701  dl, VT);
702  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT,
703  Op.getOperand(0), ANDC));
704  }
705 
706  // If the RHS is a constant, see if we can change it. Don't alter a -1
707  // constant because that's a 'not' op, and that is better for combining
708  // and codegen.
709  if (!C->isAllOnesValue()) {
710  if (NewMask.isSubsetOf(C->getAPIntValue())) {
711  // We're flipping all demanded bits. Flip the undemanded bits too.
712  SDValue New = TLO.DAG.getNOT(dl, Op.getOperand(0), VT);
713  return TLO.CombineTo(Op, New);
714  }
715  // If we can't turn this into a 'not', try to shrink the constant.
716  if (ShrinkDemandedConstant(Op, NewMask, TLO))
717  return true;
718  }
719  }
720 
721  Known = std::move(KnownOut);
722  break;
723  }
724  case ISD::SELECT:
725  if (SimplifyDemandedBits(Op.getOperand(2), NewMask, Known, TLO, Depth+1))
726  return true;
727  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known2, TLO, Depth+1))
728  return true;
729  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
730  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
731 
732  // If the operands are constants, see if we can simplify them.
733  if (ShrinkDemandedConstant(Op, NewMask, TLO))
734  return true;
735 
736  // Only known if known in both the LHS and RHS.
737  Known.One &= Known2.One;
738  Known.Zero &= Known2.Zero;
739  break;
740  case ISD::SELECT_CC:
741  if (SimplifyDemandedBits(Op.getOperand(3), NewMask, Known, TLO, Depth+1))
742  return true;
743  if (SimplifyDemandedBits(Op.getOperand(2), NewMask, Known2, TLO, Depth+1))
744  return true;
745  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
746  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
747 
748  // If the operands are constants, see if we can simplify them.
749  if (ShrinkDemandedConstant(Op, NewMask, TLO))
750  return true;
751 
752  // Only known if known in both the LHS and RHS.
753  Known.One &= Known2.One;
754  Known.Zero &= Known2.Zero;
755  break;
756  case ISD::SETCC: {
757  SDValue Op0 = Op.getOperand(0);
758  SDValue Op1 = Op.getOperand(1);
759  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
760  // If (1) we only need the sign-bit, (2) the setcc operands are the same
761  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
762  // -1, we may be able to bypass the setcc.
763  if (NewMask.isSignMask() && Op0.getScalarValueSizeInBits() == BitWidth &&
764  getBooleanContents(VT) ==
765  BooleanContent::ZeroOrNegativeOneBooleanContent) {
766  // If we're testing X < 0, then this compare isn't needed - just use X!
767  // FIXME: We're limiting to integer types here, but this should also work
768  // if we don't care about FP signed-zero. The use of SETLT with FP means
769  // that we don't care about NaNs.
770  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
772  return TLO.CombineTo(Op, Op0);
773 
774  // TODO: Should we check for other forms of sign-bit comparisons?
775  // Examples: X <= -1, X >= 0
776  }
777  if (getBooleanContents(Op0.getValueType()) ==
779  BitWidth > 1)
780  Known.Zero.setBitsFrom(1);
781  break;
782  }
783  case ISD::SHL:
784  if (ConstantSDNode *SA = isConstOrConstSplat(Op.getOperand(1))) {
785  SDValue InOp = Op.getOperand(0);
786 
787  // If the shift count is an invalid immediate, don't do anything.
788  if (SA->getAPIntValue().uge(BitWidth))
789  break;
790 
791  unsigned ShAmt = SA->getZExtValue();
792 
793  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
794  // single shift. We can do this if the bottom bits (which are shifted
795  // out) are never demanded.
796  if (InOp.getOpcode() == ISD::SRL) {
797  if (ConstantSDNode *SA2 = isConstOrConstSplat(InOp.getOperand(1))) {
798  if (ShAmt && (NewMask & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) {
799  if (SA2->getAPIntValue().ult(BitWidth)) {
800  unsigned C1 = SA2->getZExtValue();
801  unsigned Opc = ISD::SHL;
802  int Diff = ShAmt-C1;
803  if (Diff < 0) {
804  Diff = -Diff;
805  Opc = ISD::SRL;
806  }
807 
808  SDValue NewSA =
809  TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
810  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
811  InOp.getOperand(0),
812  NewSA));
813  }
814  }
815  }
816  }
817 
818  if (SimplifyDemandedBits(InOp, NewMask.lshr(ShAmt), Known, TLO, Depth+1))
819  return true;
820 
821  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
822  // are not demanded. This will likely allow the anyext to be folded away.
823  if (InOp.getNode()->getOpcode() == ISD::ANY_EXTEND) {
824  SDValue InnerOp = InOp.getOperand(0);
825  EVT InnerVT = InnerOp.getValueType();
826  unsigned InnerBits = InnerVT.getScalarSizeInBits();
827  if (ShAmt < InnerBits && NewMask.getActiveBits() <= InnerBits &&
828  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
829  EVT ShTy = getShiftAmountTy(InnerVT, DL);
830  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
831  ShTy = InnerVT;
832  SDValue NarrowShl =
833  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
834  TLO.DAG.getConstant(ShAmt, dl, ShTy));
835  return
836  TLO.CombineTo(Op,
837  TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
838  }
839  // Repeat the SHL optimization above in cases where an extension
840  // intervenes: (shl (anyext (shr x, c1)), c2) to
841  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
842  // aren't demanded (as above) and that the shifted upper c1 bits of
843  // x aren't demanded.
844  if (InOp.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
845  InnerOp.hasOneUse()) {
846  if (ConstantSDNode *SA2 = isConstOrConstSplat(InnerOp.getOperand(1))) {
847  unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
848  if (InnerShAmt < ShAmt &&
849  InnerShAmt < InnerBits &&
850  NewMask.getActiveBits() <= (InnerBits - InnerShAmt + ShAmt) &&
851  NewMask.countTrailingZeros() >= ShAmt) {
852  SDValue NewSA =
853  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
854  Op.getOperand(1).getValueType());
855  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
856  InnerOp.getOperand(0));
857  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT,
858  NewExt, NewSA));
859  }
860  }
861  }
862  }
863 
864  Known.Zero <<= ShAmt;
865  Known.One <<= ShAmt;
866  // low bits known zero.
867  Known.Zero.setLowBits(ShAmt);
868  }
869  break;
870  case ISD::SRL:
871  if (ConstantSDNode *SA = isConstOrConstSplat(Op.getOperand(1))) {
872  SDValue InOp = Op.getOperand(0);
873 
874  // If the shift count is an invalid immediate, don't do anything.
875  if (SA->getAPIntValue().uge(BitWidth))
876  break;
877 
878  unsigned ShAmt = SA->getZExtValue();
879  APInt InDemandedMask = (NewMask << ShAmt);
880 
881  // If the shift is exact, then it does demand the low bits (and knows that
882  // they are zero).
883  if (Op->getFlags().hasExact())
884  InDemandedMask.setLowBits(ShAmt);
885 
886  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
887  // single shift. We can do this if the top bits (which are shifted out)
888  // are never demanded.
889  if (InOp.getOpcode() == ISD::SHL) {
890  if (ConstantSDNode *SA2 = isConstOrConstSplat(InOp.getOperand(1))) {
891  if (ShAmt &&
892  (NewMask & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) {
893  if (SA2->getAPIntValue().ult(BitWidth)) {
894  unsigned C1 = SA2->getZExtValue();
895  unsigned Opc = ISD::SRL;
896  int Diff = ShAmt-C1;
897  if (Diff < 0) {
898  Diff = -Diff;
899  Opc = ISD::SHL;
900  }
901 
902  SDValue NewSA =
903  TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
904  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
905  InOp.getOperand(0),
906  NewSA));
907  }
908  }
909  }
910  }
911 
912  // Compute the new bits that are at the top now.
913  if (SimplifyDemandedBits(InOp, InDemandedMask, Known, TLO, Depth+1))
914  return true;
915  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
916  Known.Zero.lshrInPlace(ShAmt);
917  Known.One.lshrInPlace(ShAmt);
918 
919  Known.Zero.setHighBits(ShAmt); // High bits known zero.
920  }
921  break;
922  case ISD::SRA:
923  // If this is an arithmetic shift right and only the low-bit is set, we can
924  // always convert this into a logical shr, even if the shift amount is
925  // variable. The low bit of the shift cannot be an input sign bit unless
926  // the shift amount is >= the size of the datatype, which is undefined.
927  if (NewMask.isOneValue())
928  return TLO.CombineTo(Op,
929  TLO.DAG.getNode(ISD::SRL, dl, VT, Op.getOperand(0),
930  Op.getOperand(1)));
931 
932  if (ConstantSDNode *SA = isConstOrConstSplat(Op.getOperand(1))) {
933  // If the shift count is an invalid immediate, don't do anything.
934  if (SA->getAPIntValue().uge(BitWidth))
935  break;
936 
937  unsigned ShAmt = SA->getZExtValue();
938  APInt InDemandedMask = (NewMask << ShAmt);
939 
940  // If the shift is exact, then it does demand the low bits (and knows that
941  // they are zero).
942  if (Op->getFlags().hasExact())
943  InDemandedMask.setLowBits(ShAmt);
944 
945  // If any of the demanded bits are produced by the sign extension, we also
946  // demand the input sign bit.
947  if (NewMask.countLeadingZeros() < ShAmt)
948  InDemandedMask.setSignBit();
949 
950  if (SimplifyDemandedBits(Op.getOperand(0), InDemandedMask, Known, TLO,
951  Depth+1))
952  return true;
953  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
954  Known.Zero.lshrInPlace(ShAmt);
955  Known.One.lshrInPlace(ShAmt);
956 
957  // If the input sign bit is known to be zero, or if none of the top bits
958  // are demanded, turn this into an unsigned shift right.
959  if (Known.Zero[BitWidth - ShAmt - 1] ||
960  NewMask.countLeadingZeros() >= ShAmt) {
961  SDNodeFlags Flags;
962  Flags.setExact(Op->getFlags().hasExact());
963  return TLO.CombineTo(Op,
964  TLO.DAG.getNode(ISD::SRL, dl, VT, Op.getOperand(0),
965  Op.getOperand(1), Flags));
966  }
967 
968  int Log2 = NewMask.exactLogBase2();
969  if (Log2 >= 0) {
970  // The bit must come from the sign.
971  SDValue NewSA =
972  TLO.DAG.getConstant(BitWidth - 1 - Log2, dl,
973  Op.getOperand(1).getValueType());
974  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT,
975  Op.getOperand(0), NewSA));
976  }
977 
978  if (Known.One[BitWidth - ShAmt - 1])
979  // New bits are known one.
980  Known.One.setHighBits(ShAmt);
981  }
982  break;
983  case ISD::SIGN_EXTEND_INREG: {
984  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
985  unsigned ExVTBits = ExVT.getScalarSizeInBits();
986 
987  // If we only care about the highest bit, don't bother shifting right.
988  if (NewMask.isSignMask()) {
989  SDValue InOp = Op.getOperand(0);
990  bool AlreadySignExtended =
991  TLO.DAG.ComputeNumSignBits(InOp) >= BitWidth-ExVTBits+1;
992  // However if the input is already sign extended we expect the sign
993  // extension to be dropped altogether later and do not simplify.
994  if (!AlreadySignExtended) {
995  // Compute the correct shift amount type, which must be getShiftAmountTy
996  // for scalar types after legalization.
997  EVT ShiftAmtTy = VT;
998  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
999  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1000 
1001  SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ExVTBits, dl,
1002  ShiftAmtTy);
1003  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, InOp,
1004  ShiftAmt));
1005  }
1006  }
1007 
1008  // If none of the extended bits are demanded, eliminate the sextinreg.
1009  if (NewMask.getActiveBits() <= ExVTBits)
1010  return TLO.CombineTo(Op, Op.getOperand(0));
1011 
1012  APInt InputDemandedBits = NewMask.getLoBits(ExVTBits);
1013 
1014  // Since the sign extended bits are demanded, we know that the sign
1015  // bit is demanded.
1016  InputDemandedBits.setBit(ExVTBits - 1);
1017 
1018  if (SimplifyDemandedBits(Op.getOperand(0), InputDemandedBits,
1019  Known, TLO, Depth+1))
1020  return true;
1021  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1022 
1023  // If the sign bit of the input is known set or clear, then we know the
1024  // top bits of the result.
1025 
1026  // If the input sign bit is known zero, convert this into a zero extension.
1027  if (Known.Zero[ExVTBits - 1])
1028  return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(
1029  Op.getOperand(0), dl, ExVT.getScalarType()));
1030 
1031  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1032  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1033  Known.One.setBitsFrom(ExVTBits);
1034  Known.Zero &= Mask;
1035  } else { // Input sign bit unknown
1036  Known.Zero &= Mask;
1037  Known.One &= Mask;
1038  }
1039  break;
1040  }
1041  case ISD::BUILD_PAIR: {
1042  EVT HalfVT = Op.getOperand(0).getValueType();
1043  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1044 
1045  APInt MaskLo = NewMask.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1046  APInt MaskHi = NewMask.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1047 
1048  KnownBits KnownLo, KnownHi;
1049 
1050  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1051  return true;
1052 
1053  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1054  return true;
1055 
1056  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1057  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1058 
1059  Known.One = KnownLo.One.zext(BitWidth) |
1060  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1061  break;
1062  }
1063  case ISD::ZERO_EXTEND: {
1064  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1065 
1066  // If none of the top bits are demanded, convert this into an any_extend.
1067  if (NewMask.getActiveBits() <= OperandBitWidth)
1068  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1069  Op.getOperand(0)));
1070 
1071  APInt InMask = NewMask.trunc(OperandBitWidth);
1072  if (SimplifyDemandedBits(Op.getOperand(0), InMask, Known, TLO, Depth+1))
1073  return true;
1074  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1075  Known = Known.zext(BitWidth);
1076  Known.Zero.setBitsFrom(OperandBitWidth);
1077  break;
1078  }
1079  case ISD::SIGN_EXTEND: {
1080  unsigned InBits = Op.getOperand(0).getValueType().getScalarSizeInBits();
1081 
1082  // If none of the top bits are demanded, convert this into an any_extend.
1083  if (NewMask.getActiveBits() <= InBits)
1084  return TLO.CombineTo(Op,TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1085  Op.getOperand(0)));
1086 
1087  // Since some of the sign extended bits are demanded, we know that the sign
1088  // bit is demanded.
1089  APInt InDemandedBits = NewMask.trunc(InBits);
1090  InDemandedBits.setBit(InBits - 1);
1091 
1092  if (SimplifyDemandedBits(Op.getOperand(0), InDemandedBits, Known, TLO,
1093  Depth+1))
1094  return true;
1095  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1096  // If the sign bit is known one, the top bits match.
1097  Known = Known.sext(BitWidth);
1098 
1099  // If the sign bit is known zero, convert this to a zero extend.
1100  if (Known.isNonNegative())
1101  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT,
1102  Op.getOperand(0)));
1103  break;
1104  }
1105  case ISD::ANY_EXTEND: {
1106  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1107  APInt InMask = NewMask.trunc(OperandBitWidth);
1108  if (SimplifyDemandedBits(Op.getOperand(0), InMask, Known, TLO, Depth+1))
1109  return true;
1110  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1111  Known = Known.zext(BitWidth);
1112  break;
1113  }
1114  case ISD::TRUNCATE: {
1115  // Simplify the input, using demanded bit information, and compute the known
1116  // zero/one bits live out.
1117  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1118  APInt TruncMask = NewMask.zext(OperandBitWidth);
1119  if (SimplifyDemandedBits(Op.getOperand(0), TruncMask, Known, TLO, Depth+1))
1120  return true;
1121  Known = Known.trunc(BitWidth);
1122 
1123  // If the input is only used by this truncate, see if we can shrink it based
1124  // on the known demanded bits.
1125  if (Op.getOperand(0).getNode()->hasOneUse()) {
1126  SDValue In = Op.getOperand(0);
1127  switch (In.getOpcode()) {
1128  default: break;
1129  case ISD::SRL:
1130  // Shrink SRL by a constant if none of the high bits shifted in are
1131  // demanded.
1132  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1133  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1134  // undesirable.
1135  break;
1137  if (!ShAmt)
1138  break;
1139  SDValue Shift = In.getOperand(1);
1140  if (TLO.LegalTypes()) {
1141  uint64_t ShVal = ShAmt->getZExtValue();
1142  Shift = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));
1143  }
1144 
1145  if (ShAmt->getZExtValue() < BitWidth) {
1146  APInt HighBits = APInt::getHighBitsSet(OperandBitWidth,
1147  OperandBitWidth - BitWidth);
1148  HighBits.lshrInPlace(ShAmt->getZExtValue());
1149  HighBits = HighBits.trunc(BitWidth);
1150 
1151  if (!(HighBits & NewMask)) {
1152  // None of the shifted in bits are needed. Add a truncate of the
1153  // shift input, then shift it.
1154  SDValue NewTrunc = TLO.DAG.getNode(ISD::TRUNCATE, dl, VT,
1155  In.getOperand(0));
1156  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc,
1157  Shift));
1158  }
1159  }
1160  break;
1161  }
1162  }
1163 
1164  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1165  break;
1166  }
1167  case ISD::AssertZext: {
1168  // AssertZext demands all of the high bits, plus any of the low bits
1169  // demanded by its users.
1170  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1171  APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
1172  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | NewMask,
1173  Known, TLO, Depth+1))
1174  return true;
1175  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1176 
1177  Known.Zero |= ~InMask;
1178  break;
1179  }
1180  case ISD::BITCAST:
1181  // If this is an FP->Int bitcast and if the sign bit is the only
1182  // thing demanded, turn this into a FGETSIGN.
1183  if (!TLO.LegalOperations() && !VT.isVector() &&
1184  !Op.getOperand(0).getValueType().isVector() &&
1185  NewMask == APInt::getSignMask(Op.getValueSizeInBits()) &&
1187  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
1189  if ((OpVTLegal || i32Legal) && VT.isSimple() &&
1190  Op.getOperand(0).getValueType() != MVT::f16 &&
1191  Op.getOperand(0).getValueType() != MVT::f128) {
1192  // Cannot eliminate/lower SHL for f128 yet.
1193  EVT Ty = OpVTLegal ? VT : MVT::i32;
1194  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1195  // place. We expect the SHL to be eliminated by other optimizations.
1196  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Op.getOperand(0));
1197  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1198  if (!OpVTLegal && OpVTSizeInBits > 32)
1199  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
1200  unsigned ShVal = Op.getValueSizeInBits() - 1;
1201  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1202  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
1203  }
1204  }
1205  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
1206  // recursive call where Known may be useful to the caller.
1207  if (Depth > 0) {
1208  TLO.DAG.computeKnownBits(Op, Known, Depth);
1209  return false;
1210  }
1211  break;
1212  case ISD::ADD:
1213  case ISD::MUL:
1214  case ISD::SUB: {
1215  // Add, Sub, and Mul don't demand any bits in positions beyond that
1216  // of the highest bit demanded of them.
1217  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
1218  unsigned NewMaskLZ = NewMask.countLeadingZeros();
1219  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - NewMaskLZ);
1220  if (SimplifyDemandedBits(Op0, LoMask, Known2, TLO, Depth + 1) ||
1221  SimplifyDemandedBits(Op1, LoMask, Known2, TLO, Depth + 1) ||
1222  // See if the operation should be performed at a smaller bit width.
1223  ShrinkDemandedOp(Op, BitWidth, NewMask, TLO)) {
1224  SDNodeFlags Flags = Op.getNode()->getFlags();
1225  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1226  // Disable the nsw and nuw flags. We can no longer guarantee that we
1227  // won't wrap after simplification.
1228  Flags.setNoSignedWrap(false);
1229  Flags.setNoUnsignedWrap(false);
1230  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1,
1231  Flags);
1232  return TLO.CombineTo(Op, NewOp);
1233  }
1234  return true;
1235  }
1236 
1237  // If we have a constant operand, we may be able to turn it into -1 if we
1238  // do not demand the high bits. This can make the constant smaller to
1239  // encode, allow more general folding, or match specialized instruction
1240  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
1241  // is probably not useful (and could be detrimental).
1243  APInt HighMask = APInt::getHighBitsSet(NewMask.getBitWidth(), NewMaskLZ);
1244  if (C && !C->isAllOnesValue() && !C->isOne() &&
1245  (C->getAPIntValue() | HighMask).isAllOnesValue()) {
1246  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
1247  // We can't guarantee that the new math op doesn't wrap, so explicitly
1248  // clear those flags to prevent folding with a potential existing node
1249  // that has those flags set.
1250  SDNodeFlags Flags;
1251  Flags.setNoSignedWrap(false);
1252  Flags.setNoUnsignedWrap(false);
1253  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
1254  return TLO.CombineTo(Op, NewOp);
1255  }
1256 
1258  }
1259  default:
1260  // Just use computeKnownBits to compute output bits.
1261  TLO.DAG.computeKnownBits(Op, Known, Depth);
1262  break;
1263  }
1264 
1265  // If we know the value of all of the demanded bits, return this as a
1266  // constant.
1267  if (NewMask.isSubsetOf(Known.Zero|Known.One)) {
1268  // Avoid folding to a constant if any OpaqueConstant is involved.
1269  const SDNode *N = Op.getNode();
1271  E = SDNodeIterator::end(N); I != E; ++I) {
1272  SDNode *Op = *I;
1273  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
1274  if (C->isOpaque())
1275  return false;
1276  }
1277  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
1278  }
1279 
1280  return false;
1281 }
1282 
1284  const APInt &DemandedElts,
1285  APInt &KnownUndef,
1286  APInt &KnownZero,
1287  DAGCombinerInfo &DCI) const {
1288  SelectionDAG &DAG = DCI.DAG;
1289  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
1290  !DCI.isBeforeLegalizeOps());
1291 
1292  bool Simplified =
1293  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
1294  if (Simplified)
1295  DCI.CommitTargetLoweringOpt(TLO);
1296  return Simplified;
1297 }
1298 
1300  SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef,
1301  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
1302  bool AssumeSingleUse) const {
1303  EVT VT = Op.getValueType();
1304  APInt DemandedElts = DemandedEltMask;
1305  unsigned NumElts = DemandedElts.getBitWidth();
1306  assert(VT.isVector() && "Expected vector op");
1307  assert(VT.getVectorNumElements() == NumElts &&
1308  "Mask size mismatches value type element count!");
1309 
1310  KnownUndef = KnownZero = APInt::getNullValue(NumElts);
1311 
1312  // Undef operand.
1313  if (Op.isUndef()) {
1314  KnownUndef.setAllBits();
1315  return false;
1316  }
1317 
1318  // If Op has other users, assume that all elements are needed.
1319  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
1320  DemandedElts.setAllBits();
1321 
1322  // Not demanding any elements from Op.
1323  if (DemandedElts == 0) {
1324  KnownUndef.setAllBits();
1325  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1326  }
1327 
1328  // Limit search depth.
1329  if (Depth >= 6)
1330  return false;
1331 
1332  SDLoc DL(Op);
1333  unsigned EltSizeInBits = VT.getScalarSizeInBits();
1334 
1335  switch (Op.getOpcode()) {
1336  case ISD::SCALAR_TO_VECTOR: {
1337  if (!DemandedElts[0]) {
1338  KnownUndef.setAllBits();
1339  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1340  }
1341  KnownUndef.setHighBits(NumElts - 1);
1342  break;
1343  }
1344  case ISD::BITCAST: {
1345  SDValue Src = Op.getOperand(0);
1346  EVT SrcVT = Src.getValueType();
1347 
1348  // We only handle vectors here.
1349  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
1350  if (!SrcVT.isVector())
1351  break;
1352 
1353  // Fast handling of 'identity' bitcasts.
1354  unsigned NumSrcElts = SrcVT.getVectorNumElements();
1355  if (NumSrcElts == NumElts)
1356  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
1357  KnownZero, TLO, Depth + 1);
1358 
1359  APInt SrcZero, SrcUndef;
1360  APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
1361 
1362  // Bitcast from 'large element' src vector to 'small element' vector, we
1363  // must demand a source element if any DemandedElt maps to it.
1364  if ((NumElts % NumSrcElts) == 0) {
1365  unsigned Scale = NumElts / NumSrcElts;
1366  for (unsigned i = 0; i != NumElts; ++i)
1367  if (DemandedElts[i])
1368  SrcDemandedElts.setBit(i / Scale);
1369 
1370  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1371  TLO, Depth + 1))
1372  return true;
1373 
1374  // If the src element is zero/undef then all the output elements will be -
1375  // only demanded elements are guaranteed to be correct.
1376  for (unsigned i = 0; i != NumSrcElts; ++i) {
1377  if (SrcDemandedElts[i]) {
1378  if (SrcZero[i])
1379  KnownZero.setBits(i * Scale, (i + 1) * Scale);
1380  if (SrcUndef[i])
1381  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
1382  }
1383  }
1384  }
1385 
1386  // Bitcast from 'small element' src vector to 'large element' vector, we
1387  // demand all smaller source elements covered by the larger demanded element
1388  // of this vector.
1389  if ((NumSrcElts % NumElts) == 0) {
1390  unsigned Scale = NumSrcElts / NumElts;
1391  for (unsigned i = 0; i != NumElts; ++i)
1392  if (DemandedElts[i])
1393  SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
1394 
1395  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1396  TLO, Depth + 1))
1397  return true;
1398 
1399  // If all the src elements covering an output element are zero/undef, then
1400  // the output element will be as well, assuming it was demanded.
1401  for (unsigned i = 0; i != NumElts; ++i) {
1402  if (DemandedElts[i]) {
1403  if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
1404  KnownZero.setBit(i);
1405  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
1406  KnownUndef.setBit(i);
1407  }
1408  }
1409  }
1410  break;
1411  }
1412  case ISD::BUILD_VECTOR: {
1413  // Check all elements and simplify any unused elements with UNDEF.
1414  if (!DemandedElts.isAllOnesValue()) {
1415  // Don't simplify BROADCASTS.
1416  if (llvm::any_of(Op->op_values(),
1417  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
1418  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
1419  bool Updated = false;
1420  for (unsigned i = 0; i != NumElts; ++i) {
1421  if (!DemandedElts[i] && !Ops[i].isUndef()) {
1422  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
1423  KnownUndef.setBit(i);
1424  Updated = true;
1425  }
1426  }
1427  if (Updated)
1428  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
1429  }
1430  }
1431  for (unsigned i = 0; i != NumElts; ++i) {
1432  SDValue SrcOp = Op.getOperand(i);
1433  if (SrcOp.isUndef()) {
1434  KnownUndef.setBit(i);
1435  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
1436  (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
1437  KnownZero.setBit(i);
1438  }
1439  }
1440  break;
1441  }
1442  case ISD::CONCAT_VECTORS: {
1443  EVT SubVT = Op.getOperand(0).getValueType();
1444  unsigned NumSubVecs = Op.getNumOperands();
1445  unsigned NumSubElts = SubVT.getVectorNumElements();
1446  for (unsigned i = 0; i != NumSubVecs; ++i) {
1447  SDValue SubOp = Op.getOperand(i);
1448  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1449  APInt SubUndef, SubZero;
1450  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
1451  Depth + 1))
1452  return true;
1453  KnownUndef.insertBits(SubUndef, i * NumSubElts);
1454  KnownZero.insertBits(SubZero, i * NumSubElts);
1455  }
1456  break;
1457  }
1458  case ISD::INSERT_SUBVECTOR: {
1459  if (!isa<ConstantSDNode>(Op.getOperand(2)))
1460  break;
1461  SDValue Base = Op.getOperand(0);
1462  SDValue Sub = Op.getOperand(1);
1463  EVT SubVT = Sub.getValueType();
1464  unsigned NumSubElts = SubVT.getVectorNumElements();
1465  const APInt& Idx = cast<ConstantSDNode>(Op.getOperand(2))->getAPIntValue();
1466  if (Idx.uge(NumElts - NumSubElts))
1467  break;
1468  unsigned SubIdx = Idx.getZExtValue();
1469  APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
1470  APInt SubUndef, SubZero;
1471  if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
1472  Depth + 1))
1473  return true;
1474  APInt BaseElts = DemandedElts;
1475  BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
1476  if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
1477  Depth + 1))
1478  return true;
1479  KnownUndef.insertBits(SubUndef, SubIdx);
1480  KnownZero.insertBits(SubZero, SubIdx);
1481  break;
1482  }
1483  case ISD::EXTRACT_SUBVECTOR: {
1484  SDValue Src = Op.getOperand(0);
1486  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1487  if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
1488  // Offset the demanded elts by the subvector index.
1489  uint64_t Idx = SubIdx->getZExtValue();
1490  APInt SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1491  APInt SrcUndef, SrcZero;
1492  if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
1493  Depth + 1))
1494  return true;
1495  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
1496  KnownZero = SrcZero.extractBits(NumElts, Idx);
1497  }
1498  break;
1499  }
1500  case ISD::INSERT_VECTOR_ELT: {
1501  SDValue Vec = Op.getOperand(0);
1502  SDValue Scl = Op.getOperand(1);
1503  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1504 
1505  // For a legal, constant insertion index, if we don't need this insertion
1506  // then strip it, else remove it from the demanded elts.
1507  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
1508  unsigned Idx = CIdx->getZExtValue();
1509  if (!DemandedElts[Idx])
1510  return TLO.CombineTo(Op, Vec);
1511  DemandedElts.clearBit(Idx);
1512 
1513  if (SimplifyDemandedVectorElts(Vec, DemandedElts, KnownUndef,
1514  KnownZero, TLO, Depth + 1))
1515  return true;
1516 
1517  KnownUndef.clearBit(Idx);
1518  if (Scl.isUndef())
1519  KnownUndef.setBit(Idx);
1520 
1521  KnownZero.clearBit(Idx);
1522  if (isNullConstant(Scl) || isNullFPConstant(Scl))
1523  KnownZero.setBit(Idx);
1524  break;
1525  }
1526 
1527  APInt VecUndef, VecZero;
1528  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
1529  Depth + 1))
1530  return true;
1531  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
1532  break;
1533  }
1534  case ISD::VSELECT: {
1535  // Try to transform the select condition based on the current demanded
1536  // elements.
1537  // TODO: If a condition element is undef, we can choose from one arm of the
1538  // select (and if one arm is undef, then we can propagate that to the
1539  // result).
1540  // TODO - add support for constant vselect masks (see IR version of this).
1541  APInt UnusedUndef, UnusedZero;
1542  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
1543  UnusedZero, TLO, Depth + 1))
1544  return true;
1545 
1546  // See if we can simplify either vselect operand.
1547  APInt DemandedLHS(DemandedElts);
1548  APInt DemandedRHS(DemandedElts);
1549  APInt UndefLHS, ZeroLHS;
1550  APInt UndefRHS, ZeroRHS;
1551  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
1552  ZeroLHS, TLO, Depth + 1))
1553  return true;
1554  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
1555  ZeroRHS, TLO, Depth + 1))
1556  return true;
1557 
1558  KnownUndef = UndefLHS & UndefRHS;
1559  KnownZero = ZeroLHS & ZeroRHS;
1560  break;
1561  }
1562  case ISD::VECTOR_SHUFFLE: {
1563  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1564 
1565  // Collect demanded elements from shuffle operands..
1566  APInt DemandedLHS(NumElts, 0);
1567  APInt DemandedRHS(NumElts, 0);
1568  for (unsigned i = 0; i != NumElts; ++i) {
1569  int M = ShuffleMask[i];
1570  if (M < 0 || !DemandedElts[i])
1571  continue;
1572  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1573  if (M < (int)NumElts)
1574  DemandedLHS.setBit(M);
1575  else
1576  DemandedRHS.setBit(M - NumElts);
1577  }
1578 
1579  // See if we can simplify either shuffle operand.
1580  APInt UndefLHS, ZeroLHS;
1581  APInt UndefRHS, ZeroRHS;
1582  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
1583  ZeroLHS, TLO, Depth + 1))
1584  return true;
1585  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
1586  ZeroRHS, TLO, Depth + 1))
1587  return true;
1588 
1589  // Simplify mask using undef elements from LHS/RHS.
1590  bool Updated = false;
1591  bool IdentityLHS = true, IdentityRHS = true;
1592  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
1593  for (unsigned i = 0; i != NumElts; ++i) {
1594  int &M = NewMask[i];
1595  if (M < 0)
1596  continue;
1597  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
1598  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
1599  Updated = true;
1600  M = -1;
1601  }
1602  IdentityLHS &= (M < 0) || (M == (int)i);
1603  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
1604  }
1605 
1606  // Update legal shuffle masks based on demanded elements if it won't reduce
1607  // to Identity which can cause premature removal of the shuffle mask.
1608  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps &&
1609  isShuffleMaskLegal(NewMask, VT))
1610  return TLO.CombineTo(Op,
1611  TLO.DAG.getVectorShuffle(VT, DL, Op.getOperand(0),
1612  Op.getOperand(1), NewMask));
1613 
1614  // Propagate undef/zero elements from LHS/RHS.
1615  for (unsigned i = 0; i != NumElts; ++i) {
1616  int M = ShuffleMask[i];
1617  if (M < 0) {
1618  KnownUndef.setBit(i);
1619  } else if (M < (int)NumElts) {
1620  if (UndefLHS[M])
1621  KnownUndef.setBit(i);
1622  if (ZeroLHS[M])
1623  KnownZero.setBit(i);
1624  } else {
1625  if (UndefRHS[M - NumElts])
1626  KnownUndef.setBit(i);
1627  if (ZeroRHS[M - NumElts])
1628  KnownZero.setBit(i);
1629  }
1630  }
1631  break;
1632  }
1633  case ISD::ADD:
1634  case ISD::SUB: {
1635  APInt SrcUndef, SrcZero;
1636  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
1637  SrcZero, TLO, Depth + 1))
1638  return true;
1639  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1640  KnownZero, TLO, Depth + 1))
1641  return true;
1642  KnownZero &= SrcZero;
1643  KnownUndef &= SrcUndef;
1644  break;
1645  }
1646  case ISD::TRUNCATE:
1647  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1648  KnownZero, TLO, Depth + 1))
1649  return true;
1650  break;
1651  default: {
1652  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1653  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
1654  KnownZero, TLO, Depth))
1655  return true;
1656  break;
1657  }
1658  }
1659 
1660  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
1661  return false;
1662 }
1663 
1664 /// Determine which of the bits specified in Mask are known to be either zero or
1665 /// one and return them in the Known.
1667  KnownBits &Known,
1668  const APInt &DemandedElts,
1669  const SelectionDAG &DAG,
1670  unsigned Depth) const {
1674  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1675  "Should use MaskedValueIsZero if you don't know whether Op"
1676  " is a target node!");
1677  Known.resetAll();
1678 }
1679 
1681  KnownBits &Known,
1682  const APInt &DemandedElts,
1683  const SelectionDAG &DAG,
1684  unsigned Depth) const {
1685  assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex");
1686 
1687  if (unsigned Align = DAG.InferPtrAlignment(Op)) {
1688  // The low bits are known zero if the pointer is aligned.
1689  Known.Zero.setLowBits(Log2_32(Align));
1690  }
1691 }
1692 
1693 /// This method can be implemented by targets that want to expose additional
1694 /// information about sign bits to the DAG Combiner.
1696  const APInt &,
1697  const SelectionDAG &,
1698  unsigned Depth) const {
1702  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1703  "Should use ComputeNumSignBits if you don't know whether Op"
1704  " is a target node!");
1705  return 1;
1706 }
1707 
1709  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
1710  TargetLoweringOpt &TLO, unsigned Depth) const {
1714  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1715  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
1716  " is a target node!");
1717  return false;
1718 }
1719 
1721  const SelectionDAG &DAG,
1722  bool SNaN,
1723  unsigned Depth) const {
1727  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1728  "Should use isKnownNeverNaN if you don't know whether Op"
1729  " is a target node!");
1730  return false;
1731 }
1732 
1733 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
1734 // work with truncating build vectors and vectors with elements of less than
1735 // 8 bits.
1737  if (!N)
1738  return false;
1739 
1740  APInt CVal;
1741  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
1742  CVal = CN->getAPIntValue();
1743  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
1744  auto *CN = BV->getConstantSplatNode();
1745  if (!CN)
1746  return false;
1747 
1748  // If this is a truncating build vector, truncate the splat value.
1749  // Otherwise, we may fail to match the expected values below.
1750  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
1751  CVal = CN->getAPIntValue();
1752  if (BVEltWidth < CVal.getBitWidth())
1753  CVal = CVal.trunc(BVEltWidth);
1754  } else {
1755  return false;
1756  }
1757 
1758  switch (getBooleanContents(N->getValueType(0))) {
1760  return CVal[0];
1762  return CVal.isOneValue();
1764  return CVal.isAllOnesValue();
1765  }
1766 
1767  llvm_unreachable("Invalid boolean contents");
1768 }
1769 
1771  if (!N)
1772  return false;
1773 
1774  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
1775  if (!CN) {
1777  if (!BV)
1778  return false;
1779 
1780  // Only interested in constant splats, we don't care about undef
1781  // elements in identifying boolean constants and getConstantSplatNode
1782  // returns NULL if all ops are undef;
1783  CN = BV->getConstantSplatNode();
1784  if (!CN)
1785  return false;
1786  }
1787 
1789  return !CN->getAPIntValue()[0];
1790 
1791  return CN->isNullValue();
1792 }
1793 
1795  bool SExt) const {
1796  if (VT == MVT::i1)
1797  return N->isOne();
1798 
1800  switch (Cnt) {
1802  // An extended value of 1 is always true, unless its original type is i1,
1803  // in which case it will be sign extended to -1.
1804  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
1807  return N->isAllOnesValue() && SExt;
1808  }
1809  llvm_unreachable("Unexpected enumeration.");
1810 }
1811 
1812 /// This helper function of SimplifySetCC tries to optimize the comparison when
1813 /// either operand of the SetCC node is a bitwise-and instruction.
1814 SDValue TargetLowering::simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
1815  ISD::CondCode Cond,
1816  DAGCombinerInfo &DCI,
1817  const SDLoc &DL) const {
1818  // Match these patterns in any of their permutations:
1819  // (X & Y) == Y
1820  // (X & Y) != Y
1821  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
1822  std::swap(N0, N1);
1823 
1824  EVT OpVT = N0.getValueType();
1825  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
1826  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
1827  return SDValue();
1828 
1829  SDValue X, Y;
1830  if (N0.getOperand(0) == N1) {
1831  X = N0.getOperand(1);
1832  Y = N0.getOperand(0);
1833  } else if (N0.getOperand(1) == N1) {
1834  X = N0.getOperand(0);
1835  Y = N0.getOperand(1);
1836  } else {
1837  return SDValue();
1838  }
1839 
1840  SelectionDAG &DAG = DCI.DAG;
1841  SDValue Zero = DAG.getConstant(0, DL, OpVT);
1842  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
1843  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
1844  // Note that where Y is variable and is known to have at most one bit set
1845  // (for example, if it is Z & 1) we cannot do this; the expressions are not
1846  // equivalent when Y == 0.
1847  Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
1848  if (DCI.isBeforeLegalizeOps() ||
1849  isCondCodeLegal(Cond, N0.getSimpleValueType()))
1850  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
1851  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
1852  // If the target supports an 'and-not' or 'and-complement' logic operation,
1853  // try to use that to make a comparison operation more efficient.
1854  // But don't do this transform if the mask is a single bit because there are
1855  // more efficient ways to deal with that case (for example, 'bt' on x86 or
1856  // 'rlwinm' on PPC).
1857 
1858  // Bail out if the compare operand that we want to turn into a zero is
1859  // already a zero (otherwise, infinite loop).
1860  auto *YConst = dyn_cast<ConstantSDNode>(Y);
1861  if (YConst && YConst->isNullValue())
1862  return SDValue();
1863 
1864  // Transform this into: ~X & Y == 0.
1865  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
1866  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
1867  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
1868  }
1869 
1870  return SDValue();
1871 }
1872 
1873 /// There are multiple IR patterns that could be checking whether certain
1874 /// truncation of a signed number would be lossy or not. The pattern which is
1875 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
1876 /// We are looking for the following pattern: (KeptBits is a constant)
1877 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
1878 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
1879 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
1880 /// We will unfold it into the natural trunc+sext pattern:
1881 /// ((%x << C) a>> C) dstcond %x
1882 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
1883 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
1884  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
1885  const SDLoc &DL) const {
1886  // We must be comparing with a constant.
1887  ConstantSDNode *C1;
1888  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
1889  return SDValue();
1890 
1891  // N0 should be: add %x, (1 << (KeptBits-1))
1892  if (N0->getOpcode() != ISD::ADD)
1893  return SDValue();
1894 
1895  // And we must be 'add'ing a constant.
1896  ConstantSDNode *C01;
1897  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
1898  return SDValue();
1899 
1900  SDValue X = N0->getOperand(0);
1901  EVT XVT = X.getValueType();
1902 
1903  // Validate constants ...
1904 
1905  APInt I1 = C1->getAPIntValue();
1906 
1907  ISD::CondCode NewCond;
1908  if (Cond == ISD::CondCode::SETULT) {
1909  NewCond = ISD::CondCode::SETEQ;
1910  } else if (Cond == ISD::CondCode::SETULE) {
1911  NewCond = ISD::CondCode::SETEQ;
1912  // But need to 'canonicalize' the constant.
1913  I1 += 1;
1914  } else if (Cond == ISD::CondCode::SETUGT) {
1915  NewCond = ISD::CondCode::SETNE;
1916  // But need to 'canonicalize' the constant.
1917  I1 += 1;
1918  } else if (Cond == ISD::CondCode::SETUGE) {
1919  NewCond = ISD::CondCode::SETNE;
1920  } else
1921  return SDValue();
1922 
1923  APInt I01 = C01->getAPIntValue();
1924 
1925  auto checkConstants = [&I1, &I01]() -> bool {
1926  // Both of them must be power-of-two, and the constant from setcc is bigger.
1927  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
1928  };
1929 
1930  if (checkConstants()) {
1931  // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
1932  } else {
1933  // What if we invert constants? (and the target predicate)
1934  I1.negate();
1935  I01.negate();
1936  NewCond = getSetCCInverse(NewCond, /*isInteger=*/true);
1937  if (!checkConstants())
1938  return SDValue();
1939  // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
1940  }
1941 
1942  // They are power-of-two, so which bit is set?
1943  const unsigned KeptBits = I1.logBase2();
1944  const unsigned KeptBitsMinusOne = I01.logBase2();
1945 
1946  // Magic!
1947  if (KeptBits != (KeptBitsMinusOne + 1))
1948  return SDValue();
1949  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
1950 
1951  // We don't want to do this in every single case.
1952  SelectionDAG &DAG = DCI.DAG;
1953  if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
1954  XVT, KeptBits))
1955  return SDValue();
1956 
1957  const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
1958  assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
1959 
1960  // Unfold into: ((%x << C) a>> C) cond %x
1961  // Where 'cond' will be either 'eq' or 'ne'.
1962  SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
1963  SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
1964  SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
1965  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
1966 
1967  return T2;
1968 }
1969 
1970 /// Try to simplify a setcc built with the specified operands and cc. If it is
1971 /// unable to simplify it, return a null SDValue.
1973  ISD::CondCode Cond, bool foldBooleans,
1974  DAGCombinerInfo &DCI,
1975  const SDLoc &dl) const {
1976  SelectionDAG &DAG = DCI.DAG;
1977  EVT OpVT = N0.getValueType();
1978 
1979  // These setcc operations always fold.
1980  switch (Cond) {
1981  default: break;
1982  case ISD::SETFALSE:
1983  case ISD::SETFALSE2: return DAG.getBoolConstant(false, dl, VT, OpVT);
1984  case ISD::SETTRUE:
1985  case ISD::SETTRUE2: return DAG.getBoolConstant(true, dl, VT, OpVT);
1986  }
1987 
1988  // Ensure that the constant occurs on the RHS and fold constant comparisons.
1989  // TODO: Handle non-splat vector constants. All undef causes trouble.
1990  ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
1991  if (isConstOrConstSplat(N0) &&
1992  (DCI.isBeforeLegalizeOps() ||
1993  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
1994  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
1995 
1996  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
1997  const APInt &C1 = N1C->getAPIntValue();
1998 
1999  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
2000  // equality comparison, then we're just comparing whether X itself is
2001  // zero.
2002  if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
2003  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
2004  N0.getOperand(1).getOpcode() == ISD::Constant) {
2005  const APInt &ShAmt
2006  = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
2007  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2008  ShAmt == Log2_32(N0.getValueSizeInBits())) {
2009  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
2010  // (srl (ctlz x), 5) == 0 -> X != 0
2011  // (srl (ctlz x), 5) != 1 -> X != 0
2012  Cond = ISD::SETNE;
2013  } else {
2014  // (srl (ctlz x), 5) != 0 -> X == 0
2015  // (srl (ctlz x), 5) == 1 -> X == 0
2016  Cond = ISD::SETEQ;
2017  }
2018  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
2019  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
2020  Zero, Cond);
2021  }
2022  }
2023 
2024  SDValue CTPOP = N0;
2025  // Look through truncs that don't change the value of a ctpop.
2026  if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
2027  CTPOP = N0.getOperand(0);
2028 
2029  if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
2030  (N0 == CTPOP ||
2032  EVT CTVT = CTPOP.getValueType();
2033  SDValue CTOp = CTPOP.getOperand(0);
2034 
2035  // (ctpop x) u< 2 -> (x & x-1) == 0
2036  // (ctpop x) u> 1 -> (x & x-1) != 0
2037  if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
2038  SDValue Sub = DAG.getNode(ISD::SUB, dl, CTVT, CTOp,
2039  DAG.getConstant(1, dl, CTVT));
2040  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Sub);
2042  return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
2043  }
2044 
2045  // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal.
2046  }
2047 
2048  // (zext x) == C --> x == (trunc C)
2049  // (sext x) == C --> x == (trunc C)
2050  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2051  DCI.isBeforeLegalize() && N0->hasOneUse()) {
2052  unsigned MinBits = N0.getValueSizeInBits();
2053  SDValue PreExt;
2054  bool Signed = false;
2055  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
2056  // ZExt
2057  MinBits = N0->getOperand(0).getValueSizeInBits();
2058  PreExt = N0->getOperand(0);
2059  } else if (N0->getOpcode() == ISD::AND) {
2060  // DAGCombine turns costly ZExts into ANDs
2061  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
2062  if ((C->getAPIntValue()+1).isPowerOf2()) {
2063  MinBits = C->getAPIntValue().countTrailingOnes();
2064  PreExt = N0->getOperand(0);
2065  }
2066  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
2067  // SExt
2068  MinBits = N0->getOperand(0).getValueSizeInBits();
2069  PreExt = N0->getOperand(0);
2070  Signed = true;
2071  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
2072  // ZEXTLOAD / SEXTLOAD
2073  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
2074  MinBits = LN0->getMemoryVT().getSizeInBits();
2075  PreExt = N0;
2076  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
2077  Signed = true;
2078  MinBits = LN0->getMemoryVT().getSizeInBits();
2079  PreExt = N0;
2080  }
2081  }
2082 
2083  // Figure out how many bits we need to preserve this constant.
2084  unsigned ReqdBits = Signed ?
2085  C1.getBitWidth() - C1.getNumSignBits() + 1 :
2086  C1.getActiveBits();
2087 
2088  // Make sure we're not losing bits from the constant.
2089  if (MinBits > 0 &&
2090  MinBits < C1.getBitWidth() &&
2091  MinBits >= ReqdBits) {
2092  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
2093  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
2094  // Will get folded away.
2095  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
2096  if (MinBits == 1 && C1 == 1)
2097  // Invert the condition.
2098  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
2099  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2100  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
2101  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
2102  }
2103 
2104  // If truncating the setcc operands is not desirable, we can still
2105  // simplify the expression in some cases:
2106  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
2107  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
2108  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
2109  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
2110  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
2111  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
2112  SDValue TopSetCC = N0->getOperand(0);
2113  unsigned N0Opc = N0->getOpcode();
2114  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
2115  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
2116  TopSetCC.getOpcode() == ISD::SETCC &&
2117  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
2118  (isConstFalseVal(N1C) ||
2119  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
2120 
2121  bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
2122  (!N1C->isNullValue() && Cond == ISD::SETNE);
2123 
2124  if (!Inverse)
2125  return TopSetCC;
2126 
2128  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
2129  TopSetCC.getOperand(0).getValueType().isInteger());
2130  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
2131  TopSetCC.getOperand(1),
2132  InvCond);
2133  }
2134  }
2135  }
2136 
2137  // If the LHS is '(and load, const)', the RHS is 0, the test is for
2138  // equality or unsigned, and all 1 bits of the const are in the same
2139  // partial word, see if we can shorten the load.
2140  if (DCI.isBeforeLegalize() &&
2141  !ISD::isSignedIntSetCC(Cond) &&
2142  N0.getOpcode() == ISD::AND && C1 == 0 &&
2143  N0.getNode()->hasOneUse() &&
2144  isa<LoadSDNode>(N0.getOperand(0)) &&
2145  N0.getOperand(0).getNode()->hasOneUse() &&
2146  isa<ConstantSDNode>(N0.getOperand(1))) {
2147  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
2148  APInt bestMask;
2149  unsigned bestWidth = 0, bestOffset = 0;
2150  if (!Lod->isVolatile() && Lod->isUnindexed()) {
2151  unsigned origWidth = N0.getValueSizeInBits();
2152  unsigned maskWidth = origWidth;
2153  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
2154  // 8 bits, but have to be careful...
2155  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
2156  origWidth = Lod->getMemoryVT().getSizeInBits();
2157  const APInt &Mask =
2158  cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
2159  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
2160  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
2161  for (unsigned offset=0; offset<origWidth/width; offset++) {
2162  if (Mask.isSubsetOf(newMask)) {
2163  if (DAG.getDataLayout().isLittleEndian())
2164  bestOffset = (uint64_t)offset * (width/8);
2165  else
2166  bestOffset = (origWidth/width - offset - 1) * (width/8);
2167  bestMask = Mask.lshr(offset * (width/8) * 8);
2168  bestWidth = width;
2169  break;
2170  }
2171  newMask <<= width;
2172  }
2173  }
2174  }
2175  if (bestWidth) {
2176  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
2177  if (newVT.isRound()) {
2178  EVT PtrType = Lod->getOperand(1).getValueType();
2179  SDValue Ptr = Lod->getBasePtr();
2180  if (bestOffset != 0)
2181  Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
2182  DAG.getConstant(bestOffset, dl, PtrType));
2183  unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
2184  SDValue NewLoad = DAG.getLoad(
2185  newVT, dl, Lod->getChain(), Ptr,
2186  Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
2187  return DAG.getSetCC(dl, VT,
2188  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
2189  DAG.getConstant(bestMask.trunc(bestWidth),
2190  dl, newVT)),
2191  DAG.getConstant(0LL, dl, newVT), Cond);
2192  }
2193  }
2194  }
2195 
2196  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
2197  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
2198  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
2199 
2200  // If the comparison constant has bits in the upper part, the
2201  // zero-extended value could never match.
2202  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
2203  C1.getBitWidth() - InSize))) {
2204  switch (Cond) {
2205  case ISD::SETUGT:
2206  case ISD::SETUGE:
2207  case ISD::SETEQ:
2208  return DAG.getConstant(0, dl, VT);
2209  case ISD::SETULT:
2210  case ISD::SETULE:
2211  case ISD::SETNE:
2212  return DAG.getConstant(1, dl, VT);
2213  case ISD::SETGT:
2214  case ISD::SETGE:
2215  // True if the sign bit of C1 is set.
2216  return DAG.getConstant(C1.isNegative(), dl, VT);
2217  case ISD::SETLT:
2218  case ISD::SETLE:
2219  // True if the sign bit of C1 isn't set.
2220  return DAG.getConstant(C1.isNonNegative(), dl, VT);
2221  default:
2222  break;
2223  }
2224  }
2225 
2226  // Otherwise, we can perform the comparison with the low bits.
2227  switch (Cond) {
2228  case ISD::SETEQ:
2229  case ISD::SETNE:
2230  case ISD::SETUGT:
2231  case ISD::SETUGE:
2232  case ISD::SETULT:
2233  case ISD::SETULE: {
2234  EVT newVT = N0.getOperand(0).getValueType();
2235  if (DCI.isBeforeLegalizeOps() ||
2236  (isOperationLegal(ISD::SETCC, newVT) &&
2237  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
2238  EVT NewSetCCVT =
2239  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), newVT);
2240  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
2241 
2242  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
2243  NewConst, Cond);
2244  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
2245  }
2246  break;
2247  }
2248  default:
2249  break; // todo, be more careful with signed comparisons
2250  }
2251  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2252  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2253  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
2254  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
2255  EVT ExtDstTy = N0.getValueType();
2256  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
2257 
2258  // If the constant doesn't fit into the number of bits for the source of
2259  // the sign extension, it is impossible for both sides to be equal.
2260  if (C1.getMinSignedBits() > ExtSrcTyBits)
2261  return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
2262 
2263  SDValue ZextOp;
2264  EVT Op0Ty = N0.getOperand(0).getValueType();
2265  if (Op0Ty == ExtSrcTy) {
2266  ZextOp = N0.getOperand(0);
2267  } else {
2268  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
2269  ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
2270  DAG.getConstant(Imm, dl, Op0Ty));
2271  }
2272  if (!DCI.isCalledByLegalizer())
2273  DCI.AddToWorklist(ZextOp.getNode());
2274  // Otherwise, make this a use of a zext.
2275  return DAG.getSetCC(dl, VT, ZextOp,
2277  ExtDstTyBits,
2278  ExtSrcTyBits),
2279  dl, ExtDstTy),
2280  Cond);
2281  } else if ((N1C->isNullValue() || N1C->isOne()) &&
2282  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2283  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
2284  if (N0.getOpcode() == ISD::SETCC &&
2285  isTypeLegal(VT) && VT.bitsLE(N0.getValueType())) {
2286  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
2287  if (TrueWhenTrue)
2288  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
2289  // Invert the condition.
2290  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
2291  CC = ISD::getSetCCInverse(CC,
2292  N0.getOperand(0).getValueType().isInteger());
2293  if (DCI.isBeforeLegalizeOps() ||
2294  isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
2295  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
2296  }
2297 
2298  if ((N0.getOpcode() == ISD::XOR ||
2299  (N0.getOpcode() == ISD::AND &&
2300  N0.getOperand(0).getOpcode() == ISD::XOR &&
2301  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
2302  isa<ConstantSDNode>(N0.getOperand(1)) &&
2303  cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
2304  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
2305  // can only do this if the top bits are known zero.
2306  unsigned BitWidth = N0.getValueSizeInBits();
2307  if (DAG.MaskedValueIsZero(N0,
2308  APInt::getHighBitsSet(BitWidth,
2309  BitWidth-1))) {
2310  // Okay, get the un-inverted input value.
2311  SDValue Val;
2312  if (N0.getOpcode() == ISD::XOR) {
2313  Val = N0.getOperand(0);
2314  } else {
2315  assert(N0.getOpcode() == ISD::AND &&
2316  N0.getOperand(0).getOpcode() == ISD::XOR);
2317  // ((X^1)&1)^1 -> X & 1
2318  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
2319  N0.getOperand(0).getOperand(0),
2320  N0.getOperand(1));
2321  }
2322 
2323  return DAG.getSetCC(dl, VT, Val, N1,
2324  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2325  }
2326  } else if (N1C->isOne() &&
2327  (VT == MVT::i1 ||
2328  getBooleanContents(N0->getValueType(0)) ==
2330  SDValue Op0 = N0;
2331  if (Op0.getOpcode() == ISD::TRUNCATE)
2332  Op0 = Op0.getOperand(0);
2333 
2334  if ((Op0.getOpcode() == ISD::XOR) &&
2335  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
2336  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
2337  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
2338  Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
2339  return DAG.getSetCC(dl, VT, Op0.getOperand(0), Op0.getOperand(1),
2340  Cond);
2341  }
2342  if (Op0.getOpcode() == ISD::AND &&
2343  isa<ConstantSDNode>(Op0.getOperand(1)) &&
2344  cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
2345  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
2346  if (Op0.getValueType().bitsGT(VT))
2347  Op0 = DAG.getNode(ISD::AND, dl, VT,
2348  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
2349  DAG.getConstant(1, dl, VT));
2350  else if (Op0.getValueType().bitsLT(VT))
2351  Op0 = DAG.getNode(ISD::AND, dl, VT,
2352  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
2353  DAG.getConstant(1, dl, VT));
2354 
2355  return DAG.getSetCC(dl, VT, Op0,
2356  DAG.getConstant(0, dl, Op0.getValueType()),
2357  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2358  }
2359  if (Op0.getOpcode() == ISD::AssertZext &&
2360  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
2361  return DAG.getSetCC(dl, VT, Op0,
2362  DAG.getConstant(0, dl, Op0.getValueType()),
2363  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2364  }
2365  }
2366 
2367  if (SDValue V =
2368  optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
2369  return V;
2370  }
2371 
2372  // These simplifications apply to splat vectors as well.
2373  // TODO: Handle more splat vector cases.
2374  if (auto *N1C = isConstOrConstSplat(N1)) {
2375  const APInt &C1 = N1C->getAPIntValue();
2376 
2377  APInt MinVal, MaxVal;
2378  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
2379  if (ISD::isSignedIntSetCC(Cond)) {
2380  MinVal = APInt::getSignedMinValue(OperandBitSize);
2381  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
2382  } else {
2383  MinVal = APInt::getMinValue(OperandBitSize);
2384  MaxVal = APInt::getMaxValue(OperandBitSize);
2385  }
2386 
2387  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
2388  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
2389  // X >= MIN --> true
2390  if (C1 == MinVal)
2391  return DAG.getBoolConstant(true, dl, VT, OpVT);
2392 
2393  if (!VT.isVector()) { // TODO: Support this for vectors.
2394  // X >= C0 --> X > (C0 - 1)
2395  APInt C = C1 - 1;
2396  ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
2397  if ((DCI.isBeforeLegalizeOps() ||
2398  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2399  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2401  return DAG.getSetCC(dl, VT, N0,
2402  DAG.getConstant(C, dl, N1.getValueType()),
2403  NewCC);
2404  }
2405  }
2406  }
2407 
2408  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
2409  // X <= MAX --> true
2410  if (C1 == MaxVal)
2411  return DAG.getBoolConstant(true, dl, VT, OpVT);
2412 
2413  // X <= C0 --> X < (C0 + 1)
2414  if (!VT.isVector()) { // TODO: Support this for vectors.
2415  APInt C = C1 + 1;
2416  ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
2417  if ((DCI.isBeforeLegalizeOps() ||
2418  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2419  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2421  return DAG.getSetCC(dl, VT, N0,
2422  DAG.getConstant(C, dl, N1.getValueType()),
2423  NewCC);
2424  }
2425  }
2426  }
2427 
2428  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
2429  if (C1 == MinVal)
2430  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
2431 
2432  // TODO: Support this for vectors after legalize ops.
2433  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2434  // Canonicalize setlt X, Max --> setne X, Max
2435  if (C1 == MaxVal)
2436  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2437 
2438  // If we have setult X, 1, turn it into seteq X, 0
2439  if (C1 == MinVal+1)
2440  return DAG.getSetCC(dl, VT, N0,
2441  DAG.getConstant(MinVal, dl, N0.getValueType()),
2442  ISD::SETEQ);
2443  }
2444  }
2445 
2446  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
2447  if (C1 == MaxVal)
2448  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
2449 
2450  // TODO: Support this for vectors after legalize ops.
2451  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2452  // Canonicalize setgt X, Min --> setne X, Min
2453  if (C1 == MinVal)
2454  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2455 
2456  // If we have setugt X, Max-1, turn it into seteq X, Max
2457  if (C1 == MaxVal-1)
2458  return DAG.getSetCC(dl, VT, N0,
2459  DAG.getConstant(MaxVal, dl, N0.getValueType()),
2460  ISD::SETEQ);
2461  }
2462  }
2463 
2464  // If we have "setcc X, C0", check to see if we can shrink the immediate
2465  // by changing cc.
2466  // TODO: Support this for vectors after legalize ops.
2467  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2468  // SETUGT X, SINTMAX -> SETLT X, 0
2469  if (Cond == ISD::SETUGT &&
2470  C1 == APInt::getSignedMaxValue(OperandBitSize))
2471  return DAG.getSetCC(dl, VT, N0,
2472  DAG.getConstant(0, dl, N1.getValueType()),
2473  ISD::SETLT);
2474 
2475  // SETULT X, SINTMIN -> SETGT X, -1
2476  if (Cond == ISD::SETULT &&
2477  C1 == APInt::getSignedMinValue(OperandBitSize)) {
2478  SDValue ConstMinusOne =
2479  DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
2480  N1.getValueType());
2481  return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
2482  }
2483  }
2484  }
2485 
2486  // Back to non-vector simplifications.
2487  // TODO: Can we do these for vector splats?
2488  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
2489  const APInt &C1 = N1C->getAPIntValue();
2490 
2491  // Fold bit comparisons when we can.
2492  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2493  (VT == N0.getValueType() ||
2494  (isTypeLegal(VT) && VT.bitsLE(N0.getValueType()))) &&
2495  N0.getOpcode() == ISD::AND) {
2496  auto &DL = DAG.getDataLayout();
2497  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2498  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2499  !DCI.isBeforeLegalize());
2500  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
2501  // Perform the xform if the AND RHS is a single bit.
2502  if (AndRHS->getAPIntValue().isPowerOf2()) {
2503  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2504  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2505  DAG.getConstant(AndRHS->getAPIntValue().logBase2(), dl,
2506  ShiftTy)));
2507  }
2508  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
2509  // (X & 8) == 8 --> (X & 8) >> 3
2510  // Perform the xform if C1 is a single bit.
2511  if (C1.isPowerOf2()) {
2512  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2513  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2514  DAG.getConstant(C1.logBase2(), dl,
2515  ShiftTy)));
2516  }
2517  }
2518  }
2519  }
2520 
2521  if (C1.getMinSignedBits() <= 64 &&
2523  // (X & -256) == 256 -> (X >> 8) == 1
2524  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2525  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
2526  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2527  const APInt &AndRHSC = AndRHS->getAPIntValue();
2528  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
2529  unsigned ShiftBits = AndRHSC.countTrailingZeros();
2530  auto &DL = DAG.getDataLayout();
2531  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2532  !DCI.isBeforeLegalize());
2533  EVT CmpTy = N0.getValueType();
2534  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
2535  DAG.getConstant(ShiftBits, dl,
2536  ShiftTy));
2537  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
2538  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
2539  }
2540  }
2541  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
2542  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
2543  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
2544  // X < 0x100000000 -> (X >> 32) < 1
2545  // X >= 0x100000000 -> (X >> 32) >= 1
2546  // X <= 0x0ffffffff -> (X >> 32) < 1
2547  // X > 0x0ffffffff -> (X >> 32) >= 1
2548  unsigned ShiftBits;
2549  APInt NewC = C1;
2550  ISD::CondCode NewCond = Cond;
2551  if (AdjOne) {
2552  ShiftBits = C1.countTrailingOnes();
2553  NewC = NewC + 1;
2554  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2555  } else {
2556  ShiftBits = C1.countTrailingZeros();
2557  }
2558  NewC.lshrInPlace(ShiftBits);
2559  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
2561  auto &DL = DAG.getDataLayout();
2562  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2563  !DCI.isBeforeLegalize());
2564  EVT CmpTy = N0.getValueType();
2565  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
2566  DAG.getConstant(ShiftBits, dl, ShiftTy));
2567  SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
2568  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
2569  }
2570  }
2571  }
2572  }
2573 
2574  if (isa<ConstantFPSDNode>(N0.getNode())) {
2575  // Constant fold or commute setcc.
2576  SDValue O = DAG.FoldSetCC(VT, N0, N1, Cond, dl);
2577  if (O.getNode()) return O;
2578  } else if (auto *CFP = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
2579  // If the RHS of an FP comparison is a constant, simplify it away in
2580  // some cases.
2581  if (CFP->getValueAPF().isNaN()) {
2582  // If an operand is known to be a nan, we can fold it.
2583  switch (ISD::getUnorderedFlavor(Cond)) {
2584  default: llvm_unreachable("Unknown flavor!");
2585  case 0: // Known false.
2586  return DAG.getBoolConstant(false, dl, VT, OpVT);
2587  case 1: // Known true.
2588  return DAG.getBoolConstant(true, dl, VT, OpVT);
2589  case 2: // Undefined.
2590  return DAG.getUNDEF(VT);
2591  }
2592  }
2593 
2594  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
2595  // constant if knowing that the operand is non-nan is enough. We prefer to
2596  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
2597  // materialize 0.0.
2598  if (Cond == ISD::SETO || Cond == ISD::SETUO)
2599  return DAG.getSetCC(dl, VT, N0, N0, Cond);
2600 
2601  // setcc (fneg x), C -> setcc swap(pred) x, -C
2602  if (N0.getOpcode() == ISD::FNEG) {
2604  if (DCI.isBeforeLegalizeOps() ||
2605  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
2606  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
2607  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
2608  }
2609  }
2610 
2611  // If the condition is not legal, see if we can find an equivalent one
2612  // which is legal.
2613  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
2614  // If the comparison was an awkward floating-point == or != and one of
2615  // the comparison operands is infinity or negative infinity, convert the
2616  // condition to a less-awkward <= or >=.
2617  if (CFP->getValueAPF().isInfinity()) {
2618  if (CFP->getValueAPF().isNegative()) {
2619  if (Cond == ISD::SETOEQ &&
2621  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
2622  if (Cond == ISD::SETUEQ &&
2624  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
2625  if (Cond == ISD::SETUNE &&
2627  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
2628  if (Cond == ISD::SETONE &&
2630  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
2631  } else {
2632  if (Cond == ISD::SETOEQ &&
2634  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
2635  if (Cond == ISD::SETUEQ &&
2637  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
2638  if (Cond == ISD::SETUNE &&
2640  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
2641  if (Cond == ISD::SETONE &&
2643  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
2644  }
2645  }
2646  }
2647  }
2648 
2649  if (N0 == N1) {
2650  // The sext(setcc()) => setcc() optimization relies on the appropriate
2651  // constant being emitted.
2652 
2653  bool EqTrue = ISD::isTrueWhenEqual(Cond);
2654 
2655  // We can always fold X == X for integer setcc's.
2656  if (N0.getValueType().isInteger())
2657  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2658 
2659  unsigned UOF = ISD::getUnorderedFlavor(Cond);
2660  if (UOF == 2) // FP operators that are undefined on NaNs.
2661  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2662  if (UOF == unsigned(EqTrue))
2663  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2664  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
2665  // if it is not already.
2666  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
2667  if (NewCond != Cond &&
2668  (DCI.isBeforeLegalizeOps() ||
2669  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
2670  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
2671  }
2672 
2673  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2674  N0.getValueType().isInteger()) {
2675  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
2676  N0.getOpcode() == ISD::XOR) {
2677  // Simplify (X+Y) == (X+Z) --> Y == Z
2678  if (N0.getOpcode() == N1.getOpcode()) {
2679  if (N0.getOperand(0) == N1.getOperand(0))
2680  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
2681  if (N0.getOperand(1) == N1.getOperand(1))
2682  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
2683  if (isCommutativeBinOp(N0.getOpcode())) {
2684  // If X op Y == Y op X, try other combinations.
2685  if (N0.getOperand(0) == N1.getOperand(1))
2686  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
2687  Cond);
2688  if (N0.getOperand(1) == N1.getOperand(0))
2689  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
2690  Cond);
2691  }
2692  }
2693 
2694  // If RHS is a legal immediate value for a compare instruction, we need
2695  // to be careful about increasing register pressure needlessly.
2696  bool LegalRHSImm = false;
2697 
2698  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
2699  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2700  // Turn (X+C1) == C2 --> X == C2-C1
2701  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
2702  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2703  DAG.getConstant(RHSC->getAPIntValue()-
2704  LHSR->getAPIntValue(),
2705  dl, N0.getValueType()), Cond);
2706  }
2707 
2708  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
2709  if (N0.getOpcode() == ISD::XOR)
2710  // If we know that all of the inverted bits are zero, don't bother
2711  // performing the inversion.
2712  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
2713  return
2714  DAG.getSetCC(dl, VT, N0.getOperand(0),
2715  DAG.getConstant(LHSR->getAPIntValue() ^
2716  RHSC->getAPIntValue(),
2717  dl, N0.getValueType()),
2718  Cond);
2719  }
2720 
2721  // Turn (C1-X) == C2 --> X == C1-C2
2722  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
2723  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
2724  return
2725  DAG.getSetCC(dl, VT, N0.getOperand(1),
2726  DAG.getConstant(SUBC->getAPIntValue() -
2727  RHSC->getAPIntValue(),
2728  dl, N0.getValueType()),
2729  Cond);
2730  }
2731  }
2732 
2733  // Could RHSC fold directly into a compare?
2734  if (RHSC->getValueType(0).getSizeInBits() <= 64)
2735  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
2736  }
2737 
2738  // Simplify (X+Z) == X --> Z == 0
2739  // Don't do this if X is an immediate that can fold into a cmp
2740  // instruction and X+Z has other uses. It could be an induction variable
2741  // chain, and the transform would increase register pressure.
2742  if (!LegalRHSImm || N0.getNode()->hasOneUse()) {
2743  if (N0.getOperand(0) == N1)
2744  return DAG.getSetCC(dl, VT, N0.getOperand(1),
2745  DAG.getConstant(0, dl, N0.getValueType()), Cond);
2746  if (N0.getOperand(1) == N1) {
2747  if (isCommutativeBinOp(N0.getOpcode()))
2748  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2749  DAG.getConstant(0, dl, N0.getValueType()),
2750  Cond);
2751  if (N0.getNode()->hasOneUse()) {
2752  assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
2753  auto &DL = DAG.getDataLayout();
2754  // (Z-X) == X --> Z == X<<1
2755  SDValue SH = DAG.getNode(
2756  ISD::SHL, dl, N1.getValueType(), N1,
2757  DAG.getConstant(1, dl,
2758  getShiftAmountTy(N1.getValueType(), DL,
2759  !DCI.isBeforeLegalize())));
2760  if (!DCI.isCalledByLegalizer())
2761  DCI.AddToWorklist(SH.getNode());
2762  return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond);
2763  }
2764  }
2765  }
2766  }
2767 
2768  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
2769  N1.getOpcode() == ISD::XOR) {
2770  // Simplify X == (X+Z) --> Z == 0
2771  if (N1.getOperand(0) == N0)
2772  return DAG.getSetCC(dl, VT, N1.getOperand(1),
2773  DAG.getConstant(0, dl, N1.getValueType()), Cond);
2774  if (N1.getOperand(1) == N0) {
2775  if (isCommutativeBinOp(N1.getOpcode()))
2776  return DAG.getSetCC(dl, VT, N1.getOperand(0),
2777  DAG.getConstant(0, dl, N1.getValueType()), Cond);
2778  if (N1.getNode()->hasOneUse()) {
2779  assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
2780  auto &DL = DAG.getDataLayout();
2781  // X == (Z-X) --> X<<1 == Z
2782  SDValue SH = DAG.getNode(
2783  ISD::SHL, dl, N1.getValueType(), N0,
2784  DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL,
2785  !DCI.isBeforeLegalize())));
2786  if (!DCI.isCalledByLegalizer())
2787  DCI.AddToWorklist(SH.getNode());
2788  return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond);
2789  }
2790  }
2791  }
2792 
2793  if (SDValue V = simplifySetCCWithAnd(VT, N0, N1, Cond, DCI, dl))
2794  return V;
2795  }
2796 
2797  // Fold away ALL boolean setcc's.
2798  SDValue Temp;
2799  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
2800  EVT OpVT = N0.getValueType();
2801  switch (Cond) {
2802  default: llvm_unreachable("Unknown integer setcc!");
2803  case ISD::SETEQ: // X == Y -> ~(X^Y)
2804  Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
2805  N0 = DAG.getNOT(dl, Temp, OpVT);
2806  if (!DCI.isCalledByLegalizer())
2807  DCI.AddToWorklist(Temp.getNode());
2808  break;
2809  case ISD::SETNE: // X != Y --> (X^Y)
2810  N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
2811  break;
2812  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
2813  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
2814  Temp = DAG.getNOT(dl, N0, OpVT);
2815  N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
2816  if (!DCI.isCalledByLegalizer())
2817  DCI.AddToWorklist(Temp.getNode());
2818  break;
2819  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
2820  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
2821  Temp = DAG.getNOT(dl, N1, OpVT);
2822  N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
2823  if (!DCI.isCalledByLegalizer())
2824  DCI.AddToWorklist(Temp.getNode());
2825  break;
2826  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
2827  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
2828  Temp = DAG.getNOT(dl, N0, OpVT);
2829  N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
2830  if (!DCI.isCalledByLegalizer())
2831  DCI.AddToWorklist(Temp.getNode());
2832  break;
2833  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
2834  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
2835  Temp = DAG.getNOT(dl, N1, OpVT);
2836  N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
2837  break;
2838  }
2839  if (VT.getScalarType() != MVT::i1) {
2840  if (!DCI.isCalledByLegalizer())
2841  DCI.AddToWorklist(N0.getNode());
2842  // FIXME: If running after legalize, we probably can't do this.
2844  N0 = DAG.getNode(ExtendCode, dl, VT, N0);
2845  }
2846  return N0;
2847  }
2848 
2849  // Could not fold it.
2850  return SDValue();
2851 }
2852 
2853 /// Returns true (and the GlobalValue and the offset) if the node is a
2854 /// GlobalAddress + offset.
2856  int64_t &Offset) const {
2857  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
2858  GA = GASD->getGlobal();
2859  Offset += GASD->getOffset();
2860  return true;
2861  }
2862 
2863  if (N->getOpcode() == ISD::ADD) {
2864  SDValue N1 = N->getOperand(0);
2865  SDValue N2 = N->getOperand(1);
2866  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
2867  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
2868  Offset += V->getSExtValue();
2869  return true;
2870  }
2871  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
2872  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
2873  Offset += V->getSExtValue();
2874  return true;
2875  }
2876  }
2877  }
2878 
2879  return false;
2880 }
2881 
2883  DAGCombinerInfo &DCI) const {
2884  // Default implementation: no optimization.
2885  return SDValue();
2886 }
2887 
2888 //===----------------------------------------------------------------------===//
2889 // Inline Assembler Implementation Methods
2890 //===----------------------------------------------------------------------===//
2891 
2894  unsigned S = Constraint.size();
2895 
2896  if (S == 1) {
2897  switch (Constraint[0]) {
2898  default: break;
2899  case 'r': return C_RegisterClass;
2900  case 'm': // memory
2901  case 'o': // offsetable
2902  case 'V': // not offsetable
2903  return C_Memory;
2904  case 'i': // Simple Integer or Relocatable Constant
2905  case 'n': // Simple Integer
2906  case 'E': // Floating Point Constant
2907  case 'F': // Floating Point Constant
2908  case 's': // Relocatable Constant
2909  case 'p': // Address.
2910  case 'X': // Allow ANY value.
2911  case 'I': // Target registers.
2912  case 'J':
2913  case 'K':
2914  case 'L':
2915  case 'M':
2916  case 'N':
2917  case 'O':
2918  case 'P':
2919  case '<':
2920  case '>':
2921  return C_Other;
2922  }
2923  }
2924 
2925  if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') {
2926  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
2927  return C_Memory;
2928  return C_Register;
2929  }
2930  return C_Unknown;
2931 }
2932 
2933 /// Try to replace an X constraint, which matches anything, with another that
2934 /// has more specific requirements based on the type of the corresponding
2935 /// operand.
2936 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const{
2937  if (ConstraintVT.isInteger())
2938  return "r";
2939  if (ConstraintVT.isFloatingPoint())
2940  return "f"; // works for many targets
2941  return nullptr;
2942 }
2943 
2944 /// Lower the specified operand into the Ops vector.
2945 /// If it is invalid, don't add anything to Ops.
2947  std::string &Constraint,
2948  std::vector<SDValue> &Ops,
2949  SelectionDAG &DAG) const {
2950 
2951  if (Constraint.length() > 1) return;
2952 
2953  char ConstraintLetter = Constraint[0];
2954  switch (ConstraintLetter) {
2955  default: break;
2956  case 'X': // Allows any operand; labels (basic block) use this.
2957  if (Op.getOpcode() == ISD::BasicBlock) {
2958  Ops.push_back(Op);
2959  return;
2960  }
2962  case 'i': // Simple Integer or Relocatable Constant
2963  case 'n': // Simple Integer
2964  case 's': { // Relocatable Constant
2965  // These operands are interested in values of the form (GV+C), where C may
2966  // be folded in as an offset of GV, or it may be explicitly added. Also, it
2967  // is possible and fine if either GV or C are missing.
2970 
2971  // If we have "(add GV, C)", pull out GV/C
2972  if (Op.getOpcode() == ISD::ADD) {
2973  C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2975  if (!C || !GA) {
2976  C = dyn_cast<ConstantSDNode>(Op.getOperand(0));
2978  }
2979  if (!C || !GA) {
2980  C = nullptr;
2981  GA = nullptr;
2982  }
2983  }
2984 
2985  // If we find a valid operand, map to the TargetXXX version so that the
2986  // value itself doesn't get selected.
2987  if (GA) { // Either &GV or &GV+C
2988  if (ConstraintLetter != 'n') {
2989  int64_t Offs = GA->getOffset();
2990  if (C) Offs += C->getZExtValue();
2991  Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(),
2992  C ? SDLoc(C) : SDLoc(),
2993  Op.getValueType(), Offs));
2994  }
2995  return;
2996  }
2997  if (C) { // just C, no GV.
2998  // Simple constants are not allowed for 's'.
2999  if (ConstraintLetter != 's') {
3000  // gcc prints these as sign extended. Sign extend value to 64 bits
3001  // now; without this it would get ZExt'd later in
3002  // ScheduleDAGSDNodes::EmitNode, which is very generic.
3003  Ops.push_back(DAG.getTargetConstant(C->getSExtValue(),
3004  SDLoc(C), MVT::i64));
3005  }
3006  return;
3007  }
3008  break;
3009  }
3010  }
3011 }
3012 
3013 std::pair<unsigned, const TargetRegisterClass *>
3015  StringRef Constraint,
3016  MVT VT) const {
3017  if (Constraint.empty() || Constraint[0] != '{')
3018  return std::make_pair(0u, static_cast<TargetRegisterClass*>(nullptr));
3019  assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?");
3020 
3021  // Remove the braces from around the name.
3022  StringRef RegName(Constraint.data()+1, Constraint.size()-2);
3023 
3024  std::pair<unsigned, const TargetRegisterClass*> R =
3025  std::make_pair(0u, static_cast<const TargetRegisterClass*>(nullptr));
3026 
3027  // Figure out which register class contains this reg.
3028  for (const TargetRegisterClass *RC : RI->regclasses()) {
3029  // If none of the value types for this register class are valid, we
3030  // can't use it. For example, 64-bit reg classes on 32-bit targets.
3031  if (!isLegalRC(*RI, *RC))
3032  continue;
3033 
3034  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
3035  I != E; ++I) {
3036  if (RegName.equals_lower(RI->getRegAsmName(*I))) {
3037  std::pair<unsigned, const TargetRegisterClass*> S =
3038  std::make_pair(*I, RC);
3039 
3040  // If this register class has the requested value type, return it,
3041  // otherwise keep searching and return the first class found
3042  // if no other is found which explicitly has the requested type.
3043  if (RI->isTypeLegalForClass(*RC, VT))
3044  return S;
3045  if (!R.second)
3046  R = S;
3047  }
3048  }
3049  }
3050 
3051  return R;
3052 }
3053 
3054 //===----------------------------------------------------------------------===//
3055 // Constraint Selection.
3056 
3057 /// Return true of this is an input operand that is a matching constraint like
3058 /// "4".
3060  assert(!ConstraintCode.empty() && "No known constraint!");
3061  return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
3062 }
3063 
3064 /// If this is an input matching constraint, this method returns the output
3065 /// operand it matches.
3067  assert(!ConstraintCode.empty() && "No known constraint!");
3068  return atoi(ConstraintCode.c_str());
3069 }
3070 
3071 /// Split up the constraint string from the inline assembly value into the
3072 /// specific constraints and their prefixes, and also tie in the associated
3073 /// operand values.
3074 /// If this returns an empty vector, and if the constraint string itself
3075 /// isn't empty, there was an error parsing.
3078  const TargetRegisterInfo *TRI,
3079  ImmutableCallSite CS) const {
3080  /// Information about all of the constraints.
3081  AsmOperandInfoVector ConstraintOperands;
3082  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
3083  unsigned maCount = 0; // Largest number of multiple alternative constraints.
3084 
3085  // Do a prepass over the constraints, canonicalizing them, and building up the
3086  // ConstraintOperands list.
3087  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
3088  unsigned ResNo = 0; // ResNo - The result number of the next output.
3089 
3090  for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
3091  ConstraintOperands.emplace_back(std::move(CI));
3092  AsmOperandInfo &OpInfo = ConstraintOperands.back();
3093 
3094  // Update multiple alternative constraint count.
3095  if (OpInfo.multipleAlternatives.size() > maCount)
3096  maCount = OpInfo.multipleAlternatives.size();
3097 
3098  OpInfo.ConstraintVT = MVT::Other;
3099 
3100  // Compute the value type for each operand.
3101  switch (OpInfo.Type) {
3102  case InlineAsm::isOutput:
3103  // Indirect outputs just consume an argument.
3104  if (OpInfo.isIndirect) {
3105  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
3106  break;
3107  }
3108 
3109  // The return value of the call is this value. As such, there is no
3110  // corresponding argument.
3111  assert(!CS.getType()->isVoidTy() &&
3112  "Bad inline asm!");
3113  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
3114  OpInfo.ConstraintVT =
3115  getSimpleValueType(DL, STy->getElementType(ResNo));
3116  } else {
3117  assert(ResNo == 0 && "Asm only has one result!");
3118  OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
3119  }
3120  ++ResNo;
3121  break;
3122  case InlineAsm::isInput:
3123  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
3124  break;
3125  case InlineAsm::isClobber:
3126  // Nothing to do.
3127  break;
3128  }
3129 
3130  if (OpInfo.CallOperandVal) {
3131  llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
3132  if (OpInfo.isIndirect) {
3133  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
3134  if (!PtrTy)
3135  report_fatal_error("Indirect operand for inline asm not a pointer!");
3136  OpTy = PtrTy->getElementType();
3137  }
3138 
3139  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
3140  if (StructType *STy = dyn_cast<StructType>(OpTy))
3141  if (STy->getNumElements() == 1)
3142  OpTy = STy->getElementType(0);
3143 
3144  // If OpTy is not a single value, it may be a struct/union that we
3145  // can tile with integers.
3146  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
3147  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
3148  switch (BitSize) {
3149  default: break;
3150  case 1:
3151  case 8:
3152  case 16:
3153  case 32:
3154  case 64:
3155  case 128:
3156  OpInfo.ConstraintVT =
3157  MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
3158  break;
3159  }
3160  } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
3161  unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
3162  OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
3163  } else {
3164  OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
3165  }
3166  }
3167  }
3168 
3169  // If we have multiple alternative constraints, select the best alternative.
3170  if (!ConstraintOperands.empty()) {
3171  if (maCount) {
3172  unsigned bestMAIndex = 0;
3173  int bestWeight = -1;
3174  // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
3175  int weight = -1;
3176  unsigned maIndex;
3177  // Compute the sums of the weights for each alternative, keeping track
3178  // of the best (highest weight) one so far.
3179  for (maIndex = 0; maIndex < maCount; ++maIndex) {
3180  int weightSum = 0;
3181  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3182  cIndex != eIndex; ++cIndex) {
3183  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
3184  if (OpInfo.Type == InlineAsm::isClobber)
3185  continue;
3186 
3187  // If this is an output operand with a matching input operand,
3188  // look up the matching input. If their types mismatch, e.g. one
3189  // is an integer, the other is floating point, or their sizes are
3190  // different, flag it as an maCantMatch.
3191  if (OpInfo.hasMatchingInput()) {
3192  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3193  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3194  if ((OpInfo.ConstraintVT.isInteger() !=
3195  Input.ConstraintVT.isInteger()) ||
3196  (OpInfo.ConstraintVT.getSizeInBits() !=
3197  Input.ConstraintVT.getSizeInBits())) {
3198  weightSum = -1; // Can't match.
3199  break;
3200  }
3201  }
3202  }
3203  weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
3204  if (weight == -1) {
3205  weightSum = -1;
3206  break;
3207  }
3208  weightSum += weight;
3209  }
3210  // Update best.
3211  if (weightSum > bestWeight) {
3212  bestWeight = weightSum;
3213  bestMAIndex = maIndex;
3214  }
3215  }
3216 
3217  // Now select chosen alternative in each constraint.
3218  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3219  cIndex != eIndex; ++cIndex) {
3220  AsmOperandInfo& cInfo = ConstraintOperands[cIndex];
3221  if (cInfo.Type == InlineAsm::isClobber)
3222  continue;
3223  cInfo.selectAlternative(bestMAIndex);
3224  }
3225  }
3226  }
3227 
3228  // Check and hook up tied operands, choose constraint code to use.
3229  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3230  cIndex != eIndex; ++cIndex) {
3231  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
3232 
3233  // If this is an output operand with a matching input operand, look up the
3234  // matching input. If their types mismatch, e.g. one is an integer, the
3235  // other is floating point, or their sizes are different, flag it as an
3236  // error.
3237  if (OpInfo.hasMatchingInput()) {
3238  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3239 
3240  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3241  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
3243  OpInfo.ConstraintVT);
3244  std::pair<unsigned, const TargetRegisterClass *> InputRC =
3246  Input.ConstraintVT);
3247  if ((OpInfo.ConstraintVT.isInteger() !=
3248  Input.ConstraintVT.isInteger()) ||
3249  (MatchRC.second != InputRC.second)) {
3250  report_fatal_error("Unsupported asm: input constraint"
3251  " with a matching output constraint of"
3252  " incompatible type!");
3253  }
3254  }
3255  }
3256  }
3257 
3258  return ConstraintOperands;
3259 }
3260 
3261 /// Return an integer indicating how general CT is.
3263  switch (CT) {
3266  return 0;
3268  return 1;
3270  return 2;
3272  return 3;
3273  }
3274  llvm_unreachable("Invalid constraint type");
3275 }
3276 
3277 /// Examine constraint type and operand type and determine a weight value.
3278 /// This object must already have been set up with the operand type
3279 /// and the current alternative constraint selected.
3282  AsmOperandInfo &info, int maIndex) const {
3284  if (maIndex >= (int)info.multipleAlternatives.size())
3285  rCodes = &info.Codes;
3286  else
3287  rCodes = &info.multipleAlternatives[maIndex].Codes;
3288  ConstraintWeight BestWeight = CW_Invalid;
3289 
3290  // Loop over the options, keeping track of the most general one.
3291  for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
3292  ConstraintWeight weight =
3293  getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
3294  if (weight > BestWeight)
3295  BestWeight = weight;
3296  }
3297 
3298  return BestWeight;
3299 }
3300 
3301 /// Examine constraint type and operand type and determine a weight value.
3302 /// This object must already have been set up with the operand type
3303 /// and the current alternative constraint selected.
3306  AsmOperandInfo &info, const char *constraint) const {
3307  ConstraintWeight weight = CW_Invalid;
3308  Value *CallOperandVal = info.CallOperandVal;
3309  // If we don't have a value, we can't do a match,
3310  // but allow it at the lowest weight.
3311  if (!CallOperandVal)
3312  return CW_Default;
3313  // Look at the constraint type.
3314  switch (*constraint) {
3315  case 'i': // immediate integer.
3316  case 'n': // immediate integer with a known value.
3317  if (isa<ConstantInt>(CallOperandVal))
3318  weight = CW_Constant;
3319  break;
3320  case 's': // non-explicit intregal immediate.
3321  if (isa<GlobalValue>(CallOperandVal))
3322  weight = CW_Constant;
3323  break;
3324  case 'E': // immediate float if host format.
3325  case 'F': // immediate float.
3326  if (isa<ConstantFP>(CallOperandVal))
3327  weight = CW_Constant;
3328  break;
3329  case '<': // memory operand with autodecrement.
3330  case '>': // memory operand with autoincrement.
3331  case 'm': // memory operand.
3332  case 'o': // offsettable memory operand
3333  case 'V': // non-offsettable memory operand
3334  weight = CW_Memory;
3335  break;
3336  case 'r': // general register.
3337  case 'g': // general register, memory operand or immediate integer.
3338  // note: Clang converts "g" to "imr".
3339  if (CallOperandVal->getType()->isIntegerTy())
3340  weight = CW_Register;
3341  break;
3342  case 'X': // any operand.
3343  default:
3344  weight = CW_Default;
3345  break;
3346  }
3347  return weight;
3348 }
3349 
3350 /// If there are multiple different constraints that we could pick for this
3351 /// operand (e.g. "imr") try to pick the 'best' one.
3352 /// This is somewhat tricky: constraints fall into four classes:
3353 /// Other -> immediates and magic values
3354 /// Register -> one specific register
3355 /// RegisterClass -> a group of regs
3356 /// Memory -> memory
3357 /// Ideally, we would pick the most specific constraint possible: if we have
3358 /// something that fits into a register, we would pick it. The problem here
3359 /// is that if we have something that could either be in a register or in
3360 /// memory that use of the register could cause selection of *other*
3361 /// operands to fail: they might only succeed if we pick memory. Because of
3362 /// this the heuristic we use is:
3363 ///
3364 /// 1) If there is an 'other' constraint, and if the operand is valid for
3365 /// that constraint, use it. This makes us take advantage of 'i'
3366 /// constraints when available.
3367 /// 2) Otherwise, pick the most general constraint present. This prefers
3368 /// 'm' over 'r', for example.
3369 ///
3371  const TargetLowering &TLI,
3372  SDValue Op, SelectionDAG *DAG) {
3373  assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
3374  unsigned BestIdx = 0;
3376  int BestGenerality = -1;
3377 
3378  // Loop over the options, keeping track of the most general one.
3379  for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
3381  TLI.getConstraintType(OpInfo.Codes[i]);
3382 
3383  // If this is an 'other' constraint, see if the operand is valid for it.
3384  // For example, on X86 we might have an 'rI' constraint. If the operand
3385  // is an integer in the range [0..31] we want to use I (saving a load
3386  // of a register), otherwise we must use 'r'.
3387  if (CType == TargetLowering::C_Other && Op.getNode()) {
3388  assert(OpInfo.Codes[i].size() == 1 &&
3389  "Unhandled multi-letter 'other' constraint");
3390  std::vector<SDValue> ResultOps;
3391  TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
3392  ResultOps, *DAG);
3393  if (!ResultOps.empty()) {
3394  BestType = CType;
3395  BestIdx = i;
3396  break;
3397  }
3398  }
3399 
3400  // Things with matching constraints can only be registers, per gcc
3401  // documentation. This mainly affects "g" constraints.
3402  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
3403  continue;
3404 
3405  // This constraint letter is more general than the previous one, use it.
3406  int Generality = getConstraintGenerality(CType);
3407  if (Generality > BestGenerality) {
3408  BestType = CType;
3409  BestIdx = i;
3410  BestGenerality = Generality;
3411  }
3412  }
3413 
3414  OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
3415  OpInfo.ConstraintType = BestType;
3416 }
3417 
3418 /// Determines the constraint code and constraint type to use for the specific
3419 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
3421  SDValue Op,
3422  SelectionDAG *DAG) const {
3423  assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
3424 
3425  // Single-letter constraints ('r') are very common.
3426  if (OpInfo.Codes.size() == 1) {
3427  OpInfo.ConstraintCode = OpInfo.Codes[0];
3429  } else {
3430  ChooseConstraint(OpInfo, *this, Op, DAG);
3431  }
3432 
3433  // 'X' matches anything.
3434  if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
3435  // Labels and constants are handled elsewhere ('X' is the only thing
3436  // that matches labels). For Functions, the type here is the type of
3437  // the result, which is not what we want to look at; leave them alone.
3438  Value *v = OpInfo.CallOperandVal;
3439  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
3440  OpInfo.CallOperandVal = v;
3441  return;
3442  }
3443 
3444  // Otherwise, try to resolve it to something we know about by looking at
3445  // the actual operand type.
3446  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
3447  OpInfo.ConstraintCode = Repl;
3449  }
3450  }
3451 }
3452 
3453 /// Given an exact SDIV by a constant, create a multiplication
3454 /// with the multiplicative inverse of the constant.
3456  const SDLoc &dl, SelectionDAG &DAG,
3457  SmallVectorImpl<SDNode *> &Created) {
3458  SDValue Op0 = N->getOperand(0);
3459  SDValue Op1 = N->getOperand(1);
3460  EVT VT = N->getValueType(0);
3461  EVT SVT = VT.getScalarType();
3462  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3463  EVT ShSVT = ShVT.getScalarType();
3464 
3465  bool UseSRA = false;
3466  SmallVector<SDValue, 16> Shifts, Factors;
3467 
3468  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
3469  if (C->isNullValue())
3470  return false;
3471  APInt Divisor = C->getAPIntValue();
3472  unsigned Shift = Divisor.countTrailingZeros();
3473  if (Shift) {
3474  Divisor.ashrInPlace(Shift);
3475  UseSRA = true;
3476  }
3477  // Calculate the multiplicative inverse, using Newton's method.
3478  APInt t;
3479  APInt Factor = Divisor;
3480  while ((t = Divisor * Factor) != 1)
3481  Factor *= APInt(Divisor.getBitWidth(), 2) - t;
3482  Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
3483  Factors.push_back(DAG.getConstant(Factor, dl, SVT));
3484  return true;
3485  };
3486 
3487  // Collect all magic values from the build vector.
3488  if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
3489  return SDValue();
3490 
3491  SDValue Shift, Factor;
3492  if (VT.isVector()) {
3493  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
3494  Factor = DAG.getBuildVector(VT, dl, Factors);
3495  } else {
3496  Shift = Shifts[0];
3497  Factor = Factors[0];
3498  }
3499 
3500  SDValue Res = Op0;
3501 
3502  // Shift the value upfront if it is even, so the LSB is one.
3503  if (UseSRA) {
3504  // TODO: For UDIV use SRL instead of SRA.
3505  SDNodeFlags Flags;
3506  Flags.setExact(true);
3507  Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
3508  Created.push_back(Res.getNode());
3509  }
3510 
3511  return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
3512 }
3513 
3515  SelectionDAG &DAG,
3516  SmallVectorImpl<SDNode *> &Created) const {
3518  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3519  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
3520  return SDValue(N,0); // Lower SDIV as SDIV
3521  return SDValue();
3522 }
3523 
3524 /// Given an ISD::SDIV node expressing a divide by constant,
3525 /// return a DAG expression to select that will generate the same value by
3526 /// multiplying by a magic number.
3527 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3529  bool IsAfterLegalization,
3530  SmallVectorImpl<SDNode *> &Created) const {
3531  SDLoc dl(N);
3532  EVT VT = N->getValueType(0);
3533  EVT SVT = VT.getScalarType();
3534  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
3535  EVT ShSVT = ShVT.getScalarType();
3536  unsigned EltBits = VT.getScalarSizeInBits();
3537 
3538  // Check to see if we can do this.
3539  // FIXME: We should be more aggressive here.
3540  if (!isTypeLegal(VT))
3541  return SDValue();
3542 
3543  // If the sdiv has an 'exact' bit we can use a simpler lowering.
3544  if (N->getFlags().hasExact())
3545  return BuildExactSDIV(*this, N, dl, DAG, Created);
3546 
3547  SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
3548 
3549  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
3550  if (C->isNullValue())
3551  return false;
3552 
3553  const APInt &Divisor = C->getAPIntValue();
3554  APInt::ms magics = Divisor.magic();
3555  int NumeratorFactor = 0;
3556  int ShiftMask = -1;
3557 
3558  if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
3559  // If d is +1/-1, we just multiply the numerator by +1/-1.
3560  NumeratorFactor = Divisor.getSExtValue();
3561  magics.m = 0;
3562  magics.s = 0;
3563  ShiftMask = 0;
3564  } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
3565  // If d > 0 and m < 0, add the numerator.
3566  NumeratorFactor = 1;
3567  } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
3568  // If d < 0 and m > 0, subtract the numerator.
3569  NumeratorFactor = -1;
3570  }
3571 
3572  MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
3573  Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
3574  Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
3575  ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
3576  return true;
3577  };
3578 
3579  SDValue N0 = N->getOperand(0);
3580  SDValue N1 = N->getOperand(1);
3581 
3582  // Collect the shifts / magic values from each element.
3583  if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
3584  return SDValue();
3585 
3586  SDValue MagicFactor, Factor, Shift, ShiftMask;
3587  if (VT.isVector()) {
3588  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
3589  Factor = DAG.getBuildVector(VT, dl, Factors);
3590  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
3591  ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
3592  } else {
3593  MagicFactor = MagicFactors[0];
3594  Factor = Factors[0];
3595  Shift = Shifts[0];
3596  ShiftMask = ShiftMasks[0];
3597  }
3598 
3599  // Multiply the numerator (operand 0) by the magic value.
3600  // FIXME: We should support doing a MUL in a wider type.
3601  SDValue Q;
3602  if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
3604  Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
3605  else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
3607  SDValue LoHi =
3608  DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
3609  Q = SDValue(LoHi.getNode(), 1);
3610  } else
3611  return SDValue(); // No mulhs or equivalent.
3612  Created.push_back(Q.getNode());
3613 
3614  // (Optionally) Add/subtract the numerator using Factor.
3615  Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
3616  Created.push_back(Factor.getNode());
3617  Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
3618  Created.push_back(Q.getNode());
3619 
3620  // Shift right algebraic by shift value.
3621  Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
3622  Created.push_back(Q.getNode());
3623 
3624  // Extract the sign bit, mask it and add it to the quotient.
3625  SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
3626  SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
3627  Created.push_back(T.getNode());
3628  T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
3629  Created.push_back(T.getNode());
3630  return DAG.getNode(ISD::ADD, dl, VT, Q, T);
3631 }
3632 
3633 /// Given an ISD::UDIV node expressing a divide by constant,
3634 /// return a DAG expression to select that will generate the same value by
3635 /// multiplying by a magic number.
3636 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3638  bool IsAfterLegalization,
3639  SmallVectorImpl<SDNode *> &Created) const {
3640  SDLoc dl(N);
3641  EVT VT = N->getValueType(0);
3642  EVT SVT = VT.getScalarType();
3643  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
3644  EVT ShSVT = ShVT.getScalarType();
3645  unsigned EltBits = VT.getScalarSizeInBits();
3646 
3647  // Check to see if we can do this.
3648  // FIXME: We should be more aggressive here.
3649  if (!isTypeLegal(VT))
3650  return SDValue();
3651 
3652  bool UseNPQ = false;
3653  SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
3654 
3655  auto BuildUDIVPattern = [&](ConstantSDNode *C) {
3656  if (C->isNullValue())
3657  return false;
3658  // FIXME: We should use a narrower constant when the upper
3659  // bits are known to be zero.
3660  APInt Divisor = C->getAPIntValue();
3661  APInt::mu magics = Divisor.magicu();
3662  unsigned PreShift = 0, PostShift = 0;
3663 
3664  // If the divisor is even, we can avoid using the expensive fixup by
3665  // shifting the divided value upfront.
3666  if (magics.a != 0 && !Divisor[0]) {
3667  PreShift = Divisor.countTrailingZeros();
3668  // Get magic number for the shifted divisor.
3669  magics = Divisor.lshr(PreShift).magicu(PreShift);
3670  assert(magics.a == 0 && "Should use cheap fixup now");
3671  }
3672 
3673  APInt Magic = magics.m;
3674 
3675  unsigned SelNPQ;
3676  if (magics.a == 0 || Divisor.isOneValue()) {
3677  assert(magics.s < Divisor.getBitWidth() &&
3678  "We shouldn't generate an undefined shift!");
3679  PostShift = magics.s;
3680  SelNPQ = false;
3681  } else {
3682  PostShift = magics.s - 1;
3683  SelNPQ = true;
3684  }
3685 
3686  PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
3687  MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
3688  NPQFactors.push_back(
3689  DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
3690  : APInt::getNullValue(EltBits),
3691  dl, SVT));
3692  PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
3693  UseNPQ |= SelNPQ;
3694  return true;
3695  };
3696 
3697  SDValue N0 = N->getOperand(0);
3698  SDValue N1 = N->getOperand(1);
3699 
3700  // Collect the shifts/magic values from each element.
3701  if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
3702  return SDValue();
3703 
3704  SDValue PreShift, PostShift, MagicFactor, NPQFactor;
3705  if (VT.isVector()) {
3706  PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
3707  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
3708  NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
3709  PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
3710  } else {
3711  PreShift = PreShifts[0];
3712  MagicFactor = MagicFactors[0];
3713  PostShift = PostShifts[0];
3714  }
3715 
3716  SDValue Q = N0;
3717  Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
3718  Created.push_back(Q.getNode());
3719 
3720  // FIXME: We should support doing a MUL in a wider type.
3721  auto GetMULHU = [&](SDValue X, SDValue Y) {
3722  if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
3724  return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
3725  if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
3727  SDValue LoHi =
3728  DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
3729  return SDValue(LoHi.getNode(), 1);
3730  }
3731  return SDValue(); // No mulhu or equivalent
3732  };
3733 
3734  // Multiply the numerator (operand 0) by the magic value.
3735  Q = GetMULHU(Q, MagicFactor);
3736  if (!Q)
3737  return SDValue();
3738 
3739  Created.push_back(Q.getNode());
3740 
3741  if (UseNPQ) {
3742  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
3743  Created.push_back(NPQ.getNode());
3744 
3745  // For vectors we might have a mix of non-NPQ/NPQ paths, so use
3746  // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
3747  if (VT.isVector())
3748  NPQ = GetMULHU(NPQ, NPQFactor);
3749  else
3750  NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
3751 
3752  Created.push_back(NPQ.getNode());
3753 
3754  Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
3755  Created.push_back(Q.getNode());
3756  }
3757 
3758  Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
3759  Created.push_back(Q.getNode());
3760 
3761  SDValue One = DAG.getConstant(1, dl, VT);
3762  SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
3763  return DAG.getSelect(dl, VT, IsOne, N0, Q);
3764 }
3765 
3766 bool TargetLowering::
3768  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
3769  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
3770  "be a constant integer");
3771  return true;
3772  }
3773 
3774  return false;
3775 }
3776 
3777 //===----------------------------------------------------------------------===//
3778 // Legalization Utilities
3779 //===----------------------------------------------------------------------===//
3780 
3781 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
3782  SDValue LHS, SDValue RHS,
3783  SmallVectorImpl<SDValue> &Result,
3784  EVT HiLoVT, SelectionDAG &DAG,
3786  SDValue LH, SDValue RL, SDValue RH) const {
3787  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
3788  Opcode == ISD::SMUL_LOHI);
3789 
3790  bool HasMULHS = (Kind == MulExpansionKind::Always) ||
3792  bool HasMULHU = (Kind == MulExpansionKind::Always) ||
3794  bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
3796  bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
3798 
3799  if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
3800  return false;
3801 
3802  unsigned OuterBitSize = VT.getScalarSizeInBits();
3803  unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
3804  unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
3805  unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
3806 
3807  // LL, LH, RL, and RH must be either all NULL or all set to a value.
3808  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
3809  (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
3810 
3811  SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
3812  auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
3813  bool Signed) -> bool {
3814  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
3815  Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
3816  Hi = SDValue(Lo.getNode(), 1);
3817  return true;
3818  }
3819  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
3820  Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
3821  Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
3822  return true;
3823  }
3824  return false;
3825  };
3826 
3827  SDValue Lo, Hi;
3828 
3829  if (!LL.getNode() && !RL.getNode() &&
3831  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
3832  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
3833  }
3834 
3835  if (!LL.getNode())
3836  return false;
3837 
3838  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
3839  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
3840  DAG.MaskedValueIsZero(RHS, HighMask)) {
3841  // The inputs are both zero-extended.
3842  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
3843  Result.push_back(Lo);
3844  Result.push_back(Hi);
3845  if (Opcode != ISD::MUL) {
3846  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
3847  Result.push_back(Zero);
3848  Result.push_back(Zero);
3849  }
3850  return true;
3851  }
3852  }
3853 
3854  if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
3855  RHSSB > InnerBitSize) {
3856  // The input values are both sign-extended.
3857  // TODO non-MUL case?
3858  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
3859  Result.push_back(Lo);
3860  Result.push_back(Hi);
3861  return true;
3862  }
3863  }
3864 
3865  unsigned ShiftAmount = OuterBitSize - InnerBitSize;
3866  EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
3867  if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
3868  // FIXME getShiftAmountTy does not always return a sensible result when VT
3869  // is an illegal type, and so the type may be too small to fit the shift
3870  // amount. Override it with i32. The shift will have to be legalized.
3871  ShiftAmountTy = MVT::i32;
3872  }
3873  SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
3874 
3875  if (!LH.getNode() && !RH.getNode() &&
3878  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
3879  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
3880  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
3881  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
3882  }
3883 
3884  if (!LH.getNode())
3885  return false;
3886 
3887  if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
3888  return false;
3889 
3890  Result.push_back(Lo);
3891 
3892  if (Opcode == ISD::MUL) {
3893  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
3894  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
3895  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
3896  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
3897  Result.push_back(Hi);
3898  return true;
3899  }
3900 
3901  // Compute the full width result.
3902  auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
3903  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3904  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
3905  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3906  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
3907  };
3908 
3909  SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
3910  if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
3911  return false;
3912 
3913  // This is effectively the add part of a multiply-add of half-sized operands,
3914  // so it cannot overflow.
3915  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
3916 
3917  if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
3918  return false;
3919 
3920  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
3921  Merge(Lo, Hi));
3922 
3923  SDValue Carry = Next.getValue(1);
3924  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3925  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
3926 
3927  if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
3928  return false;
3929 
3930  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
3931  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
3932  Carry);
3933  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
3934 
3935  if (Opcode == ISD::SMUL_LOHI) {
3936  SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
3937  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
3938  Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
3939 
3940  NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
3941  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
3942  Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
3943  }
3944 
3945  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3946  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
3947  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3948  return true;
3949 }
3950 
3953  SDValue LL, SDValue LH, SDValue RL,
3954  SDValue RH) const {
3956  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
3957  N->getOperand(0), N->getOperand(1), Result, HiLoVT,
3958  DAG, Kind, LL, LH, RL, RH);
3959  if (Ok) {
3960  assert(Result.size() == 2);
3961  Lo = Result[0];
3962  Hi = Result[1];
3963  }
3964  return Ok;
3965 }
3966 
3968  SelectionDAG &DAG) const {
3969  EVT VT = Node->getOperand(0).getValueType();
3970  EVT NVT = Node->getValueType(0);
3971  SDLoc dl(SDValue(Node, 0));
3972 
3973  // FIXME: Only f32 to i64 conversions are supported.
3974  if (VT != MVT::f32 || NVT != MVT::i64)
3975  return false;
3976 
3977  // Expand f32 -> i64 conversion
3978  // This algorithm comes from compiler-rt's implementation of fixsfdi:
3979  // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c
3980  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(),
3981  VT.getSizeInBits());
3982  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
3983  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
3984  SDValue Bias = DAG.getConstant(127, dl, IntVT);
3985  SDValue SignMask = DAG.getConstant(APInt::getSignMask(VT.getSizeInBits()), dl,
3986  IntVT);
3987  SDValue SignLowBit = DAG.getConstant(VT.getSizeInBits() - 1, dl, IntVT);
3988  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
3989 
3990  SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Node->getOperand(0));
3991 
3992  auto &DL = DAG.getDataLayout();
3993  SDValue ExponentBits = DAG.getNode(
3994  ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
3995  DAG.getZExtOrTrunc(ExponentLoBit, dl, getShiftAmountTy(IntVT, DL)));
3996  SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
3997 
3998  SDValue Sign = DAG.getNode(
3999  ISD::SRA, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
4000  DAG.getZExtOrTrunc(SignLowBit, dl, getShiftAmountTy(IntVT, DL)));
4001  Sign = DAG.getSExtOrTrunc(Sign, dl, NVT);
4002 
4003  SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
4004  DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
4005  DAG.getConstant(0x00800000, dl, IntVT));
4006 
4007  R = DAG.getZExtOrTrunc(R, dl, NVT);
4008 
4009  R = DAG.getSelectCC(
4010  dl, Exponent, ExponentLoBit,
4011  DAG.getNode(ISD::SHL, dl, NVT, R,
4012  DAG.getZExtOrTrunc(
4013  DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
4014  dl, getShiftAmountTy(IntVT, DL))),
4015  DAG.getNode(ISD::SRL, dl, NVT, R,
4016  DAG.getZExtOrTrunc(
4017  DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
4018  dl, getShiftAmountTy(IntVT, DL))),
4019  ISD::SETGT);
4020 
4021  SDValue Ret = DAG.getNode(ISD::SUB, dl, NVT,
4022  DAG.getNode(ISD::XOR, dl, NVT, R, Sign),
4023  Sign);
4024 
4025  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
4026  DAG.getConstant(0, dl, NVT), Ret, ISD::SETLT);
4027  return true;
4028 }
4029 
4031  SelectionDAG &DAG) const {
4032  SDLoc SL(LD);
4033  SDValue Chain = LD->getChain();
4034  SDValue BasePTR = LD->getBasePtr();
4035  EVT SrcVT = LD->getMemoryVT();
4037 
4038  unsigned NumElem = SrcVT.getVectorNumElements();
4039 
4040  EVT SrcEltVT = SrcVT.getScalarType();
4041  EVT DstEltVT = LD->getValueType(0).getScalarType();
4042 
4043  unsigned Stride = SrcEltVT.getSizeInBits() / 8;
4044  assert(SrcEltVT.isByteSized());
4045 
4047  SmallVector<SDValue, 8> LoadChains;
4048 
4049  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
4050  SDValue ScalarLoad =
4051  DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
4052  LD->getPointerInfo().getWithOffset(Idx * Stride),
4053  SrcEltVT, MinAlign(LD->getAlignment(), Idx * Stride),
4054  LD->getMemOperand()->getFlags(), LD->getAAInfo());
4055 
4056  BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, Stride);
4057 
4058  Vals.push_back(ScalarLoad.getValue(0));
4059  LoadChains.push_back(ScalarLoad.getValue(1));
4060  }
4061 
4062  SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
4063  SDValue Value = DAG.getBuildVector(LD->getValueType(0), SL, Vals);
4064 
4065  return DAG.getMergeValues({ Value, NewChain }, SL);
4066 }
4067 
4069  SelectionDAG &DAG) const {
4070  SDLoc SL(ST);
4071 
4072  SDValue Chain = ST->getChain();
4073  SDValue BasePtr = ST->getBasePtr();
4074  SDValue Value = ST->getValue();
4075  EVT StVT = ST->getMemoryVT();
4076 
4077  // The type of the data we want to save
4078  EVT RegVT = Value.getValueType();
4079  EVT RegSclVT = RegVT.getScalarType();
4080 
4081  // The type of data as saved in memory.
4082  EVT MemSclVT = StVT.getScalarType();
4083 
4084  EVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
4085  unsigned NumElem = StVT.getVectorNumElements();
4086 
4087  // A vector must always be stored in memory as-is, i.e. without any padding
4088  // between the elements, since various code depend on it, e.g. in the
4089  // handling of a bitcast of a vector type to int, which may be done with a
4090  // vector store followed by an integer load. A vector that does not have
4091  // elements that are byte-sized must therefore be stored as an integer
4092  // built out of the extracted vector elements.
4093  if (!MemSclVT.isByteSized()) {
4094  unsigned NumBits = StVT.getSizeInBits();
4095  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
4096 
4097  SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
4098 
4099  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
4100  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
4101  DAG.getConstant(Idx, SL, IdxVT));
4102  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
4103  SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
4104  unsigned ShiftIntoIdx =
4105  (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
4106  SDValue ShiftAmount =
4107  DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
4108  SDValue ShiftedElt =
4109  DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
4110  CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
4111  }
4112 
4113  return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
4114  ST->getAlignment(), ST->getMemOperand()->getFlags(),
4115  ST->getAAInfo());
4116  }
4117 
4118  // Store Stride in bytes
4119  unsigned Stride = MemSclVT.getSizeInBits() / 8;
4120  assert (Stride && "Zero stride!");
4121  // Extract each of the elements from the original vector and save them into
4122  // memory individually.
4123  SmallVector<SDValue, 8> Stores;
4124  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
4125  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
4126  DAG.getConstant(Idx, SL, IdxVT));
4127 
4128  SDValue Ptr = DAG.getObjectPtrOffset(SL, BasePtr, Idx * Stride);
4129 
4130  // This scalar TruncStore may be illegal, but we legalize it later.
4131  SDValue Store = DAG.getTruncStore(
4132  Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
4133  MemSclVT, MinAlign(ST->getAlignment(), Idx * Stride),
4134  ST->getMemOperand()->getFlags(), ST->getAAInfo());
4135 
4136  Stores.push_back(Store);
4137  }
4138 
4139  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
4140 }
4141 
4142 std::pair<SDValue, SDValue>
4145  "unaligned indexed loads not implemented!");
4146  SDValue Chain = LD->getChain();
4147  SDValue Ptr = LD->getBasePtr();
4148  EVT VT = LD->getValueType(0);
4149  EVT LoadedVT = LD->getMemoryVT();
4150  SDLoc dl(LD);
4151  auto &MF = DAG.getMachineFunction();
4152 
4153  if (VT.isFloatingPoint() || VT.isVector()) {
4154  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
4155  if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
4156  if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
4157  LoadedVT.isVector()) {
4158  // Scalarize the load and let the individual components be handled.
4159  SDValue Scalarized = scalarizeVectorLoad(LD, DAG);
4160  if (Scalarized->getOpcode() == ISD::MERGE_VALUES)
4161  return std::make_pair(Scalarized.getOperand(0), Scalarized.getOperand(1));
4162  return std::make_pair(Scalarized.getValue(0), Scalarized.getValue(1));
4163  }
4164 
4165  // Expand to a (misaligned) integer load of the same size,
4166  // then bitconvert to floating point or vector.
4167  SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
4168  LD->getMemOperand());
4169  SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
4170  if (LoadedVT != VT)
4171  Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
4172  ISD::ANY_EXTEND, dl, VT, Result);
4173 
4174  return std::make_pair(Result, newLoad.getValue(1));
4175  }
4176 
4177  // Copy the value to a (aligned) stack slot using (unaligned) integer
4178  // loads and stores, then do a (aligned) load from the stack slot.
4179  MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
4180  unsigned LoadedBytes = LoadedVT.getStoreSize();
4181  unsigned RegBytes = RegVT.getSizeInBits() / 8;
4182  unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
4183 
4184  // Make sure the stack slot is also aligned for the register type.
4185  SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
4186  auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
4187  SmallVector<SDValue, 8> Stores;
4188  SDValue StackPtr = StackBase;
4189  unsigned Offset = 0;
4190 
4191  EVT PtrVT = Ptr.getValueType();
4192  EVT StackPtrVT = StackPtr.getValueType();
4193 
4194  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
4195  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
4196 
4197  // Do all but one copies using the full register width.
4198  for (unsigned i = 1; i < NumRegs; i++) {
4199  // Load one integer register's worth from the original location.
4200  SDValue Load = DAG.getLoad(
4201  RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
4203  LD->getAAInfo());
4204  // Follow the load with a store to the stack slot. Remember the store.
4205  Stores.push_back(DAG.getStore(
4206  Load.getValue(1), dl, Load, StackPtr,
4208  // Increment the pointers.
4209  Offset += RegBytes;
4210 
4211  Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
4212  StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
4213  }
4214 
4215  // The last copy may be partial. Do an extending load.
4216  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
4217  8 * (LoadedBytes - Offset));
4218  SDValue Load =
4219  DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
4220  LD->getPointerInfo().getWithOffset(Offset), MemVT,
4221  MinAlign(LD->getAlignment(), Offset),
4222  LD->getMemOperand()->getFlags(), LD->getAAInfo());
4223  // Follow the load with a store to the stack slot. Remember the store.
4224  // On big-endian machines this requires a truncating store to ensure
4225  // that the bits end up in the right place.
4226  Stores.push_back(DAG.getTruncStore(
4227  Load.getValue(1), dl, Load, StackPtr,
4228  MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
4229 
4230  // The order of the stores doesn't matter - say it with a TokenFactor.
4231  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
4232 
4233  // Finally, perform the original load only redirected to the stack slot.
4234  Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
4236  LoadedVT);
4237 
4238  // Callers expect a MERGE_VALUES node.
4239  return std::make_pair(Load, TF);
4240  }
4241 
4242  assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
4243  "Unaligned load of unsupported type.");
4244 
4245  // Compute the new VT that is half the size of the old one. This is an
4246  // integer MVT.
4247  unsigned NumBits = LoadedVT.getSizeInBits();
4248  EVT NewLoadedVT;
4249  NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
4250  NumBits >>= 1;
4251 
4252  unsigned Alignment = LD->getAlignment();
4253  unsigned IncrementSize = NumBits / 8;
4254  ISD::LoadExtType HiExtType = LD->getExtensionType();
4255 
4256  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
4257  if (HiExtType == ISD::NON_EXTLOAD)
4258  HiExtType = ISD::ZEXTLOAD;
4259 
4260  // Load the value in two parts
4261  SDValue Lo, Hi;
4262  if (DAG.getDataLayout().isLittleEndian()) {
4263  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
4264  NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
4265  LD->getAAInfo());
4266 
4267  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4268  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
4269  LD->getPointerInfo().getWithOffset(IncrementSize),
4270  NewLoadedVT, MinAlign(Alignment, IncrementSize),
4271  LD->getMemOperand()->getFlags(), LD->getAAInfo());
4272  } else {
4273  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
4274  NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
4275  LD->getAAInfo());
4276 
4277  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4278  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
4279  LD->getPointerInfo().getWithOffset(IncrementSize),
4280  NewLoadedVT, MinAlign(Alignment, IncrementSize),
4281  LD->getMemOperand()->getFlags(), LD->getAAInfo());
4282  }
4283 
4284  // aggregate the two parts
4285  SDValue ShiftAmount =
4286  DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
4287  DAG.getDataLayout()));
4288  SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
4289  Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
4290 
4291  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4292  Hi.getValue(1));
4293 
4294  return std::make_pair(Result, TF);
4295 }
4296 
4298  SelectionDAG &DAG) const {
4300  "unaligned indexed stores not implemented!");
4301  SDValue Chain = ST->getChain();
4302  SDValue Ptr = ST->getBasePtr();
4303  SDValue Val = ST->getValue();
4304  EVT VT = Val.getValueType();
4305  int Alignment = ST->getAlignment();
4306  auto &MF = DAG.getMachineFunction();
4307  EVT MemVT = ST->getMemoryVT();
4308 
4309  SDLoc dl(ST);
4310  if (MemVT.isFloatingPoint() || MemVT.isVector()) {
4311  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
4312  if (isTypeLegal(intVT)) {
4313  if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
4314  MemVT.isVector()) {
4315  // Scalarize the store and let the individual components be handled.
4316  SDValue Result = scalarizeVectorStore(ST, DAG);
4317 
4318  return Result;
4319  }
4320  // Expand to a bitconvert of the value to the integer type of the
4321  // same size, then a (misaligned) int store.
4322  // FIXME: Does not handle truncating floating point stores!
4323  SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
4324  Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
4325  Alignment, ST->getMemOperand()->getFlags());
4326  return Result;
4327  }
4328  // Do a (aligned) store to a stack slot, then copy from the stack slot
4329  // to the final destination using (unaligned) integer loads and stores.
4330  EVT StoredVT = ST->getMemoryVT();
4331  MVT RegVT =
4332  getRegisterType(*DAG.getContext(),
4334  StoredVT.getSizeInBits()));
4335  EVT PtrVT = Ptr.getValueType();
4336  unsigned StoredBytes = StoredVT.getStoreSize();
4337  unsigned RegBytes = RegVT.getSizeInBits() / 8;
4338  unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
4339 
4340  // Make sure the stack slot is also aligned for the register type.
4341  SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT);
4342  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4343 
4344  // Perform the original store, only redirected to the stack slot.
4345  SDValue Store = DAG.getTruncStore(
4346  Chain, dl, Val, StackPtr,
4347  MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoredVT);
4348 
4349  EVT StackPtrVT = StackPtr.getValueType();
4350 
4351  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
4352  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
4353  SmallVector<SDValue, 8> Stores;
4354  unsigned Offset = 0;
4355 
4356  // Do all but one copies using the full register width.
4357  for (unsigned i = 1; i < NumRegs; i++) {
4358  // Load one integer register's worth from the stack slot.
4359  SDValue Load = DAG.getLoad(
4360  RegVT, dl, Store, StackPtr,
4362  // Store it to the final location. Remember the store.
4363  Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
4364  ST->getPointerInfo().getWithOffset(Offset),
4365  MinAlign(ST->getAlignment(), Offset),
4366  ST->getMemOperand()->getFlags()));
4367  // Increment the pointers.
4368  Offset += RegBytes;
4369  StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
4370  Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
4371  }
4372 
4373  // The last store may be partial. Do a truncating store. On big-endian
4374  // machines this requires an extending load from the stack slot to ensure
4375  // that the bits are in the right place.
4376  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
4377  8 * (StoredBytes - Offset));
4378 
4379  // Load from the stack slot.
4380  SDValue Load = DAG.getExtLoad(
4381  ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
4382  MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT);
4383 
4384  Stores.push_back(
4385  DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
4386  ST->getPointerInfo().getWithOffset(Offset), MemVT,
4387  MinAlign(ST->getAlignment(), Offset),
4388  ST->getMemOperand()->getFlags(), ST->getAAInfo()));
4389  // The order of the stores doesn't matter - say it with a TokenFactor.
4390  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
4391  return Result;
4392  }
4393 
4394  assert(ST->getMemoryVT().isInteger() &&
4395  !ST->getMemoryVT().isVector() &&
4396  "Unaligned store of unknown type.");
4397  // Get the half-size VT
4398  EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext());
4399  int NumBits = NewStoredVT.getSizeInBits();
4400  int IncrementSize = NumBits / 8;
4401 
4402  // Divide the stored value in two parts.
4403  SDValue ShiftAmount =
4404  DAG.getConstant(NumBits, dl, getShiftAmountTy(Val.getValueType(),
4405  DAG.getDataLayout()));
4406  SDValue Lo = Val;
4407  SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
4408 
4409  // Store the two parts
4410  SDValue Store1, Store2;
4411  Store1 = DAG.getTruncStore(Chain, dl,
4412  DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
4413  Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
4414  ST->getMemOperand()->getFlags());
4415 
4416  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4417  Alignment = MinAlign(Alignment, IncrementSize);
4418  Store2 = DAG.getTruncStore(
4419  Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
4420  ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
4421  ST->getMemOperand()->getFlags(), ST->getAAInfo());
4422 
4423  SDValue Result =
4424  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
4425  return Result;
4426 }
4427 
4428 SDValue
4430  const SDLoc &DL, EVT DataVT,
4431  SelectionDAG &DAG,
4432  bool IsCompressedMemory) const {
4433  SDValue Increment;
4434  EVT AddrVT = Addr.getValueType();
4435  EVT MaskVT = Mask.getValueType();
4436  assert(DataVT.getVectorNumElements() == MaskVT.getVectorNumElements() &&
4437  "Incompatible types of Data and Mask");
4438  if (IsCompressedMemory) {
4439  // Incrementing the pointer according to number of '1's in the mask.
4440  EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
4441  SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
4442  if (MaskIntVT.getSizeInBits() < 32) {
4443  MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
4444  MaskIntVT = MVT::i32;
4445  }
4446 
4447  // Count '1's with POPCNT.
4448  Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
4449  Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
4450  // Scale is an element size in bytes.
4451  SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
4452  AddrVT);
4453  Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
4454  } else
4455  Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);
4456 
4457  return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
4458 }
4459 
4461  SDValue Idx,
4462  EVT VecVT,
4463  const SDLoc &dl) {
4464  if (isa<ConstantSDNode>(Idx))
4465  return Idx;
4466 
4467  EVT IdxVT = Idx.getValueType();
4468  unsigned NElts = VecVT.getVectorNumElements();
4469  if (isPowerOf2_32(NElts)) {
4471  Log2_32(NElts));
4472  return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
4473  DAG.getConstant(Imm, dl, IdxVT));
4474  }
4475 
4476  return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
4477  DAG.getConstant(NElts - 1, dl, IdxVT));
4478 }
4479 
4481  SDValue VecPtr, EVT VecVT,
4482  SDValue Index) const {
4483  SDLoc dl(Index);
4484  // Make sure the index type is big enough to compute in.
4485  Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
4486 
4487  EVT EltVT = VecVT.getVectorElementType();
4488 
4489  // Calculate the element offset and add it to the pointer.
4490  unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
4491  assert(EltSize * 8 == EltVT.getSizeInBits() &&
4492  "Converting bits to bytes lost precision");
4493 
4494  Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl);
4495 
4496  EVT IdxVT = Index.getValueType();
4497 
4498  Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
4499  DAG.getConstant(EltSize, dl, IdxVT));
4500  return DAG.getNode(ISD::ADD, dl, IdxVT, VecPtr, Index);
4501 }
4502 
4503 //===----------------------------------------------------------------------===//
4504 // Implementation of Emulated TLS Model
4505 //===----------------------------------------------------------------------===//
4506 
4508  SelectionDAG &DAG) const {
4509  // Access to address of TLS varialbe xyz is lowered to a function call:
4510  // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
4511  EVT PtrVT = getPointerTy(DAG.getDataLayout());
4512  PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
4513  SDLoc dl(GA);
4514 
4515  ArgListTy Args;
4516  ArgListEntry Entry;
4517  std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
4518  Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
4519  StringRef EmuTlsVarName(NameString);
4520  GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
4521  assert(EmuTlsVar && "Cannot find EmuTlsVar ");
4522  Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
4523  Entry.Ty = VoidPtrType;
4524  Args.push_back(Entry);
4525 
4526  SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
4527 
4529  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
4530  CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
4531  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
4532 
4533  // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
4534  // At last for X86 targets, maybe good for other targets too?
4536  MFI.setAdjustsStack(true); // Is this only for X86 target?
4537  MFI.setHasCalls(true);
4538 
4539  assert((GA->getOffset() == 0) &&
4540  "Emulated TLS must have zero offset in GlobalAddressSDNode");
4541  return CallResult.first;
4542 }
4543 
4545  SelectionDAG &DAG) const {
4546  assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
4547  if (!isCtlzFast())
4548  return SDValue();
4549  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
4550  SDLoc dl(Op);
4551  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
4552  if (C->isNullValue() && CC == ISD::SETEQ) {
4553  EVT VT = Op.getOperand(0).getValueType();
4554  SDValue Zext = Op.getOperand(0);
4555  if (VT.bitsLT(MVT::i32)) {
4556  VT = MVT::i32;
4557  Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
4558  }
4559  unsigned Log2b = Log2_32(VT.getSizeInBits());
4560  SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
4561  SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
4562  DAG.getConstant(Log2b, dl, MVT::i32));
4563  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
4564  }
4565  }
4566  return SDValue();
4567 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:541
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
static MVT getIntegerVT(unsigned BitWidth)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:552
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:837
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isInteger() const
Return true if this is an integer or a vector integer type.
void setSignBit()
Set the sign bit to 1.
Definition: APInt.h:1407
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
bool isUndef() const
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
const GlobalValue * getGlobal() const
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1557
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:562
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1412
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:951
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:322
ConstantSDNode * isConstOrConstSplat(SDValue N)
Returns the SDNode if it is a constant splat BuildVector or constant int.
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:334
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
unsigned getParamAlignment(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Definition: CallSite.h:406
Various leaf nodes.
Definition: ISDOpcodes.h:60
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:64
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:265
iterator begin() const
Definition: ArrayRef.h:137
#define LLVM_FALLTHROUGH
Definition: Compiler.h:86
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:223
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
unsigned s
shift amount
Definition: APInt.h:1951
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:138
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:858
const SDValue & getValue() const
unsigned Reg
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:648
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
int MatchingInput
MatchingInput - If this is not -1, this is an output constraint where an input constraint is required...
Definition: InlineAsm.h:131
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
The two locations do not alias at all.
Definition: AliasAnalysis.h:85
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1323
static ISD::NodeType getExtendForContent(BooleanContent Content)
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:396
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
unsigned getAlignment() const
bool isBitwiseNot(SDValue V)
Returns true if V is a bitwise not operation.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:811
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1384
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:360
unsigned const TargetRegisterInfo * TRI
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, ImmutableCallSite CS) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
F(f)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:964
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL) const
Soften the operands of a comparison.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is &#39;desirable&#39; to us...
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal...
Definition: ISDOpcodes.h:965
bool isConstTrueVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:329
CallLoweringInfo & setDiscardResult(bool Value=true)
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode *> &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators...
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Magic data for optimising unsigned division by a constant.
Definition: APInt.h:1955