LLVM  4.0.0
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the TargetLowering class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/BitVector.h"
16 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalVariable.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCExpr.h"
35 #include <cctype>
36 using namespace llvm;
37 
38 /// NOTE: The TargetMachine owns TLOF.
40  : TargetLoweringBase(tm) {}
41 
42 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43  return nullptr;
44 }
45 
48 }
49 
50 /// Check whether a given call node is in tail position within its function. If
51 /// so, it sets Chain to the input chain of the tail call.
53  SDValue &Chain) const {
54  const Function *F = DAG.getMachineFunction().getFunction();
55 
56  // Conservatively require the attributes of the call to match those of
57  // the return. Ignore noalias because it doesn't affect the call sequence.
58  AttributeSet CallerAttrs = F->getAttributes();
59  if (AttrBuilder(CallerAttrs, AttributeSet::ReturnIndex)
60  .removeAttribute(Attribute::NoAlias).hasAttributes())
61  return false;
62 
63  // It's not safe to eliminate the sign / zero extension of the return value.
64  if (CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::ZExt) ||
65  CallerAttrs.hasAttribute(AttributeSet::ReturnIndex, Attribute::SExt))
66  return false;
67 
68  // Check if the only use is a function return node.
69  return isUsedByReturnOnly(Node, Chain);
70 }
71 
73  const uint32_t *CallerPreservedMask,
74  const SmallVectorImpl<CCValAssign> &ArgLocs,
75  const SmallVectorImpl<SDValue> &OutVals) const {
76  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
77  const CCValAssign &ArgLoc = ArgLocs[I];
78  if (!ArgLoc.isRegLoc())
79  continue;
80  unsigned Reg = ArgLoc.getLocReg();
81  // Only look at callee saved registers.
82  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
83  continue;
84  // Check that we pass the value used for the caller.
85  // (We look for a CopyFromReg reading a virtual register that is used
86  // for the function live-in value of register Reg)
87  SDValue Value = OutVals[I];
88  if (Value->getOpcode() != ISD::CopyFromReg)
89  return false;
90  unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
91  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
92  return false;
93  }
94  return true;
95 }
96 
97 /// \brief Set CallLoweringInfo attribute flags based on a call instruction
98 /// and called function attributes.
100  unsigned AttrIdx) {
101  isSExt = CS->paramHasAttr(AttrIdx, Attribute::SExt);
102  isZExt = CS->paramHasAttr(AttrIdx, Attribute::ZExt);
103  isInReg = CS->paramHasAttr(AttrIdx, Attribute::InReg);
104  isSRet = CS->paramHasAttr(AttrIdx, Attribute::StructRet);
105  isNest = CS->paramHasAttr(AttrIdx, Attribute::Nest);
106  isByVal = CS->paramHasAttr(AttrIdx, Attribute::ByVal);
107  isInAlloca = CS->paramHasAttr(AttrIdx, Attribute::InAlloca);
108  isReturned = CS->paramHasAttr(AttrIdx, Attribute::Returned);
109  isSwiftSelf = CS->paramHasAttr(AttrIdx, Attribute::SwiftSelf);
110  isSwiftError = CS->paramHasAttr(AttrIdx, Attribute::SwiftError);
111  Alignment = CS->getParamAlignment(AttrIdx);
112 }
113 
114 /// Generate a libcall taking the given operands as arguments and returning a
115 /// result of type RetVT.
116 std::pair<SDValue, SDValue>
118  ArrayRef<SDValue> Ops, bool isSigned,
119  const SDLoc &dl, bool doesNotReturn,
120  bool isReturnValueUsed) const {
122  Args.reserve(Ops.size());
123 
125  for (SDValue Op : Ops) {
126  Entry.Node = Op;
127  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
128  Entry.isSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
129  Entry.isZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
130  Args.push_back(Entry);
131  }
132 
133  if (LC == RTLIB::UNKNOWN_LIBCALL)
134  report_fatal_error("Unsupported library call operation!");
135  SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
136  getPointerTy(DAG.getDataLayout()));
137 
138  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
140  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, isSigned);
141  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
142  .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
143  .setNoReturn(doesNotReturn).setDiscardResult(!isReturnValueUsed)
144  .setSExtResult(signExtend).setZExtResult(!signExtend);
145  return LowerCallTo(CLI);
146 }
147 
148 /// Soften the operands of a comparison. This code is shared among BR_CC,
149 /// SELECT_CC, and SETCC handlers.
151  SDValue &NewLHS, SDValue &NewRHS,
152  ISD::CondCode &CCCode,
153  const SDLoc &dl) const {
154  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
155  && "Unsupported setcc type!");
156 
157  // Expand into one or more soft-fp libcall(s).
159  bool ShouldInvertCC = false;
160  switch (CCCode) {
161  case ISD::SETEQ:
162  case ISD::SETOEQ:
163  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
164  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
166  break;
167  case ISD::SETNE:
168  case ISD::SETUNE:
169  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
170  (VT == MVT::f64) ? RTLIB::UNE_F64 :
172  break;
173  case ISD::SETGE:
174  case ISD::SETOGE:
175  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
176  (VT == MVT::f64) ? RTLIB::OGE_F64 :
178  break;
179  case ISD::SETLT:
180  case ISD::SETOLT:
181  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
182  (VT == MVT::f64) ? RTLIB::OLT_F64 :
184  break;
185  case ISD::SETLE:
186  case ISD::SETOLE:
187  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
188  (VT == MVT::f64) ? RTLIB::OLE_F64 :
190  break;
191  case ISD::SETGT:
192  case ISD::SETOGT:
193  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
194  (VT == MVT::f64) ? RTLIB::OGT_F64 :
196  break;
197  case ISD::SETUO:
198  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
199  (VT == MVT::f64) ? RTLIB::UO_F64 :
201  break;
202  case ISD::SETO:
203  LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
204  (VT == MVT::f64) ? RTLIB::O_F64 :
206  break;
207  case ISD::SETONE:
208  // SETONE = SETOLT | SETOGT
209  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
210  (VT == MVT::f64) ? RTLIB::OLT_F64 :
212  LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
213  (VT == MVT::f64) ? RTLIB::OGT_F64 :
215  break;
216  case ISD::SETUEQ:
217  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
218  (VT == MVT::f64) ? RTLIB::UO_F64 :
220  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
221  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
223  break;
224  default:
225  // Invert CC for unordered comparisons
226  ShouldInvertCC = true;
227  switch (CCCode) {
228  case ISD::SETULT:
229  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
230  (VT == MVT::f64) ? RTLIB::OGE_F64 :
232  break;
233  case ISD::SETULE:
234  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
235  (VT == MVT::f64) ? RTLIB::OGT_F64 :
237  break;
238  case ISD::SETUGT:
239  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
240  (VT == MVT::f64) ? RTLIB::OLE_F64 :
242  break;
243  case ISD::SETUGE:
244  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
245  (VT == MVT::f64) ? RTLIB::OLT_F64 :
247  break;
248  default: llvm_unreachable("Do not know how to soften this setcc!");
249  }
250  }
251 
252  // Use the target specific return value for comparions lib calls.
253  EVT RetVT = getCmpLibcallReturnType();
254  SDValue Ops[2] = {NewLHS, NewRHS};
255  NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, false /*sign irrelevant*/,
256  dl).first;
257  NewRHS = DAG.getConstant(0, dl, RetVT);
258 
259  CCCode = getCmpLibcallCC(LC1);
260  if (ShouldInvertCC)
261  CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);
262 
263  if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
264  SDValue Tmp = DAG.getNode(
265  ISD::SETCC, dl,
266  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
267  NewLHS, NewRHS, DAG.getCondCode(CCCode));
268  NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, false/*sign irrelevant*/,
269  dl).first;
270  NewLHS = DAG.getNode(
271  ISD::SETCC, dl,
272  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
273  NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
274  NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
275  NewRHS = SDValue();
276  }
277 }
278 
279 /// Return the entry encoding for a jump table in the current function. The
280 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
282  // In non-pic modes, just use the address of a block.
283  if (!isPositionIndependent())
285 
286  // In PIC mode, if the target supports a GPRel32 directive, use it.
287  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
289 
290  // Otherwise, use a label difference.
292 }
293 
295  SelectionDAG &DAG) const {
296  // If our PIC model is GP relative, use the global offset table as the base.
297  unsigned JTEncoding = getJumpTableEncoding();
298 
299  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
302 
303  return Table;
304 }
305 
306 /// This returns the relocation base for the given PIC jumptable, the same as
307 /// getPICJumpTableRelocBase, but as an MCExpr.
308 const MCExpr *
310  unsigned JTI,MCContext &Ctx) const{
311  // The normal PIC reloc base is the label at the start of the jump table.
312  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
313 }
314 
315 bool
317  const TargetMachine &TM = getTargetMachine();
318  const GlobalValue *GV = GA->getGlobal();
319 
320  // If the address is not even local to this DSO we will have to load it from
321  // a got and then add the offset.
322  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
323  return false;
324 
325  // If the code is position independent we will have to add a base register.
326  if (isPositionIndependent())
327  return false;
328 
329  // Otherwise we can do it.
330  return true;
331 }
332 
333 //===----------------------------------------------------------------------===//
334 // Optimization Methods
335 //===----------------------------------------------------------------------===//
336 
337 /// Check to see if the specified operand of the specified instruction is a
338 /// constant integer. If so, check to see if there are any bits set in the
339 /// constant that are not demanded. If so, shrink the constant and return true.
341  const APInt &Demanded) {
342  SDLoc dl(Op);
343 
344  // FIXME: ISD::SELECT, ISD::SELECT_CC
345  switch (Op.getOpcode()) {
346  default: break;
347  case ISD::XOR:
348  case ISD::AND:
349  case ISD::OR: {
351  if (!C) return false;
352 
353  if (Op.getOpcode() == ISD::XOR &&
354  (C->getAPIntValue() | (~Demanded)).isAllOnesValue())
355  return false;
356 
357  // if we can expand it to have all bits set, do it
358  if (C->getAPIntValue().intersects(~Demanded)) {
359  EVT VT = Op.getValueType();
360  SDValue New = DAG.getNode(Op.getOpcode(), dl, VT, Op.getOperand(0),
361  DAG.getConstant(Demanded &
362  C->getAPIntValue(),
363  dl, VT));
364  return CombineTo(Op, New);
365  }
366 
367  break;
368  }
369  }
370 
371  return false;
372 }
373 
374 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
375 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
376 /// generalized for targets with other types of implicit widening casts.
378  unsigned BitWidth,
379  const APInt &Demanded,
380  const SDLoc &dl) {
381  assert(Op.getNumOperands() == 2 &&
382  "ShrinkDemandedOp only supports binary operators!");
383  assert(Op.getNode()->getNumValues() == 1 &&
384  "ShrinkDemandedOp only supports nodes with one result!");
385 
386  // Early return, as this function cannot handle vector types.
387  if (Op.getValueType().isVector())
388  return false;
389 
390  // Don't do this if the node has another user, which may require the
391  // full value.
392  if (!Op.getNode()->hasOneUse())
393  return false;
394 
395  // Search for the smallest integer type with free casts to and from
396  // Op's type. For expedience, just check power-of-2 integer types.
397  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
398  unsigned DemandedSize = BitWidth - Demanded.countLeadingZeros();
399  unsigned SmallVTBits = DemandedSize;
400  if (!isPowerOf2_32(SmallVTBits))
401  SmallVTBits = NextPowerOf2(SmallVTBits);
402  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
403  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
404  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
405  TLI.isZExtFree(SmallVT, Op.getValueType())) {
406  // We found a type with free casts.
407  SDValue X = DAG.getNode(Op.getOpcode(), dl, SmallVT,
408  DAG.getNode(ISD::TRUNCATE, dl, SmallVT,
409  Op.getNode()->getOperand(0)),
410  DAG.getNode(ISD::TRUNCATE, dl, SmallVT,
411  Op.getNode()->getOperand(1)));
412  bool NeedZext = DemandedSize > SmallVTBits;
413  SDValue Z = DAG.getNode(NeedZext ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND,
414  dl, Op.getValueType(), X);
415  return CombineTo(Op, Z);
416  }
417  }
418  return false;
419 }
420 
421 bool
423  unsigned OpIdx,
424  const APInt &Demanded,
425  DAGCombinerInfo &DCI) {
426  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
427  SDValue Op = User->getOperand(OpIdx);
428  APInt KnownZero, KnownOne;
429 
430  if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne,
431  *this, 0, true))
432  return false;
433 
434 
435  // Old will not always be the same as Op. For example:
436  //
437  // Demanded = 0xffffff
438  // Op = i64 truncate (i32 and x, 0xffffff)
439  // In this case simplify demand bits will want to replace the 'and' node
440  // with the value 'x', which will give us:
441  // Old = i32 and x, 0xffffff
442  // New = x
443  if (Old.hasOneUse()) {
444  // For the one use case, we just commit the change.
445  DCI.CommitTargetLoweringOpt(*this);
446  return true;
447  }
448 
449  // If Old has more than one use then it must be Op, because the
450  // AssumeSingleUse flag is not propogated to recursive calls of
451  // SimplifyDemanded bits, so the only node with multiple use that
452  // it will attempt to combine will be opt.
453  assert(Old == Op);
454 
456  for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
457  if (i == OpIdx) {
458  NewOps.push_back(New);
459  continue;
460  }
461  NewOps.push_back(User->getOperand(i));
462  }
463  DAG.UpdateNodeOperands(User, NewOps);
464  // Op has less users now, so we may be able to perform additional combines
465  // with it.
466  DCI.AddToWorklist(Op.getNode());
467  // User's operands have been updated, so we may be able to do new combines
468  // with it.
469  DCI.AddToWorklist(User);
470  return true;
471 }
472 
473 /// Look at Op. At this point, we know that only the DemandedMask bits of the
474 /// result of Op are ever used downstream. If we can use this information to
475 /// simplify Op, create a new simplified DAG node and return true, returning the
476 /// original and new nodes in Old and New. Otherwise, analyze the expression and
477 /// return a mask of KnownOne and KnownZero bits for the expression (used to
478 /// simplify the caller). The KnownZero/One bits may only be accurate for those
479 /// bits in the DemandedMask.
481  const APInt &DemandedMask,
482  APInt &KnownZero,
483  APInt &KnownOne,
484  TargetLoweringOpt &TLO,
485  unsigned Depth,
486  bool AssumeSingleUse) const {
487  unsigned BitWidth = DemandedMask.getBitWidth();
488  assert(Op.getScalarValueSizeInBits() == BitWidth &&
489  "Mask size mismatches value type size!");
490  APInt NewMask = DemandedMask;
491  SDLoc dl(Op);
492  auto &DL = TLO.DAG.getDataLayout();
493 
494  // Don't know anything.
495  KnownZero = KnownOne = APInt(BitWidth, 0);
496 
497  // Other users may use these bits.
498  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
499  if (Depth != 0) {
500  // If not at the root, Just compute the KnownZero/KnownOne bits to
501  // simplify things downstream.
502  TLO.DAG.computeKnownBits(Op, KnownZero, KnownOne, Depth);
503  return false;
504  }
505  // If this is the root being simplified, allow it to have multiple uses,
506  // just set the NewMask to all bits.
507  NewMask = APInt::getAllOnesValue(BitWidth);
508  } else if (DemandedMask == 0) {
509  // Not demanding any bits from Op.
510  if (!Op.isUndef())
511  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(Op.getValueType()));
512  return false;
513  } else if (Depth == 6) { // Limit search depth.
514  return false;
515  }
516 
517  APInt KnownZero2, KnownOne2, KnownZeroOut, KnownOneOut;
518  switch (Op.getOpcode()) {
519  case ISD::Constant:
520  // We know all of the bits for a constant!
521  KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
522  KnownZero = ~KnownOne;
523  return false; // Don't fall through, will infinitely loop.
524  case ISD::BUILD_VECTOR:
525  // Collect the known bits that are shared by every constant vector element.
526  KnownZero = KnownOne = APInt::getAllOnesValue(BitWidth);
527  for (SDValue SrcOp : Op->ops()) {
528  if (!isa<ConstantSDNode>(SrcOp)) {
529  // We can only handle all constant values - bail out with no known bits.
530  KnownZero = KnownOne = APInt(BitWidth, 0);
531  return false;
532  }
533  KnownOne2 = cast<ConstantSDNode>(SrcOp)->getAPIntValue();
534  KnownZero2 = ~KnownOne2;
535 
536  // BUILD_VECTOR can implicitly truncate sources, we must handle this.
537  if (KnownOne2.getBitWidth() != BitWidth) {
538  assert(KnownOne2.getBitWidth() > BitWidth &&
539  KnownZero2.getBitWidth() > BitWidth &&
540  "Expected BUILD_VECTOR implicit truncation");
541  KnownOne2 = KnownOne2.trunc(BitWidth);
542  KnownZero2 = KnownZero2.trunc(BitWidth);
543  }
544 
545  // Known bits are the values that are shared by every element.
546  // TODO: support per-element known bits.
547  KnownOne &= KnownOne2;
548  KnownZero &= KnownZero2;
549  }
550  return false; // Don't fall through, will infinitely loop.
551  case ISD::AND:
552  // If the RHS is a constant, check to see if the LHS would be zero without
553  // using the bits from the RHS. Below, we use knowledge about the RHS to
554  // simplify the LHS, here we're using information from the LHS to simplify
555  // the RHS.
556  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
557  SDValue Op0 = Op.getOperand(0);
558  APInt LHSZero, LHSOne;
559  // Do not increment Depth here; that can cause an infinite loop.
560  TLO.DAG.computeKnownBits(Op0, LHSZero, LHSOne, Depth);
561  // If the LHS already has zeros where RHSC does, this and is dead.
562  if ((LHSZero & NewMask) == (~RHSC->getAPIntValue() & NewMask))
563  return TLO.CombineTo(Op, Op0);
564 
565  // If any of the set bits in the RHS are known zero on the LHS, shrink
566  // the constant.
567  if (TLO.ShrinkDemandedConstant(Op, ~LHSZero & NewMask))
568  return true;
569 
570  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
571  // constant, but if this 'and' is only clearing bits that were just set by
572  // the xor, then this 'and' can be eliminated by shrinking the mask of
573  // the xor. For example, for a 32-bit X:
574  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
575  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
576  LHSOne == ~RHSC->getAPIntValue()) {
577  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, Op.getValueType(),
578  Op0.getOperand(0), Op.getOperand(1));
579  return TLO.CombineTo(Op, Xor);
580  }
581  }
582 
583  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, KnownZero,
584  KnownOne, TLO, Depth+1))
585  return true;
586  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
587  if (SimplifyDemandedBits(Op.getOperand(0), ~KnownZero & NewMask,
588  KnownZero2, KnownOne2, TLO, Depth+1))
589  return true;
590  assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
591 
592  // If all of the demanded bits are known one on one side, return the other.
593  // These bits cannot contribute to the result of the 'and'.
594  if ((NewMask & ~KnownZero2 & KnownOne) == (~KnownZero2 & NewMask))
595  return TLO.CombineTo(Op, Op.getOperand(0));
596  if ((NewMask & ~KnownZero & KnownOne2) == (~KnownZero & NewMask))
597  return TLO.CombineTo(Op, Op.getOperand(1));
598  // If all of the demanded bits in the inputs are known zeros, return zero.
599  if ((NewMask & (KnownZero|KnownZero2)) == NewMask)
600  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, Op.getValueType()));
601  // If the RHS is a constant, see if we can simplify it.
602  if (TLO.ShrinkDemandedConstant(Op, ~KnownZero2 & NewMask))
603  return true;
604  // If the operation can be done in a smaller type, do so.
605  if (TLO.ShrinkDemandedOp(Op, BitWidth, NewMask, dl))
606  return true;
607 
608  // Output known-1 bits are only known if set in both the LHS & RHS.
609  KnownOne &= KnownOne2;
610  // Output known-0 are known to be clear if zero in either the LHS | RHS.
611  KnownZero |= KnownZero2;
612  break;
613  case ISD::OR:
614  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, KnownZero,
615  KnownOne, TLO, Depth+1))
616  return true;
617  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
618  if (SimplifyDemandedBits(Op.getOperand(0), ~KnownOne & NewMask,
619  KnownZero2, KnownOne2, TLO, Depth+1))
620  return true;
621  assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
622 
623  // If all of the demanded bits are known zero on one side, return the other.
624  // These bits cannot contribute to the result of the 'or'.
625  if ((NewMask & ~KnownOne2 & KnownZero) == (~KnownOne2 & NewMask))
626  return TLO.CombineTo(Op, Op.getOperand(0));
627  if ((NewMask & ~KnownOne & KnownZero2) == (~KnownOne & NewMask))
628  return TLO.CombineTo(Op, Op.getOperand(1));
629  // If all of the potentially set bits on one side are known to be set on
630  // the other side, just use the 'other' side.
631  if ((NewMask & ~KnownZero & KnownOne2) == (~KnownZero & NewMask))
632  return TLO.CombineTo(Op, Op.getOperand(0));
633  if ((NewMask & ~KnownZero2 & KnownOne) == (~KnownZero2 & NewMask))
634  return TLO.CombineTo(Op, Op.getOperand(1));
635  // If the RHS is a constant, see if we can simplify it.
636  if (TLO.ShrinkDemandedConstant(Op, NewMask))
637  return true;
638  // If the operation can be done in a smaller type, do so.
639  if (TLO.ShrinkDemandedOp(Op, BitWidth, NewMask, dl))
640  return true;
641 
642  // Output known-0 bits are only known if clear in both the LHS & RHS.
643  KnownZero &= KnownZero2;
644  // Output known-1 are known to be set if set in either the LHS | RHS.
645  KnownOne |= KnownOne2;
646  break;
647  case ISD::XOR:
648  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, KnownZero,
649  KnownOne, TLO, Depth+1))
650  return true;
651  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
652  if (SimplifyDemandedBits(Op.getOperand(0), NewMask, KnownZero2,
653  KnownOne2, TLO, Depth+1))
654  return true;
655  assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
656 
657  // If all of the demanded bits are known zero on one side, return the other.
658  // These bits cannot contribute to the result of the 'xor'.
659  if ((KnownZero & NewMask) == NewMask)
660  return TLO.CombineTo(Op, Op.getOperand(0));
661  if ((KnownZero2 & NewMask) == NewMask)
662  return TLO.CombineTo(Op, Op.getOperand(1));
663  // If the operation can be done in a smaller type, do so.
664  if (TLO.ShrinkDemandedOp(Op, BitWidth, NewMask, dl))
665  return true;
666 
667  // If all of the unknown bits are known to be zero on one side or the other
668  // (but not both) turn this into an *inclusive* or.
669  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
670  if ((NewMask & ~KnownZero & ~KnownZero2) == 0)
671  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, Op.getValueType(),
672  Op.getOperand(0),
673  Op.getOperand(1)));
674 
675  // Output known-0 bits are known if clear or set in both the LHS & RHS.
676  KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
677  // Output known-1 are known to be set if set in only one of the LHS, RHS.
678  KnownOneOut = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
679 
680  // If all of the demanded bits on one side are known, and all of the set
681  // bits on that side are also known to be set on the other side, turn this
682  // into an AND, as we know the bits will be cleared.
683  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
684  // NB: it is okay if more bits are known than are requested
685  if ((NewMask & (KnownZero|KnownOne)) == NewMask) { // all known on one side
686  if (KnownOne == KnownOne2) { // set bits are the same on both sides
687  EVT VT = Op.getValueType();
688  SDValue ANDC = TLO.DAG.getConstant(~KnownOne & NewMask, dl, VT);
689  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT,
690  Op.getOperand(0), ANDC));
691  }
692  }
693 
694  // If the RHS is a constant, see if we can simplify it.
695  // for XOR, we prefer to force bits to 1 if they will make a -1.
696  // If we can't force bits, try to shrink the constant.
697  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
698  APInt Expanded = C->getAPIntValue() | (~NewMask);
699  // If we can expand it to have all bits set, do it.
700  if (Expanded.isAllOnesValue()) {
701  if (Expanded != C->getAPIntValue()) {
702  EVT VT = Op.getValueType();
703  SDValue New = TLO.DAG.getNode(Op.getOpcode(), dl,VT, Op.getOperand(0),
704  TLO.DAG.getConstant(Expanded, dl, VT));
705  return TLO.CombineTo(Op, New);
706  }
707  // If it already has all the bits set, nothing to change
708  // but don't shrink either!
709  } else if (TLO.ShrinkDemandedConstant(Op, NewMask)) {
710  return true;
711  }
712  }
713 
714  KnownZero = KnownZeroOut;
715  KnownOne = KnownOneOut;
716  break;
717  case ISD::SELECT:
718  if (SimplifyDemandedBits(Op.getOperand(2), NewMask, KnownZero,
719  KnownOne, TLO, Depth+1))
720  return true;
721  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, KnownZero2,
722  KnownOne2, TLO, Depth+1))
723  return true;
724  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
725  assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
726 
727  // If the operands are constants, see if we can simplify them.
728  if (TLO.ShrinkDemandedConstant(Op, NewMask))
729  return true;
730 
731  // Only known if known in both the LHS and RHS.
732  KnownOne &= KnownOne2;
733  KnownZero &= KnownZero2;
734  break;
735  case ISD::SELECT_CC:
736  if (SimplifyDemandedBits(Op.getOperand(3), NewMask, KnownZero,
737  KnownOne, TLO, Depth+1))
738  return true;
739  if (SimplifyDemandedBits(Op.getOperand(2), NewMask, KnownZero2,
740  KnownOne2, TLO, Depth+1))
741  return true;
742  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
743  assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
744 
745  // If the operands are constants, see if we can simplify them.
746  if (TLO.ShrinkDemandedConstant(Op, NewMask))
747  return true;
748 
749  // Only known if known in both the LHS and RHS.
750  KnownOne &= KnownOne2;
751  KnownZero &= KnownZero2;
752  break;
753  case ISD::SHL:
754  if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
755  unsigned ShAmt = SA->getZExtValue();
756  SDValue InOp = Op.getOperand(0);
757 
758  // If the shift count is an invalid immediate, don't do anything.
759  if (ShAmt >= BitWidth)
760  break;
761 
762  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
763  // single shift. We can do this if the bottom bits (which are shifted
764  // out) are never demanded.
765  if (InOp.getOpcode() == ISD::SRL &&
766  isa<ConstantSDNode>(InOp.getOperand(1))) {
767  if (ShAmt && (NewMask & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) {
768  unsigned C1= cast<ConstantSDNode>(InOp.getOperand(1))->getZExtValue();
769  unsigned Opc = ISD::SHL;
770  int Diff = ShAmt-C1;
771  if (Diff < 0) {
772  Diff = -Diff;
773  Opc = ISD::SRL;
774  }
775 
776  SDValue NewSA =
777  TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
778  EVT VT = Op.getValueType();
779  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
780  InOp.getOperand(0), NewSA));
781  }
782  }
783 
784  if (SimplifyDemandedBits(InOp, NewMask.lshr(ShAmt),
785  KnownZero, KnownOne, TLO, Depth+1))
786  return true;
787 
788  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
789  // are not demanded. This will likely allow the anyext to be folded away.
790  if (InOp.getNode()->getOpcode() == ISD::ANY_EXTEND) {
791  SDValue InnerOp = InOp.getNode()->getOperand(0);
792  EVT InnerVT = InnerOp.getValueType();
793  unsigned InnerBits = InnerVT.getSizeInBits();
794  if (ShAmt < InnerBits && NewMask.lshr(InnerBits) == 0 &&
795  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
796  EVT ShTy = getShiftAmountTy(InnerVT, DL);
797  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
798  ShTy = InnerVT;
799  SDValue NarrowShl =
800  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
801  TLO.DAG.getConstant(ShAmt, dl, ShTy));
802  return
803  TLO.CombineTo(Op,
804  TLO.DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(),
805  NarrowShl));
806  }
807  // Repeat the SHL optimization above in cases where an extension
808  // intervenes: (shl (anyext (shr x, c1)), c2) to
809  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
810  // aren't demanded (as above) and that the shifted upper c1 bits of
811  // x aren't demanded.
812  if (InOp.hasOneUse() &&
813  InnerOp.getOpcode() == ISD::SRL &&
814  InnerOp.hasOneUse() &&
815  isa<ConstantSDNode>(InnerOp.getOperand(1))) {
816  uint64_t InnerShAmt = cast<ConstantSDNode>(InnerOp.getOperand(1))
817  ->getZExtValue();
818  if (InnerShAmt < ShAmt &&
819  InnerShAmt < InnerBits &&
820  NewMask.lshr(InnerBits - InnerShAmt + ShAmt) == 0 &&
821  NewMask.trunc(ShAmt) == 0) {
822  SDValue NewSA =
823  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
824  Op.getOperand(1).getValueType());
825  EVT VT = Op.getValueType();
826  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
827  InnerOp.getOperand(0));
828  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT,
829  NewExt, NewSA));
830  }
831  }
832  }
833 
834  KnownZero <<= SA->getZExtValue();
835  KnownOne <<= SA->getZExtValue();
836  // low bits known zero.
837  KnownZero |= APInt::getLowBitsSet(BitWidth, SA->getZExtValue());
838  }
839  break;
840  case ISD::SRL:
841  if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
842  EVT VT = Op.getValueType();
843  unsigned ShAmt = SA->getZExtValue();
844  unsigned VTSize = VT.getSizeInBits();
845  SDValue InOp = Op.getOperand(0);
846 
847  // If the shift count is an invalid immediate, don't do anything.
848  if (ShAmt >= BitWidth)
849  break;
850 
851  APInt InDemandedMask = (NewMask << ShAmt);
852 
853  // If the shift is exact, then it does demand the low bits (and knows that
854  // they are zero).
855  if (cast<BinaryWithFlagsSDNode>(Op)->Flags.hasExact())
856  InDemandedMask |= APInt::getLowBitsSet(BitWidth, ShAmt);
857 
858  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
859  // single shift. We can do this if the top bits (which are shifted out)
860  // are never demanded.
861  if (InOp.getOpcode() == ISD::SHL &&
862  isa<ConstantSDNode>(InOp.getOperand(1))) {
863  if (ShAmt && (NewMask & APInt::getHighBitsSet(VTSize, ShAmt)) == 0) {
864  unsigned C1= cast<ConstantSDNode>(InOp.getOperand(1))->getZExtValue();
865  unsigned Opc = ISD::SRL;
866  int Diff = ShAmt-C1;
867  if (Diff < 0) {
868  Diff = -Diff;
869  Opc = ISD::SHL;
870  }
871 
872  SDValue NewSA =
873  TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
874  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
875  InOp.getOperand(0), NewSA));
876  }
877  }
878 
879  // Compute the new bits that are at the top now.
880  if (SimplifyDemandedBits(InOp, InDemandedMask,
881  KnownZero, KnownOne, TLO, Depth+1))
882  return true;
883  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
884  KnownZero = KnownZero.lshr(ShAmt);
885  KnownOne = KnownOne.lshr(ShAmt);
886 
887  APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
888  KnownZero |= HighBits; // High bits known zero.
889  }
890  break;
891  case ISD::SRA:
892  // If this is an arithmetic shift right and only the low-bit is set, we can
893  // always convert this into a logical shr, even if the shift amount is
894  // variable. The low bit of the shift cannot be an input sign bit unless
895  // the shift amount is >= the size of the datatype, which is undefined.
896  if (NewMask == 1)
897  return TLO.CombineTo(Op,
898  TLO.DAG.getNode(ISD::SRL, dl, Op.getValueType(),
899  Op.getOperand(0), Op.getOperand(1)));
900 
901  if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
902  EVT VT = Op.getValueType();
903  unsigned ShAmt = SA->getZExtValue();
904 
905  // If the shift count is an invalid immediate, don't do anything.
906  if (ShAmt >= BitWidth)
907  break;
908 
909  APInt InDemandedMask = (NewMask << ShAmt);
910 
911  // If the shift is exact, then it does demand the low bits (and knows that
912  // they are zero).
913  if (cast<BinaryWithFlagsSDNode>(Op)->Flags.hasExact())
914  InDemandedMask |= APInt::getLowBitsSet(BitWidth, ShAmt);
915 
916  // If any of the demanded bits are produced by the sign extension, we also
917  // demand the input sign bit.
918  APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
919  if (HighBits.intersects(NewMask))
920  InDemandedMask |= APInt::getSignBit(VT.getScalarSizeInBits());
921 
922  if (SimplifyDemandedBits(Op.getOperand(0), InDemandedMask,
923  KnownZero, KnownOne, TLO, Depth+1))
924  return true;
925  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
926  KnownZero = KnownZero.lshr(ShAmt);
927  KnownOne = KnownOne.lshr(ShAmt);
928 
929  // Handle the sign bit, adjusted to where it is now in the mask.
930  APInt SignBit = APInt::getSignBit(BitWidth).lshr(ShAmt);
931 
932  // If the input sign bit is known to be zero, or if none of the top bits
933  // are demanded, turn this into an unsigned shift right.
934  if (KnownZero.intersects(SignBit) || (HighBits & ~NewMask) == HighBits) {
936  Flags.setExact(cast<BinaryWithFlagsSDNode>(Op)->Flags.hasExact());
937  return TLO.CombineTo(Op,
938  TLO.DAG.getNode(ISD::SRL, dl, VT, Op.getOperand(0),
939  Op.getOperand(1), &Flags));
940  }
941 
942  int Log2 = NewMask.exactLogBase2();
943  if (Log2 >= 0) {
944  // The bit must come from the sign.
945  SDValue NewSA =
946  TLO.DAG.getConstant(BitWidth - 1 - Log2, dl,
947  Op.getOperand(1).getValueType());
948  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT,
949  Op.getOperand(0), NewSA));
950  }
951 
952  if (KnownOne.intersects(SignBit))
953  // New bits are known one.
954  KnownOne |= HighBits;
955  }
956  break;
957  case ISD::SIGN_EXTEND_INREG: {
958  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
959 
960  APInt MsbMask = APInt::getHighBitsSet(BitWidth, 1);
961  // If we only care about the highest bit, don't bother shifting right.
962  if (MsbMask == NewMask) {
963  unsigned ShAmt = ExVT.getScalarSizeInBits();
964  SDValue InOp = Op.getOperand(0);
965  unsigned VTBits = Op->getValueType(0).getScalarSizeInBits();
966  bool AlreadySignExtended =
967  TLO.DAG.ComputeNumSignBits(InOp) >= VTBits-ShAmt+1;
968  // However if the input is already sign extended we expect the sign
969  // extension to be dropped altogether later and do not simplify.
970  if (!AlreadySignExtended) {
971  // Compute the correct shift amount type, which must be getShiftAmountTy
972  // for scalar types after legalization.
973  EVT ShiftAmtTy = Op.getValueType();
974  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
975  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
976 
977  SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ShAmt, dl,
978  ShiftAmtTy);
979  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl,
980  Op.getValueType(), InOp,
981  ShiftAmt));
982  }
983  }
984 
985  // Sign extension. Compute the demanded bits in the result that are not
986  // present in the input.
987  APInt NewBits =
988  APInt::getHighBitsSet(BitWidth,
989  BitWidth - ExVT.getScalarSizeInBits());
990 
991  // If none of the extended bits are demanded, eliminate the sextinreg.
992  if ((NewBits & NewMask) == 0)
993  return TLO.CombineTo(Op, Op.getOperand(0));
994 
995  APInt InSignBit =
996  APInt::getSignBit(ExVT.getScalarSizeInBits()).zext(BitWidth);
997  APInt InputDemandedBits =
998  APInt::getLowBitsSet(BitWidth,
999  ExVT.getScalarSizeInBits()) &
1000  NewMask;
1001 
1002  // Since the sign extended bits are demanded, we know that the sign
1003  // bit is demanded.
1004  InputDemandedBits |= InSignBit;
1005 
1006  if (SimplifyDemandedBits(Op.getOperand(0), InputDemandedBits,
1007  KnownZero, KnownOne, TLO, Depth+1))
1008  return true;
1009  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1010 
1011  // If the sign bit of the input is known set or clear, then we know the
1012  // top bits of the result.
1013 
1014  // If the input sign bit is known zero, convert this into a zero extension.
1015  if (KnownZero.intersects(InSignBit))
1016  return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(
1017  Op.getOperand(0), dl, ExVT.getScalarType()));
1018 
1019  if (KnownOne.intersects(InSignBit)) { // Input sign bit known set
1020  KnownOne |= NewBits;
1021  KnownZero &= ~NewBits;
1022  } else { // Input sign bit unknown
1023  KnownZero &= ~NewBits;
1024  KnownOne &= ~NewBits;
1025  }
1026  break;
1027  }
1028  case ISD::BUILD_PAIR: {
1029  EVT HalfVT = Op.getOperand(0).getValueType();
1030  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1031 
1032  APInt MaskLo = NewMask.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1033  APInt MaskHi = NewMask.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1034 
1035  APInt KnownZeroLo, KnownOneLo;
1036  APInt KnownZeroHi, KnownOneHi;
1037 
1038  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownZeroLo,
1039  KnownOneLo, TLO, Depth + 1))
1040  return true;
1041 
1042  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownZeroHi,
1043  KnownOneHi, TLO, Depth + 1))
1044  return true;
1045 
1046  KnownZero = KnownZeroLo.zext(BitWidth) |
1047  KnownZeroHi.zext(BitWidth).shl(HalfBitWidth);
1048 
1049  KnownOne = KnownOneLo.zext(BitWidth) |
1050  KnownOneHi.zext(BitWidth).shl(HalfBitWidth);
1051  break;
1052  }
1053  case ISD::ZERO_EXTEND: {
1054  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1055  APInt InMask = NewMask.trunc(OperandBitWidth);
1056 
1057  // If none of the top bits are demanded, convert this into an any_extend.
1058  APInt NewBits =
1059  APInt::getHighBitsSet(BitWidth, BitWidth - OperandBitWidth) & NewMask;
1060  if (!NewBits.intersects(NewMask))
1061  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl,
1062  Op.getValueType(),
1063  Op.getOperand(0)));
1064 
1065  if (SimplifyDemandedBits(Op.getOperand(0), InMask,
1066  KnownZero, KnownOne, TLO, Depth+1))
1067  return true;
1068  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1069  KnownZero = KnownZero.zext(BitWidth);
1070  KnownOne = KnownOne.zext(BitWidth);
1071  KnownZero |= NewBits;
1072  break;
1073  }
1074  case ISD::SIGN_EXTEND: {
1075  EVT InVT = Op.getOperand(0).getValueType();
1076  unsigned InBits = InVT.getScalarSizeInBits();
1077  APInt InMask = APInt::getLowBitsSet(BitWidth, InBits);
1078  APInt InSignBit = APInt::getBitsSet(BitWidth, InBits - 1, InBits);
1079  APInt NewBits = ~InMask & NewMask;
1080 
1081  // If none of the top bits are demanded, convert this into an any_extend.
1082  if (NewBits == 0)
1083  return TLO.CombineTo(Op,TLO.DAG.getNode(ISD::ANY_EXTEND, dl,
1084  Op.getValueType(),
1085  Op.getOperand(0)));
1086 
1087  // Since some of the sign extended bits are demanded, we know that the sign
1088  // bit is demanded.
1089  APInt InDemandedBits = InMask & NewMask;
1090  InDemandedBits |= InSignBit;
1091  InDemandedBits = InDemandedBits.trunc(InBits);
1092 
1093  if (SimplifyDemandedBits(Op.getOperand(0), InDemandedBits, KnownZero,
1094  KnownOne, TLO, Depth+1))
1095  return true;
1096  KnownZero = KnownZero.zext(BitWidth);
1097  KnownOne = KnownOne.zext(BitWidth);
1098 
1099  // If the sign bit is known zero, convert this to a zero extend.
1100  if (KnownZero.intersects(InSignBit))
1101  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl,
1102  Op.getValueType(),
1103  Op.getOperand(0)));
1104 
1105  // If the sign bit is known one, the top bits match.
1106  if (KnownOne.intersects(InSignBit)) {
1107  KnownOne |= NewBits;
1108  assert((KnownZero & NewBits) == 0);
1109  } else { // Otherwise, top bits aren't known.
1110  assert((KnownOne & NewBits) == 0);
1111  assert((KnownZero & NewBits) == 0);
1112  }
1113  break;
1114  }
1115  case ISD::ANY_EXTEND: {
1116  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1117  APInt InMask = NewMask.trunc(OperandBitWidth);
1118  if (SimplifyDemandedBits(Op.getOperand(0), InMask,
1119  KnownZero, KnownOne, TLO, Depth+1))
1120  return true;
1121  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1122  KnownZero = KnownZero.zext(BitWidth);
1123  KnownOne = KnownOne.zext(BitWidth);
1124  break;
1125  }
1126  case ISD::TRUNCATE: {
1127  // Simplify the input, using demanded bit information, and compute the known
1128  // zero/one bits live out.
1129  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1130  APInt TruncMask = NewMask.zext(OperandBitWidth);
1131  if (SimplifyDemandedBits(Op.getOperand(0), TruncMask,
1132  KnownZero, KnownOne, TLO, Depth+1))
1133  return true;
1134  KnownZero = KnownZero.trunc(BitWidth);
1135  KnownOne = KnownOne.trunc(BitWidth);
1136 
1137  // If the input is only used by this truncate, see if we can shrink it based
1138  // on the known demanded bits.
1139  if (Op.getOperand(0).getNode()->hasOneUse()) {
1140  SDValue In = Op.getOperand(0);
1141  switch (In.getOpcode()) {
1142  default: break;
1143  case ISD::SRL:
1144  // Shrink SRL by a constant if none of the high bits shifted in are
1145  // demanded.
1146  if (TLO.LegalTypes() &&
1148  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1149  // undesirable.
1150  break;
1152  if (!ShAmt)
1153  break;
1154  SDValue Shift = In.getOperand(1);
1155  if (TLO.LegalTypes()) {
1156  uint64_t ShVal = ShAmt->getZExtValue();
1157  Shift = TLO.DAG.getConstant(ShVal, dl,
1158  getShiftAmountTy(Op.getValueType(), DL));
1159  }
1160 
1161  APInt HighBits = APInt::getHighBitsSet(OperandBitWidth,
1162  OperandBitWidth - BitWidth);
1163  HighBits = HighBits.lshr(ShAmt->getZExtValue()).trunc(BitWidth);
1164 
1165  if (ShAmt->getZExtValue() < BitWidth && !(HighBits & NewMask)) {
1166  // None of the shifted in bits are needed. Add a truncate of the
1167  // shift input, then shift it.
1168  SDValue NewTrunc = TLO.DAG.getNode(ISD::TRUNCATE, dl,
1169  Op.getValueType(),
1170  In.getOperand(0));
1171  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl,
1172  Op.getValueType(),
1173  NewTrunc,
1174  Shift));
1175  }
1176  break;
1177  }
1178  }
1179 
1180  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1181  break;
1182  }
1183  case ISD::AssertZext: {
1184  // AssertZext demands all of the high bits, plus any of the low bits
1185  // demanded by its users.
1186  EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1187  APInt InMask = APInt::getLowBitsSet(BitWidth,
1188  VT.getSizeInBits());
1189  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | NewMask,
1190  KnownZero, KnownOne, TLO, Depth+1))
1191  return true;
1192  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1193 
1194  KnownZero |= ~InMask & NewMask;
1195  break;
1196  }
1197  case ISD::BITCAST:
1198  // If this is an FP->Int bitcast and if the sign bit is the only
1199  // thing demanded, turn this into a FGETSIGN.
1200  if (!TLO.LegalOperations() &&
1201  !Op.getValueType().isVector() &&
1202  !Op.getOperand(0).getValueType().isVector() &&
1203  NewMask == APInt::getSignBit(Op.getValueSizeInBits()) &&
1205  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, Op.getValueType());
1207  if ((OpVTLegal || i32Legal) && Op.getValueType().isSimple() &&
1208  Op.getOperand(0).getValueType() != MVT::f128) {
1209  // Cannot eliminate/lower SHL for f128 yet.
1210  EVT Ty = OpVTLegal ? Op.getValueType() : MVT::i32;
1211  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1212  // place. We expect the SHL to be eliminated by other optimizations.
1213  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Op.getOperand(0));
1214  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1215  if (!OpVTLegal && OpVTSizeInBits > 32)
1216  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, Op.getValueType(), Sign);
1217  unsigned ShVal = Op.getValueSizeInBits() - 1;
1218  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, Op.getValueType());
1219  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl,
1220  Op.getValueType(),
1221  Sign, ShAmt));
1222  }
1223  }
1224  break;
1225  case ISD::ADD:
1226  case ISD::MUL:
1227  case ISD::SUB: {
1228  // Add, Sub, and Mul don't demand any bits in positions beyond that
1229  // of the highest bit demanded of them.
1230  APInt LoMask = APInt::getLowBitsSet(BitWidth,
1231  BitWidth - NewMask.countLeadingZeros());
1232  if (SimplifyDemandedBits(Op.getOperand(0), LoMask, KnownZero2,
1233  KnownOne2, TLO, Depth+1) ||
1234  SimplifyDemandedBits(Op.getOperand(1), LoMask, KnownZero2,
1235  KnownOne2, TLO, Depth+1) ||
1236  // See if the operation should be performed at a smaller bit width.
1237  TLO.ShrinkDemandedOp(Op, BitWidth, NewMask, dl)) {
1238  const SDNodeFlags *Flags = Op.getNode()->getFlags();
1239  if (Flags->hasNoSignedWrap() || Flags->hasNoUnsignedWrap()) {
1240  // Disable the nsw and nuw flags. We can no longer guarantee that we
1241  // won't wrap after simplification.
1242  SDNodeFlags NewFlags = *Flags;
1243  NewFlags.setNoSignedWrap(false);
1244  NewFlags.setNoUnsignedWrap(false);
1245  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, Op.getValueType(),
1246  Op.getOperand(0), Op.getOperand(1),
1247  &NewFlags);
1248  return TLO.CombineTo(Op, NewOp);
1249  }
1250  return true;
1251  }
1253  }
1254  default:
1255  // Just use computeKnownBits to compute output bits.
1256  TLO.DAG.computeKnownBits(Op, KnownZero, KnownOne, Depth);
1257  break;
1258  }
1259 
1260  // If we know the value of all of the demanded bits, return this as a
1261  // constant.
1262  if ((NewMask & (KnownZero|KnownOne)) == NewMask) {
1263  // Avoid folding to a constant if any OpaqueConstant is involved.
1264  const SDNode *N = Op.getNode();
1266  E = SDNodeIterator::end(N); I != E; ++I) {
1267  SDNode *Op = *I;
1268  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
1269  if (C->isOpaque())
1270  return false;
1271  }
1272  return TLO.CombineTo(Op,
1273  TLO.DAG.getConstant(KnownOne, dl, Op.getValueType()));
1274  }
1275 
1276  return false;
1277 }
1278 
1279 /// Determine which of the bits specified in Mask are known to be either zero or
1280 /// one and return them in the KnownZero/KnownOne bitsets.
1282  APInt &KnownZero,
1283  APInt &KnownOne,
1284  const SelectionDAG &DAG,
1285  unsigned Depth) const {
1289  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1290  "Should use MaskedValueIsZero if you don't know whether Op"
1291  " is a target node!");
1292  KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0);
1293 }
1294 
1295 /// This method can be implemented by targets that want to expose additional
1296 /// information about sign bits to the DAG Combiner.
1298  const SelectionDAG &,
1299  unsigned Depth) const {
1303  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1304  "Should use ComputeNumSignBits if you don't know whether Op"
1305  " is a target node!");
1306  return 1;
1307 }
1308 
1310  if (!N)
1311  return false;
1312 
1313  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
1314  if (!CN) {
1316  if (!BV)
1317  return false;
1318 
1319  // Only interested in constant splats, we don't care about undef
1320  // elements in identifying boolean constants and getConstantSplatNode
1321  // returns NULL if all ops are undef;
1322  CN = BV->getConstantSplatNode();
1323  if (!CN)
1324  return false;
1325  }
1326 
1327  switch (getBooleanContents(N->getValueType(0))) {
1329  return CN->getAPIntValue()[0];
1331  return CN->isOne();
1333  return CN->isAllOnesValue();
1334  }
1335 
1336  llvm_unreachable("Invalid boolean contents");
1337 }
1338 
1340  const SDLoc &DL) const {
1341  unsigned ElementWidth = VT.getScalarSizeInBits();
1342  APInt TrueInt =
1344  ? APInt(ElementWidth, 1)
1345  : APInt::getAllOnesValue(ElementWidth);
1346  return DAG.getConstant(TrueInt, DL, VT);
1347 }
1348 
1350  if (!N)
1351  return false;
1352 
1353  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
1354  if (!CN) {
1356  if (!BV)
1357  return false;
1358 
1359  // Only interested in constant splats, we don't care about undef
1360  // elements in identifying boolean constants and getConstantSplatNode
1361  // returns NULL if all ops are undef;
1362  CN = BV->getConstantSplatNode();
1363  if (!CN)
1364  return false;
1365  }
1366 
1368  return !CN->getAPIntValue()[0];
1369 
1370  return CN->isNullValue();
1371 }
1372 
1374  bool SExt) const {
1375  if (VT == MVT::i1)
1376  return N->isOne();
1377 
1379  switch (Cnt) {
1381  // An extended value of 1 is always true, unless its original type is i1,
1382  // in which case it will be sign extended to -1.
1383  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
1386  return N->isAllOnesValue() && SExt;
1387  }
1388  llvm_unreachable("Unexpected enumeration.");
1389 }
1390 
1391 /// This helper function of SimplifySetCC tries to optimize the comparison when
1392 /// either operand of the SetCC node is a bitwise-and instruction.
1393 SDValue TargetLowering::simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
1394  ISD::CondCode Cond,
1395  DAGCombinerInfo &DCI,
1396  const SDLoc &DL) const {
1397  // Match these patterns in any of their permutations:
1398  // (X & Y) == Y
1399  // (X & Y) != Y
1400  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
1401  std::swap(N0, N1);
1402 
1403  EVT OpVT = N0.getValueType();
1404  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
1405  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
1406  return SDValue();
1407 
1408  SDValue X, Y;
1409  if (N0.getOperand(0) == N1) {
1410  X = N0.getOperand(1);
1411  Y = N0.getOperand(0);
1412  } else if (N0.getOperand(1) == N1) {
1413  X = N0.getOperand(0);
1414  Y = N0.getOperand(1);
1415  } else {
1416  return SDValue();
1417  }
1418 
1419  SelectionDAG &DAG = DCI.DAG;
1420  SDValue Zero = DAG.getConstant(0, DL, OpVT);
1421  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
1422  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
1423  // Note that where Y is variable and is known to have at most one bit set
1424  // (for example, if it is Z & 1) we cannot do this; the expressions are not
1425  // equivalent when Y == 0.
1426  Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
1427  if (DCI.isBeforeLegalizeOps() ||
1428  isCondCodeLegal(Cond, N0.getSimpleValueType()))
1429  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
1430  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
1431  // If the target supports an 'and-not' or 'and-complement' logic operation,
1432  // try to use that to make a comparison operation more efficient.
1433  // But don't do this transform if the mask is a single bit because there are
1434  // more efficient ways to deal with that case (for example, 'bt' on x86 or
1435  // 'rlwinm' on PPC).
1436 
1437  // Bail out if the compare operand that we want to turn into a zero is
1438  // already a zero (otherwise, infinite loop).
1439  auto *YConst = dyn_cast<ConstantSDNode>(Y);
1440  if (YConst && YConst->isNullValue())
1441  return SDValue();
1442 
1443  // Transform this into: ~X & Y == 0.
1444  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
1445  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
1446  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
1447  }
1448 
1449  return SDValue();
1450 }
1451 
1452 /// Try to simplify a setcc built with the specified operands and cc. If it is
1453 /// unable to simplify it, return a null SDValue.
1455  ISD::CondCode Cond, bool foldBooleans,
1456  DAGCombinerInfo &DCI,
1457  const SDLoc &dl) const {
1458  SelectionDAG &DAG = DCI.DAG;
1459 
1460  // These setcc operations always fold.
1461  switch (Cond) {
1462  default: break;
1463  case ISD::SETFALSE:
1464  case ISD::SETFALSE2: return DAG.getConstant(0, dl, VT);
1465  case ISD::SETTRUE:
1466  case ISD::SETTRUE2: {
1469  return DAG.getConstant(
1470  Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
1471  VT);
1472  }
1473  }
1474 
1475  // Ensure that the constant occurs on the RHS, and fold constant
1476  // comparisons.
1477  ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
1478  if (isa<ConstantSDNode>(N0.getNode()) &&
1479  (DCI.isBeforeLegalizeOps() ||
1480  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
1481  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
1482 
1483  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
1484  const APInt &C1 = N1C->getAPIntValue();
1485 
1486  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
1487  // equality comparison, then we're just comparing whether X itself is
1488  // zero.
1489  if (N0.getOpcode() == ISD::SRL && (C1 == 0 || C1 == 1) &&
1490  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
1491  N0.getOperand(1).getOpcode() == ISD::Constant) {
1492  const APInt &ShAmt
1493  = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
1494  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
1495  ShAmt == Log2_32(N0.getValueSizeInBits())) {
1496  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
1497  // (srl (ctlz x), 5) == 0 -> X != 0
1498  // (srl (ctlz x), 5) != 1 -> X != 0
1499  Cond = ISD::SETNE;
1500  } else {
1501  // (srl (ctlz x), 5) != 0 -> X == 0
1502  // (srl (ctlz x), 5) == 1 -> X == 0
1503  Cond = ISD::SETEQ;
1504  }
1505  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
1506  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
1507  Zero, Cond);
1508  }
1509  }
1510 
1511  SDValue CTPOP = N0;
1512  // Look through truncs that don't change the value of a ctpop.
1513  if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
1514  CTPOP = N0.getOperand(0);
1515 
1516  if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
1517  (N0 == CTPOP ||
1519  EVT CTVT = CTPOP.getValueType();
1520  SDValue CTOp = CTPOP.getOperand(0);
1521 
1522  // (ctpop x) u< 2 -> (x & x-1) == 0
1523  // (ctpop x) u> 1 -> (x & x-1) != 0
1524  if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
1525  SDValue Sub = DAG.getNode(ISD::SUB, dl, CTVT, CTOp,
1526  DAG.getConstant(1, dl, CTVT));
1527  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Sub);
1529  return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
1530  }
1531 
1532  // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal.
1533  }
1534 
1535  // (zext x) == C --> x == (trunc C)
1536  // (sext x) == C --> x == (trunc C)
1537  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
1538  DCI.isBeforeLegalize() && N0->hasOneUse()) {
1539  unsigned MinBits = N0.getValueSizeInBits();
1540  SDValue PreExt;
1541  bool Signed = false;
1542  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
1543  // ZExt
1544  MinBits = N0->getOperand(0).getValueSizeInBits();
1545  PreExt = N0->getOperand(0);
1546  } else if (N0->getOpcode() == ISD::AND) {
1547  // DAGCombine turns costly ZExts into ANDs
1548  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
1549  if ((C->getAPIntValue()+1).isPowerOf2()) {
1550  MinBits = C->getAPIntValue().countTrailingOnes();
1551  PreExt = N0->getOperand(0);
1552  }
1553  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
1554  // SExt
1555  MinBits = N0->getOperand(0).getValueSizeInBits();
1556  PreExt = N0->getOperand(0);
1557  Signed = true;
1558  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
1559  // ZEXTLOAD / SEXTLOAD
1560  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
1561  MinBits = LN0->getMemoryVT().getSizeInBits();
1562  PreExt = N0;
1563  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
1564  Signed = true;
1565  MinBits = LN0->getMemoryVT().getSizeInBits();
1566  PreExt = N0;
1567  }
1568  }
1569 
1570  // Figure out how many bits we need to preserve this constant.
1571  unsigned ReqdBits = Signed ?
1572  C1.getBitWidth() - C1.getNumSignBits() + 1 :
1573  C1.getActiveBits();
1574 
1575  // Make sure we're not losing bits from the constant.
1576  if (MinBits > 0 &&
1577  MinBits < C1.getBitWidth() &&
1578  MinBits >= ReqdBits) {
1579  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
1580  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
1581  // Will get folded away.
1582  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
1583  if (MinBits == 1 && C1 == 1)
1584  // Invert the condition.
1585  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
1586  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
1587  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
1588  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
1589  }
1590 
1591  // If truncating the setcc operands is not desirable, we can still
1592  // simplify the expression in some cases:
1593  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
1594  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
1595  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
1596  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
1597  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
1598  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
1599  SDValue TopSetCC = N0->getOperand(0);
1600  unsigned N0Opc = N0->getOpcode();
1601  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
1602  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
1603  TopSetCC.getOpcode() == ISD::SETCC &&
1604  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
1605  (isConstFalseVal(N1C) ||
1606  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
1607 
1608  bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
1609  (!N1C->isNullValue() && Cond == ISD::SETNE);
1610 
1611  if (!Inverse)
1612  return TopSetCC;
1613 
1615  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
1616  TopSetCC.getOperand(0).getValueType().isInteger());
1617  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
1618  TopSetCC.getOperand(1),
1619  InvCond);
1620 
1621  }
1622  }
1623  }
1624 
1625  // If the LHS is '(and load, const)', the RHS is 0,
1626  // the test is for equality or unsigned, and all 1 bits of the const are
1627  // in the same partial word, see if we can shorten the load.
1628  if (DCI.isBeforeLegalize() &&
1629  !ISD::isSignedIntSetCC(Cond) &&
1630  N0.getOpcode() == ISD::AND && C1 == 0 &&
1631  N0.getNode()->hasOneUse() &&
1632  isa<LoadSDNode>(N0.getOperand(0)) &&
1633  N0.getOperand(0).getNode()->hasOneUse() &&
1634  isa<ConstantSDNode>(N0.getOperand(1))) {
1635  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
1636  APInt bestMask;
1637  unsigned bestWidth = 0, bestOffset = 0;
1638  if (!Lod->isVolatile() && Lod->isUnindexed()) {
1639  unsigned origWidth = N0.getValueSizeInBits();
1640  unsigned maskWidth = origWidth;
1641  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
1642  // 8 bits, but have to be careful...
1643  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
1644  origWidth = Lod->getMemoryVT().getSizeInBits();
1645  const APInt &Mask =
1646  cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
1647  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
1648  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
1649  for (unsigned offset=0; offset<origWidth/width; offset++) {
1650  if ((newMask & Mask) == Mask) {
1651  if (!DAG.getDataLayout().isLittleEndian())
1652  bestOffset = (origWidth/width - offset - 1) * (width/8);
1653  else
1654  bestOffset = (uint64_t)offset * (width/8);
1655  bestMask = Mask.lshr(offset * (width/8) * 8);
1656  bestWidth = width;
1657  break;
1658  }
1659  newMask = newMask << width;
1660  }
1661  }
1662  }
1663  if (bestWidth) {
1664  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
1665  if (newVT.isRound()) {
1666  EVT PtrType = Lod->getOperand(1).getValueType();
1667  SDValue Ptr = Lod->getBasePtr();
1668  if (bestOffset != 0)
1669  Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
1670  DAG.getConstant(bestOffset, dl, PtrType));
1671  unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
1672  SDValue NewLoad = DAG.getLoad(
1673  newVT, dl, Lod->getChain(), Ptr,
1674  Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
1675  return DAG.getSetCC(dl, VT,
1676  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
1677  DAG.getConstant(bestMask.trunc(bestWidth),
1678  dl, newVT)),
1679  DAG.getConstant(0LL, dl, newVT), Cond);
1680  }
1681  }
1682  }
1683 
1684  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
1685  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
1686  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
1687 
1688  // If the comparison constant has bits in the upper part, the
1689  // zero-extended value could never match.
1690  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
1691  C1.getBitWidth() - InSize))) {
1692  switch (Cond) {
1693  case ISD::SETUGT:
1694  case ISD::SETUGE:
1695  case ISD::SETEQ: return DAG.getConstant(0, dl, VT);
1696  case ISD::SETULT:
1697  case ISD::SETULE:
1698  case ISD::SETNE: return DAG.getConstant(1, dl, VT);
1699  case ISD::SETGT:
1700  case ISD::SETGE:
1701  // True if the sign bit of C1 is set.
1702  return DAG.getConstant(C1.isNegative(), dl, VT);
1703  case ISD::SETLT:
1704  case ISD::SETLE:
1705  // True if the sign bit of C1 isn't set.
1706  return DAG.getConstant(C1.isNonNegative(), dl, VT);
1707  default:
1708  break;
1709  }
1710  }
1711 
1712  // Otherwise, we can perform the comparison with the low bits.
1713  switch (Cond) {
1714  case ISD::SETEQ:
1715  case ISD::SETNE:
1716  case ISD::SETUGT:
1717  case ISD::SETUGE:
1718  case ISD::SETULT:
1719  case ISD::SETULE: {
1720  EVT newVT = N0.getOperand(0).getValueType();
1721  if (DCI.isBeforeLegalizeOps() ||
1722  (isOperationLegal(ISD::SETCC, newVT) &&
1723  getCondCodeAction(Cond, newVT.getSimpleVT()) == Legal)) {
1724  EVT NewSetCCVT =
1725  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), newVT);
1726  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
1727 
1728  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
1729  NewConst, Cond);
1730  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
1731  }
1732  break;
1733  }
1734  default:
1735  break; // todo, be more careful with signed comparisons
1736  }
1737  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1738  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
1739  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
1740  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
1741  EVT ExtDstTy = N0.getValueType();
1742  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
1743 
1744  // If the constant doesn't fit into the number of bits for the source of
1745  // the sign extension, it is impossible for both sides to be equal.
1746  if (C1.getMinSignedBits() > ExtSrcTyBits)
1747  return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
1748 
1749  SDValue ZextOp;
1750  EVT Op0Ty = N0.getOperand(0).getValueType();
1751  if (Op0Ty == ExtSrcTy) {
1752  ZextOp = N0.getOperand(0);
1753  } else {
1754  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
1755  ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
1756  DAG.getConstant(Imm, dl, Op0Ty));
1757  }
1758  if (!DCI.isCalledByLegalizer())
1759  DCI.AddToWorklist(ZextOp.getNode());
1760  // Otherwise, make this a use of a zext.
1761  return DAG.getSetCC(dl, VT, ZextOp,
1763  ExtDstTyBits,
1764  ExtSrcTyBits),
1765  dl, ExtDstTy),
1766  Cond);
1767  } else if ((N1C->isNullValue() || N1C->getAPIntValue() == 1) &&
1768  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
1769  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
1770  if (N0.getOpcode() == ISD::SETCC &&
1771  isTypeLegal(VT) && VT.bitsLE(N0.getValueType())) {
1772  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (N1C->getAPIntValue() != 1);
1773  if (TrueWhenTrue)
1774  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
1775  // Invert the condition.
1776  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
1777  CC = ISD::getSetCCInverse(CC,
1778  N0.getOperand(0).getValueType().isInteger());
1779  if (DCI.isBeforeLegalizeOps() ||
1780  isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
1781  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
1782  }
1783 
1784  if ((N0.getOpcode() == ISD::XOR ||
1785  (N0.getOpcode() == ISD::AND &&
1786  N0.getOperand(0).getOpcode() == ISD::XOR &&
1787  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
1788  isa<ConstantSDNode>(N0.getOperand(1)) &&
1789  cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue() == 1) {
1790  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
1791  // can only do this if the top bits are known zero.
1792  unsigned BitWidth = N0.getValueSizeInBits();
1793  if (DAG.MaskedValueIsZero(N0,
1794  APInt::getHighBitsSet(BitWidth,
1795  BitWidth-1))) {
1796  // Okay, get the un-inverted input value.
1797  SDValue Val;
1798  if (N0.getOpcode() == ISD::XOR)
1799  Val = N0.getOperand(0);
1800  else {
1801  assert(N0.getOpcode() == ISD::AND &&
1802  N0.getOperand(0).getOpcode() == ISD::XOR);
1803  // ((X^1)&1)^1 -> X & 1
1804  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
1805  N0.getOperand(0).getOperand(0),
1806  N0.getOperand(1));
1807  }
1808 
1809  return DAG.getSetCC(dl, VT, Val, N1,
1810  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
1811  }
1812  } else if (N1C->getAPIntValue() == 1 &&
1813  (VT == MVT::i1 ||
1814  getBooleanContents(N0->getValueType(0)) ==
1816  SDValue Op0 = N0;
1817  if (Op0.getOpcode() == ISD::TRUNCATE)
1818  Op0 = Op0.getOperand(0);
1819 
1820  if ((Op0.getOpcode() == ISD::XOR) &&
1821  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
1822  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
1823  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
1824  Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
1825  return DAG.getSetCC(dl, VT, Op0.getOperand(0), Op0.getOperand(1),
1826  Cond);
1827  }
1828  if (Op0.getOpcode() == ISD::AND &&
1829  isa<ConstantSDNode>(Op0.getOperand(1)) &&
1830  cast<ConstantSDNode>(Op0.getOperand(1))->getAPIntValue() == 1) {
1831  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
1832  if (Op0.getValueType().bitsGT(VT))
1833  Op0 = DAG.getNode(ISD::AND, dl, VT,
1834  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
1835  DAG.getConstant(1, dl, VT));
1836  else if (Op0.getValueType().bitsLT(VT))
1837  Op0 = DAG.getNode(ISD::AND, dl, VT,
1838  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
1839  DAG.getConstant(1, dl, VT));
1840 
1841  return DAG.getSetCC(dl, VT, Op0,
1842  DAG.getConstant(0, dl, Op0.getValueType()),
1843  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
1844  }
1845  if (Op0.getOpcode() == ISD::AssertZext &&
1846  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
1847  return DAG.getSetCC(dl, VT, Op0,
1848  DAG.getConstant(0, dl, Op0.getValueType()),
1849  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
1850  }
1851  }
1852 
1853  APInt MinVal, MaxVal;
1854  unsigned OperandBitSize = N1C->getValueType(0).getSizeInBits();
1855  if (ISD::isSignedIntSetCC(Cond)) {
1856  MinVal = APInt::getSignedMinValue(OperandBitSize);
1857  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
1858  } else {
1859  MinVal = APInt::getMinValue(OperandBitSize);
1860  MaxVal = APInt::getMaxValue(OperandBitSize);
1861  }
1862 
1863  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
1864  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
1865  if (C1 == MinVal) return DAG.getConstant(1, dl, VT); // X >= MIN --> true
1866  // X >= C0 --> X > (C0 - 1)
1867  APInt C = C1 - 1;
1868  ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
1869  if ((DCI.isBeforeLegalizeOps() ||
1870  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
1871  (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
1873  return DAG.getSetCC(dl, VT, N0,
1874  DAG.getConstant(C, dl, N1.getValueType()),
1875  NewCC);
1876  }
1877  }
1878 
1879  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
1880  if (C1 == MaxVal) return DAG.getConstant(1, dl, VT); // X <= MAX --> true
1881  // X <= C0 --> X < (C0 + 1)
1882  APInt C = C1 + 1;
1883  ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
1884  if ((DCI.isBeforeLegalizeOps() ||
1885  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
1886  (!N1C->isOpaque() || (N1C->isOpaque() && C.getBitWidth() <= 64 &&
1888  return DAG.getSetCC(dl, VT, N0,
1889  DAG.getConstant(C, dl, N1.getValueType()),
1890  NewCC);
1891  }
1892  }
1893 
1894  if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal)
1895  return DAG.getConstant(0, dl, VT); // X < MIN --> false
1896  if ((Cond == ISD::SETGE || Cond == ISD::SETUGE) && C1 == MinVal)
1897  return DAG.getConstant(1, dl, VT); // X >= MIN --> true
1898  if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal)
1899  return DAG.getConstant(0, dl, VT); // X > MAX --> false
1900  if ((Cond == ISD::SETLE || Cond == ISD::SETULE) && C1 == MaxVal)
1901  return DAG.getConstant(1, dl, VT); // X <= MAX --> true
1902 
1903  // Canonicalize setgt X, Min --> setne X, Min
1904  if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MinVal)
1905  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
1906  // Canonicalize setlt X, Max --> setne X, Max
1907  if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MaxVal)
1908  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
1909 
1910  // If we have setult X, 1, turn it into seteq X, 0
1911  if ((Cond == ISD::SETLT || Cond == ISD::SETULT) && C1 == MinVal+1)
1912  return DAG.getSetCC(dl, VT, N0,
1913  DAG.getConstant(MinVal, dl, N0.getValueType()),
1914  ISD::SETEQ);
1915  // If we have setugt X, Max-1, turn it into seteq X, Max
1916  if ((Cond == ISD::SETGT || Cond == ISD::SETUGT) && C1 == MaxVal-1)
1917  return DAG.getSetCC(dl, VT, N0,
1918  DAG.getConstant(MaxVal, dl, N0.getValueType()),
1919  ISD::SETEQ);
1920 
1921  // If we have "setcc X, C0", check to see if we can shrink the immediate
1922  // by changing cc.
1923 
1924  // SETUGT X, SINTMAX -> SETLT X, 0
1925  if (Cond == ISD::SETUGT &&
1926  C1 == APInt::getSignedMaxValue(OperandBitSize))
1927  return DAG.getSetCC(dl, VT, N0,
1928  DAG.getConstant(0, dl, N1.getValueType()),
1929  ISD::SETLT);
1930 
1931  // SETULT X, SINTMIN -> SETGT X, -1
1932  if (Cond == ISD::SETULT &&
1933  C1 == APInt::getSignedMinValue(OperandBitSize)) {
1934  SDValue ConstMinusOne =
1935  DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
1936  N1.getValueType());
1937  return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
1938  }
1939 
1940  // Fold bit comparisons when we can.
1941  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
1942  (VT == N0.getValueType() ||
1943  (isTypeLegal(VT) && VT.bitsLE(N0.getValueType()))) &&
1944  N0.getOpcode() == ISD::AND) {
1945  auto &DL = DAG.getDataLayout();
1946  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
1947  EVT ShiftTy = DCI.isBeforeLegalize()
1948  ? getPointerTy(DL)
1949  : getShiftAmountTy(N0.getValueType(), DL);
1950  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
1951  // Perform the xform if the AND RHS is a single bit.
1952  if (AndRHS->getAPIntValue().isPowerOf2()) {
1953  return DAG.getNode(ISD::TRUNCATE, dl, VT,
1954  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
1955  DAG.getConstant(AndRHS->getAPIntValue().logBase2(), dl,
1956  ShiftTy)));
1957  }
1958  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
1959  // (X & 8) == 8 --> (X & 8) >> 3
1960  // Perform the xform if C1 is a single bit.
1961  if (C1.isPowerOf2()) {
1962  return DAG.getNode(ISD::TRUNCATE, dl, VT,
1963  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
1964  DAG.getConstant(C1.logBase2(), dl,
1965  ShiftTy)));
1966  }
1967  }
1968  }
1969  }
1970 
1971  if (C1.getMinSignedBits() <= 64 &&
1972  !isLegalICmpImmediate(C1.getSExtValue())) {
1973  // (X & -256) == 256 -> (X >> 8) == 1
1974  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
1975  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
1976  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
1977  const APInt &AndRHSC = AndRHS->getAPIntValue();
1978  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
1979  unsigned ShiftBits = AndRHSC.countTrailingZeros();
1980  auto &DL = DAG.getDataLayout();
1981  EVT ShiftTy = DCI.isBeforeLegalize()
1982  ? getPointerTy(DL)
1983  : getShiftAmountTy(N0.getValueType(), DL);
1984  EVT CmpTy = N0.getValueType();
1985  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
1986  DAG.getConstant(ShiftBits, dl,
1987  ShiftTy));
1988  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
1989  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
1990  }
1991  }
1992  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
1993  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
1994  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
1995  // X < 0x100000000 -> (X >> 32) < 1
1996  // X >= 0x100000000 -> (X >> 32) >= 1
1997  // X <= 0x0ffffffff -> (X >> 32) < 1
1998  // X > 0x0ffffffff -> (X >> 32) >= 1
1999  unsigned ShiftBits;
2000  APInt NewC = C1;
2001  ISD::CondCode NewCond = Cond;
2002  if (AdjOne) {
2003  ShiftBits = C1.countTrailingOnes();
2004  NewC = NewC + 1;
2005  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2006  } else {
2007  ShiftBits = C1.countTrailingZeros();
2008  }
2009  NewC = NewC.lshr(ShiftBits);
2010  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
2012  auto &DL = DAG.getDataLayout();
2013  EVT ShiftTy = DCI.isBeforeLegalize()
2014  ? getPointerTy(DL)
2015  : getShiftAmountTy(N0.getValueType(), DL);
2016  EVT CmpTy = N0.getValueType();
2017  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
2018  DAG.getConstant(ShiftBits, dl, ShiftTy));
2019  SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
2020  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
2021  }
2022  }
2023  }
2024  }
2025 
2026  if (isa<ConstantFPSDNode>(N0.getNode())) {
2027  // Constant fold or commute setcc.
2028  SDValue O = DAG.FoldSetCC(VT, N0, N1, Cond, dl);
2029  if (O.getNode()) return O;
2030  } else if (auto *CFP = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
2031  // If the RHS of an FP comparison is a constant, simplify it away in
2032  // some cases.
2033  if (CFP->getValueAPF().isNaN()) {
2034  // If an operand is known to be a nan, we can fold it.
2035  switch (ISD::getUnorderedFlavor(Cond)) {
2036  default: llvm_unreachable("Unknown flavor!");
2037  case 0: // Known false.
2038  return DAG.getConstant(0, dl, VT);
2039  case 1: // Known true.
2040  return DAG.getConstant(1, dl, VT);
2041  case 2: // Undefined.
2042  return DAG.getUNDEF(VT);
2043  }
2044  }
2045 
2046  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
2047  // constant if knowing that the operand is non-nan is enough. We prefer to
2048  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
2049  // materialize 0.0.
2050  if (Cond == ISD::SETO || Cond == ISD::SETUO)
2051  return DAG.getSetCC(dl, VT, N0, N0, Cond);
2052 
2053  // If the condition is not legal, see if we can find an equivalent one
2054  // which is legal.
2055  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
2056  // If the comparison was an awkward floating-point == or != and one of
2057  // the comparison operands is infinity or negative infinity, convert the
2058  // condition to a less-awkward <= or >=.
2059  if (CFP->getValueAPF().isInfinity()) {
2060  if (CFP->getValueAPF().isNegative()) {
2061  if (Cond == ISD::SETOEQ &&
2063  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
2064  if (Cond == ISD::SETUEQ &&
2066  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
2067  if (Cond == ISD::SETUNE &&
2069  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
2070  if (Cond == ISD::SETONE &&
2072  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
2073  } else {
2074  if (Cond == ISD::SETOEQ &&
2076  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
2077  if (Cond == ISD::SETUEQ &&
2079  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
2080  if (Cond == ISD::SETUNE &&
2082  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
2083  if (Cond == ISD::SETONE &&
2085  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
2086  }
2087  }
2088  }
2089  }
2090 
2091  if (N0 == N1) {
2092  // The sext(setcc()) => setcc() optimization relies on the appropriate
2093  // constant being emitted.
2094  uint64_t EqVal = 0;
2095  switch (getBooleanContents(N0.getValueType())) {
2098  EqVal = ISD::isTrueWhenEqual(Cond);
2099  break;
2101  EqVal = ISD::isTrueWhenEqual(Cond) ? -1 : 0;
2102  break;
2103  }
2104 
2105  // We can always fold X == X for integer setcc's.
2106  if (N0.getValueType().isInteger()) {
2107  return DAG.getConstant(EqVal, dl, VT);
2108  }
2109  unsigned UOF = ISD::getUnorderedFlavor(Cond);
2110  if (UOF == 2) // FP operators that are undefined on NaNs.
2111  return DAG.getConstant(EqVal, dl, VT);
2112  if (UOF == unsigned(ISD::isTrueWhenEqual(Cond)))
2113  return DAG.getConstant(EqVal, dl, VT);
2114  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
2115  // if it is not already.
2116  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
2117  if (NewCond != Cond && (DCI.isBeforeLegalizeOps() ||
2118  getCondCodeAction(NewCond, N0.getSimpleValueType()) == Legal))
2119  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
2120  }
2121 
2122  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2123  N0.getValueType().isInteger()) {
2124  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
2125  N0.getOpcode() == ISD::XOR) {
2126  // Simplify (X+Y) == (X+Z) --> Y == Z
2127  if (N0.getOpcode() == N1.getOpcode()) {
2128  if (N0.getOperand(0) == N1.getOperand(0))
2129  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
2130  if (N0.getOperand(1) == N1.getOperand(1))
2131  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
2132  if (DAG.isCommutativeBinOp(N0.getOpcode())) {
2133  // If X op Y == Y op X, try other combinations.
2134  if (N0.getOperand(0) == N1.getOperand(1))
2135  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
2136  Cond);
2137  if (N0.getOperand(1) == N1.getOperand(0))
2138  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
2139  Cond);
2140  }
2141  }
2142 
2143  // If RHS is a legal immediate value for a compare instruction, we need
2144  // to be careful about increasing register pressure needlessly.
2145  bool LegalRHSImm = false;
2146 
2147  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
2148  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2149  // Turn (X+C1) == C2 --> X == C2-C1
2150  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
2151  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2152  DAG.getConstant(RHSC->getAPIntValue()-
2153  LHSR->getAPIntValue(),
2154  dl, N0.getValueType()), Cond);
2155  }
2156 
2157  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
2158  if (N0.getOpcode() == ISD::XOR)
2159  // If we know that all of the inverted bits are zero, don't bother
2160  // performing the inversion.
2161  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
2162  return
2163  DAG.getSetCC(dl, VT, N0.getOperand(0),
2164  DAG.getConstant(LHSR->getAPIntValue() ^
2165  RHSC->getAPIntValue(),
2166  dl, N0.getValueType()),
2167  Cond);
2168  }
2169 
2170  // Turn (C1-X) == C2 --> X == C1-C2
2171  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
2172  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
2173  return
2174  DAG.getSetCC(dl, VT, N0.getOperand(1),
2175  DAG.getConstant(SUBC->getAPIntValue() -
2176  RHSC->getAPIntValue(),
2177  dl, N0.getValueType()),
2178  Cond);
2179  }
2180  }
2181 
2182  // Could RHSC fold directly into a compare?
2183  if (RHSC->getValueType(0).getSizeInBits() <= 64)
2184  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
2185  }
2186 
2187  // Simplify (X+Z) == X --> Z == 0
2188  // Don't do this if X is an immediate that can fold into a cmp
2189  // instruction and X+Z has other uses. It could be an induction variable
2190  // chain, and the transform would increase register pressure.
2191  if (!LegalRHSImm || N0.getNode()->hasOneUse()) {
2192  if (N0.getOperand(0) == N1)
2193  return DAG.getSetCC(dl, VT, N0.getOperand(1),
2194  DAG.getConstant(0, dl, N0.getValueType()), Cond);
2195  if (N0.getOperand(1) == N1) {
2196  if (DAG.isCommutativeBinOp(N0.getOpcode()))
2197  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2198  DAG.getConstant(0, dl, N0.getValueType()),
2199  Cond);
2200  if (N0.getNode()->hasOneUse()) {
2201  assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
2202  auto &DL = DAG.getDataLayout();
2203  // (Z-X) == X --> Z == X<<1
2204  SDValue SH = DAG.getNode(
2205  ISD::SHL, dl, N1.getValueType(), N1,
2206  DAG.getConstant(1, dl,
2207  getShiftAmountTy(N1.getValueType(), DL)));
2208  if (!DCI.isCalledByLegalizer())
2209  DCI.AddToWorklist(SH.getNode());
2210  return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond);
2211  }
2212  }
2213  }
2214  }
2215 
2216  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
2217  N1.getOpcode() == ISD::XOR) {
2218  // Simplify X == (X+Z) --> Z == 0
2219  if (N1.getOperand(0) == N0)
2220  return DAG.getSetCC(dl, VT, N1.getOperand(1),
2221  DAG.getConstant(0, dl, N1.getValueType()), Cond);
2222  if (N1.getOperand(1) == N0) {
2223  if (DAG.isCommutativeBinOp(N1.getOpcode()))
2224  return DAG.getSetCC(dl, VT, N1.getOperand(0),
2225  DAG.getConstant(0, dl, N1.getValueType()), Cond);
2226  if (N1.getNode()->hasOneUse()) {
2227  assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
2228  auto &DL = DAG.getDataLayout();
2229  // X == (Z-X) --> X<<1 == Z
2230  SDValue SH = DAG.getNode(
2231  ISD::SHL, dl, N1.getValueType(), N0,
2232  DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL)));
2233  if (!DCI.isCalledByLegalizer())
2234  DCI.AddToWorklist(SH.getNode());
2235  return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond);
2236  }
2237  }
2238  }
2239 
2240  if (SDValue V = simplifySetCCWithAnd(VT, N0, N1, Cond, DCI, dl))
2241  return V;
2242  }
2243 
2244  // Fold away ALL boolean setcc's.
2245  SDValue Temp;
2246  if (N0.getValueType() == MVT::i1 && foldBooleans) {
2247  switch (Cond) {
2248  default: llvm_unreachable("Unknown integer setcc!");
2249  case ISD::SETEQ: // X == Y -> ~(X^Y)
2250  Temp = DAG.getNode(ISD::XOR, dl, MVT::i1, N0, N1);
2251  N0 = DAG.getNOT(dl, Temp, MVT::i1);
2252  if (!DCI.isCalledByLegalizer())
2253  DCI.AddToWorklist(Temp.getNode());
2254  break;
2255  case ISD::SETNE: // X != Y --> (X^Y)
2256  N0 = DAG.getNode(ISD::XOR, dl, MVT::i1, N0, N1);
2257  break;
2258  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
2259  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
2260  Temp = DAG.getNOT(dl, N0, MVT::i1);
2261  N0 = DAG.getNode(ISD::AND, dl, MVT::i1, N1, Temp);
2262  if (!DCI.isCalledByLegalizer())
2263  DCI.AddToWorklist(Temp.getNode());
2264  break;
2265  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
2266  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
2267  Temp = DAG.getNOT(dl, N1, MVT::i1);
2268  N0 = DAG.getNode(ISD::AND, dl, MVT::i1, N0, Temp);
2269  if (!DCI.isCalledByLegalizer())
2270  DCI.AddToWorklist(Temp.getNode());
2271  break;
2272  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
2273  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
2274  Temp = DAG.getNOT(dl, N0, MVT::i1);
2275  N0 = DAG.getNode(ISD::OR, dl, MVT::i1, N1, Temp);
2276  if (!DCI.isCalledByLegalizer())
2277  DCI.AddToWorklist(Temp.getNode());
2278  break;
2279  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
2280  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
2281  Temp = DAG.getNOT(dl, N1, MVT::i1);
2282  N0 = DAG.getNode(ISD::OR, dl, MVT::i1, N0, Temp);
2283  break;
2284  }
2285  if (VT != MVT::i1) {
2286  if (!DCI.isCalledByLegalizer())
2287  DCI.AddToWorklist(N0.getNode());
2288  // FIXME: If running after legalize, we probably can't do this.
2289  N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, N0);
2290  }
2291  return N0;
2292  }
2293 
2294  // Could not fold it.
2295  return SDValue();
2296 }
2297 
2298 /// Returns true (and the GlobalValue and the offset) if the node is a
2299 /// GlobalAddress + offset.
2301  int64_t &Offset) const {
2302  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
2303  GA = GASD->getGlobal();
2304  Offset += GASD->getOffset();
2305  return true;
2306  }
2307 
2308  if (N->getOpcode() == ISD::ADD) {
2309  SDValue N1 = N->getOperand(0);
2310  SDValue N2 = N->getOperand(1);
2311  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
2312  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
2313  Offset += V->getSExtValue();
2314  return true;
2315  }
2316  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
2317  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
2318  Offset += V->getSExtValue();
2319  return true;
2320  }
2321  }
2322  }
2323 
2324  return false;
2325 }
2326 
2328  DAGCombinerInfo &DCI) const {
2329  // Default implementation: no optimization.
2330  return SDValue();
2331 }
2332 
2333 //===----------------------------------------------------------------------===//
2334 // Inline Assembler Implementation Methods
2335 //===----------------------------------------------------------------------===//
2336 
2339  unsigned S = Constraint.size();
2340 
2341  if (S == 1) {
2342  switch (Constraint[0]) {
2343  default: break;
2344  case 'r': return C_RegisterClass;
2345  case 'm': // memory
2346  case 'o': // offsetable
2347  case 'V': // not offsetable
2348  return C_Memory;
2349  case 'i': // Simple Integer or Relocatable Constant
2350  case 'n': // Simple Integer
2351  case 'E': // Floating Point Constant
2352  case 'F': // Floating Point Constant
2353  case 's': // Relocatable Constant
2354  case 'p': // Address.
2355  case 'X': // Allow ANY value.
2356  case 'I': // Target registers.
2357  case 'J':
2358  case 'K':
2359  case 'L':
2360  case 'M':
2361  case 'N':
2362  case 'O':
2363  case 'P':
2364  case '<':
2365  case '>':
2366  return C_Other;
2367  }
2368  }
2369 
2370  if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') {
2371  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
2372  return C_Memory;
2373  return C_Register;
2374  }
2375  return C_Unknown;
2376 }
2377 
2378 /// Try to replace an X constraint, which matches anything, with another that
2379 /// has more specific requirements based on the type of the corresponding
2380 /// operand.
2381 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const{
2382  if (ConstraintVT.isInteger())
2383  return "r";
2384  if (ConstraintVT.isFloatingPoint())
2385  return "f"; // works for many targets
2386  return nullptr;
2387 }
2388 
2389 /// Lower the specified operand into the Ops vector.
2390 /// If it is invalid, don't add anything to Ops.
2392  std::string &Constraint,
2393  std::vector<SDValue> &Ops,
2394  SelectionDAG &DAG) const {
2395 
2396  if (Constraint.length() > 1) return;
2397 
2398  char ConstraintLetter = Constraint[0];
2399  switch (ConstraintLetter) {
2400  default: break;
2401  case 'X': // Allows any operand; labels (basic block) use this.
2402  if (Op.getOpcode() == ISD::BasicBlock) {
2403  Ops.push_back(Op);
2404  return;
2405  }
2407  case 'i': // Simple Integer or Relocatable Constant
2408  case 'n': // Simple Integer
2409  case 's': { // Relocatable Constant
2410  // These operands are interested in values of the form (GV+C), where C may
2411  // be folded in as an offset of GV, or it may be explicitly added. Also, it
2412  // is possible and fine if either GV or C are missing.
2415 
2416  // If we have "(add GV, C)", pull out GV/C
2417  if (Op.getOpcode() == ISD::ADD) {
2418  C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2420  if (!C || !GA) {
2421  C = dyn_cast<ConstantSDNode>(Op.getOperand(0));
2423  }
2424  if (!C || !GA) {
2425  C = nullptr;
2426  GA = nullptr;
2427  }
2428  }
2429 
2430  // If we find a valid operand, map to the TargetXXX version so that the
2431  // value itself doesn't get selected.
2432  if (GA) { // Either &GV or &GV+C
2433  if (ConstraintLetter != 'n') {
2434  int64_t Offs = GA->getOffset();
2435  if (C) Offs += C->getZExtValue();
2436  Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(),
2437  C ? SDLoc(C) : SDLoc(),
2438  Op.getValueType(), Offs));
2439  }
2440  return;
2441  }
2442  if (C) { // just C, no GV.
2443  // Simple constants are not allowed for 's'.
2444  if (ConstraintLetter != 's') {
2445  // gcc prints these as sign extended. Sign extend value to 64 bits
2446  // now; without this it would get ZExt'd later in
2447  // ScheduleDAGSDNodes::EmitNode, which is very generic.
2448  Ops.push_back(DAG.getTargetConstant(C->getAPIntValue().getSExtValue(),
2449  SDLoc(C), MVT::i64));
2450  }
2451  return;
2452  }
2453  break;
2454  }
2455  }
2456 }
2457 
2458 std::pair<unsigned, const TargetRegisterClass *>
2460  StringRef Constraint,
2461  MVT VT) const {
2462  if (Constraint.empty() || Constraint[0] != '{')
2463  return std::make_pair(0u, static_cast<TargetRegisterClass*>(nullptr));
2464  assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?");
2465 
2466  // Remove the braces from around the name.
2467  StringRef RegName(Constraint.data()+1, Constraint.size()-2);
2468 
2469  std::pair<unsigned, const TargetRegisterClass*> R =
2470  std::make_pair(0u, static_cast<const TargetRegisterClass*>(nullptr));
2471 
2472  // Figure out which register class contains this reg.
2474  E = RI->regclass_end(); RCI != E; ++RCI) {
2475  const TargetRegisterClass *RC = *RCI;
2476 
2477  // If none of the value types for this register class are valid, we
2478  // can't use it. For example, 64-bit reg classes on 32-bit targets.
2479  if (!isLegalRC(RC))
2480  continue;
2481 
2482  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
2483  I != E; ++I) {
2484  if (RegName.equals_lower(RI->getRegAsmName(*I))) {
2485  std::pair<unsigned, const TargetRegisterClass*> S =
2486  std::make_pair(*I, RC);
2487 
2488  // If this register class has the requested value type, return it,
2489  // otherwise keep searching and return the first class found
2490  // if no other is found which explicitly has the requested type.
2491  if (RC->hasType(VT))
2492  return S;
2493  else if (!R.second)
2494  R = S;
2495  }
2496  }
2497  }
2498 
2499  return R;
2500 }
2501 
2502 //===----------------------------------------------------------------------===//
2503 // Constraint Selection.
2504 
2505 /// Return true of this is an input operand that is a matching constraint like
2506 /// "4".
2508  assert(!ConstraintCode.empty() && "No known constraint!");
2509  return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
2510 }
2511 
2512 /// If this is an input matching constraint, this method returns the output
2513 /// operand it matches.
2515  assert(!ConstraintCode.empty() && "No known constraint!");
2516  return atoi(ConstraintCode.c_str());
2517 }
2518 
2519 /// Split up the constraint string from the inline assembly value into the
2520 /// specific constraints and their prefixes, and also tie in the associated
2521 /// operand values.
2522 /// If this returns an empty vector, and if the constraint string itself
2523 /// isn't empty, there was an error parsing.
2526  const TargetRegisterInfo *TRI,
2527  ImmutableCallSite CS) const {
2528  /// Information about all of the constraints.
2529  AsmOperandInfoVector ConstraintOperands;
2530  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
2531  unsigned maCount = 0; // Largest number of multiple alternative constraints.
2532 
2533  // Do a prepass over the constraints, canonicalizing them, and building up the
2534  // ConstraintOperands list.
2535  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
2536  unsigned ResNo = 0; // ResNo - The result number of the next output.
2537 
2538  for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
2539  ConstraintOperands.emplace_back(std::move(CI));
2540  AsmOperandInfo &OpInfo = ConstraintOperands.back();
2541 
2542  // Update multiple alternative constraint count.
2543  if (OpInfo.multipleAlternatives.size() > maCount)
2544  maCount = OpInfo.multipleAlternatives.size();
2545 
2546  OpInfo.ConstraintVT = MVT::Other;
2547 
2548  // Compute the value type for each operand.
2549  switch (OpInfo.Type) {
2550  case InlineAsm::isOutput:
2551  // Indirect outputs just consume an argument.
2552  if (OpInfo.isIndirect) {
2553  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
2554  break;
2555  }
2556 
2557  // The return value of the call is this value. As such, there is no
2558  // corresponding argument.
2559  assert(!CS.getType()->isVoidTy() &&
2560  "Bad inline asm!");
2561  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
2562  OpInfo.ConstraintVT =
2563  getSimpleValueType(DL, STy->getElementType(ResNo));
2564  } else {
2565  assert(ResNo == 0 && "Asm only has one result!");
2566  OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
2567  }
2568  ++ResNo;
2569  break;
2570  case InlineAsm::isInput:
2571  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
2572  break;
2573  case InlineAsm::isClobber:
2574  // Nothing to do.
2575  break;
2576  }
2577 
2578  if (OpInfo.CallOperandVal) {
2579  llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
2580  if (OpInfo.isIndirect) {
2581  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
2582  if (!PtrTy)
2583  report_fatal_error("Indirect operand for inline asm not a pointer!");
2584  OpTy = PtrTy->getElementType();
2585  }
2586 
2587  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
2588  if (StructType *STy = dyn_cast<StructType>(OpTy))
2589  if (STy->getNumElements() == 1)
2590  OpTy = STy->getElementType(0);
2591 
2592  // If OpTy is not a single value, it may be a struct/union that we
2593  // can tile with integers.
2594  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
2595  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
2596  switch (BitSize) {
2597  default: break;
2598  case 1:
2599  case 8:
2600  case 16:
2601  case 32:
2602  case 64:
2603  case 128:
2604  OpInfo.ConstraintVT =
2605  MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
2606  break;
2607  }
2608  } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
2609  unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
2610  OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
2611  } else {
2612  OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
2613  }
2614  }
2615  }
2616 
2617  // If we have multiple alternative constraints, select the best alternative.
2618  if (!ConstraintOperands.empty()) {
2619  if (maCount) {
2620  unsigned bestMAIndex = 0;
2621  int bestWeight = -1;
2622  // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
2623  int weight = -1;
2624  unsigned maIndex;
2625  // Compute the sums of the weights for each alternative, keeping track
2626  // of the best (highest weight) one so far.
2627  for (maIndex = 0; maIndex < maCount; ++maIndex) {
2628  int weightSum = 0;
2629  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
2630  cIndex != eIndex; ++cIndex) {
2631  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
2632  if (OpInfo.Type == InlineAsm::isClobber)
2633  continue;
2634 
2635  // If this is an output operand with a matching input operand,
2636  // look up the matching input. If their types mismatch, e.g. one
2637  // is an integer, the other is floating point, or their sizes are
2638  // different, flag it as an maCantMatch.
2639  if (OpInfo.hasMatchingInput()) {
2640  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
2641  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
2642  if ((OpInfo.ConstraintVT.isInteger() !=
2643  Input.ConstraintVT.isInteger()) ||
2644  (OpInfo.ConstraintVT.getSizeInBits() !=
2645  Input.ConstraintVT.getSizeInBits())) {
2646  weightSum = -1; // Can't match.
2647  break;
2648  }
2649  }
2650  }
2651  weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
2652  if (weight == -1) {
2653  weightSum = -1;
2654  break;
2655  }
2656  weightSum += weight;
2657  }
2658  // Update best.
2659  if (weightSum > bestWeight) {
2660  bestWeight = weightSum;
2661  bestMAIndex = maIndex;
2662  }
2663  }
2664 
2665  // Now select chosen alternative in each constraint.
2666  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
2667  cIndex != eIndex; ++cIndex) {
2668  AsmOperandInfo& cInfo = ConstraintOperands[cIndex];
2669  if (cInfo.Type == InlineAsm::isClobber)
2670  continue;
2671  cInfo.selectAlternative(bestMAIndex);
2672  }
2673  }
2674  }
2675 
2676  // Check and hook up tied operands, choose constraint code to use.
2677  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
2678  cIndex != eIndex; ++cIndex) {
2679  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
2680 
2681  // If this is an output operand with a matching input operand, look up the
2682  // matching input. If their types mismatch, e.g. one is an integer, the
2683  // other is floating point, or their sizes are different, flag it as an
2684  // error.
2685  if (OpInfo.hasMatchingInput()) {
2686  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
2687 
2688  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
2689  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
2691  OpInfo.ConstraintVT);
2692  std::pair<unsigned, const TargetRegisterClass *> InputRC =
2694  Input.ConstraintVT);
2695  if ((OpInfo.ConstraintVT.isInteger() !=
2696  Input.ConstraintVT.isInteger()) ||
2697  (MatchRC.second != InputRC.second)) {
2698  report_fatal_error("Unsupported asm: input constraint"
2699  " with a matching output constraint of"
2700  " incompatible type!");
2701  }
2702  }
2703  }
2704  }
2705 
2706  return ConstraintOperands;
2707 }
2708 
2709 /// Return an integer indicating how general CT is.
2711  switch (CT) {
2714  return 0;
2716  return 1;
2718  return 2;
2720  return 3;
2721  }
2722  llvm_unreachable("Invalid constraint type");
2723 }
2724 
2725 /// Examine constraint type and operand type and determine a weight value.
2726 /// This object must already have been set up with the operand type
2727 /// and the current alternative constraint selected.
2730  AsmOperandInfo &info, int maIndex) const {
2732  if (maIndex >= (int)info.multipleAlternatives.size())
2733  rCodes = &info.Codes;
2734  else
2735  rCodes = &info.multipleAlternatives[maIndex].Codes;
2736  ConstraintWeight BestWeight = CW_Invalid;
2737 
2738  // Loop over the options, keeping track of the most general one.
2739  for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
2740  ConstraintWeight weight =
2741  getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
2742  if (weight > BestWeight)
2743  BestWeight = weight;
2744  }
2745 
2746  return BestWeight;
2747 }
2748 
2749 /// Examine constraint type and operand type and determine a weight value.
2750 /// This object must already have been set up with the operand type
2751 /// and the current alternative constraint selected.
2754  AsmOperandInfo &info, const char *constraint) const {
2755  ConstraintWeight weight = CW_Invalid;
2756  Value *CallOperandVal = info.CallOperandVal;
2757  // If we don't have a value, we can't do a match,
2758  // but allow it at the lowest weight.
2759  if (!CallOperandVal)
2760  return CW_Default;
2761  // Look at the constraint type.
2762  switch (*constraint) {
2763  case 'i': // immediate integer.
2764  case 'n': // immediate integer with a known value.
2765  if (isa<ConstantInt>(CallOperandVal))
2766  weight = CW_Constant;
2767  break;
2768  case 's': // non-explicit intregal immediate.
2769  if (isa<GlobalValue>(CallOperandVal))
2770  weight = CW_Constant;
2771  break;
2772  case 'E': // immediate float if host format.
2773  case 'F': // immediate float.
2774  if (isa<ConstantFP>(CallOperandVal))
2775  weight = CW_Constant;
2776  break;
2777  case '<': // memory operand with autodecrement.
2778  case '>': // memory operand with autoincrement.
2779  case 'm': // memory operand.
2780  case 'o': // offsettable memory operand
2781  case 'V': // non-offsettable memory operand
2782  weight = CW_Memory;
2783  break;
2784  case 'r': // general register.
2785  case 'g': // general register, memory operand or immediate integer.
2786  // note: Clang converts "g" to "imr".
2787  if (CallOperandVal->getType()->isIntegerTy())
2788  weight = CW_Register;
2789  break;
2790  case 'X': // any operand.
2791  default:
2792  weight = CW_Default;
2793  break;
2794  }
2795  return weight;
2796 }
2797 
2798 /// If there are multiple different constraints that we could pick for this
2799 /// operand (e.g. "imr") try to pick the 'best' one.
2800 /// This is somewhat tricky: constraints fall into four classes:
2801 /// Other -> immediates and magic values
2802 /// Register -> one specific register
2803 /// RegisterClass -> a group of regs
2804 /// Memory -> memory
2805 /// Ideally, we would pick the most specific constraint possible: if we have
2806 /// something that fits into a register, we would pick it. The problem here
2807 /// is that if we have something that could either be in a register or in
2808 /// memory that use of the register could cause selection of *other*
2809 /// operands to fail: they might only succeed if we pick memory. Because of
2810 /// this the heuristic we use is:
2811 ///
2812 /// 1) If there is an 'other' constraint, and if the operand is valid for
2813 /// that constraint, use it. This makes us take advantage of 'i'
2814 /// constraints when available.
2815 /// 2) Otherwise, pick the most general constraint present. This prefers
2816 /// 'm' over 'r', for example.
2817 ///
2819  const TargetLowering &TLI,
2820  SDValue Op, SelectionDAG *DAG) {
2821  assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
2822  unsigned BestIdx = 0;
2824  int BestGenerality = -1;
2825 
2826  // Loop over the options, keeping track of the most general one.
2827  for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
2829  TLI.getConstraintType(OpInfo.Codes[i]);
2830 
2831  // If this is an 'other' constraint, see if the operand is valid for it.
2832  // For example, on X86 we might have an 'rI' constraint. If the operand
2833  // is an integer in the range [0..31] we want to use I (saving a load
2834  // of a register), otherwise we must use 'r'.
2835  if (CType == TargetLowering::C_Other && Op.getNode()) {
2836  assert(OpInfo.Codes[i].size() == 1 &&
2837  "Unhandled multi-letter 'other' constraint");
2838  std::vector<SDValue> ResultOps;
2839  TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
2840  ResultOps, *DAG);
2841  if (!ResultOps.empty()) {
2842  BestType = CType;
2843  BestIdx = i;
2844  break;
2845  }
2846  }
2847 
2848  // Things with matching constraints can only be registers, per gcc
2849  // documentation. This mainly affects "g" constraints.
2850  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
2851  continue;
2852 
2853  // This constraint letter is more general than the previous one, use it.
2854  int Generality = getConstraintGenerality(CType);
2855  if (Generality > BestGenerality) {
2856  BestType = CType;
2857  BestIdx = i;
2858  BestGenerality = Generality;
2859  }
2860  }
2861 
2862  OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
2863  OpInfo.ConstraintType = BestType;
2864 }
2865 
2866 /// Determines the constraint code and constraint type to use for the specific
2867 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
2869  SDValue Op,
2870  SelectionDAG *DAG) const {
2871  assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
2872 
2873  // Single-letter constraints ('r') are very common.
2874  if (OpInfo.Codes.size() == 1) {
2875  OpInfo.ConstraintCode = OpInfo.Codes[0];
2877  } else {
2878  ChooseConstraint(OpInfo, *this, Op, DAG);
2879  }
2880 
2881  // 'X' matches anything.
2882  if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
2883  // Labels and constants are handled elsewhere ('X' is the only thing
2884  // that matches labels). For Functions, the type here is the type of
2885  // the result, which is not what we want to look at; leave them alone.
2886  Value *v = OpInfo.CallOperandVal;
2887  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
2888  OpInfo.CallOperandVal = v;
2889  return;
2890  }
2891 
2892  // Otherwise, try to resolve it to something we know about by looking at
2893  // the actual operand type.
2894  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
2895  OpInfo.ConstraintCode = Repl;
2897  }
2898  }
2899 }
2900 
2901 /// \brief Given an exact SDIV by a constant, create a multiplication
2902 /// with the multiplicative inverse of the constant.
2904  const SDLoc &dl, SelectionDAG &DAG,
2905  std::vector<SDNode *> &Created) {
2906  assert(d != 0 && "Division by zero!");
2907 
2908  // Shift the value upfront if it is even, so the LSB is one.
2909  unsigned ShAmt = d.countTrailingZeros();
2910  if (ShAmt) {
2911  // TODO: For UDIV use SRL instead of SRA.
2912  SDValue Amt =
2913  DAG.getConstant(ShAmt, dl, TLI.getShiftAmountTy(Op1.getValueType(),
2914  DAG.getDataLayout()));
2916  Flags.setExact(true);
2917  Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, &Flags);
2918  Created.push_back(Op1.getNode());
2919  d = d.ashr(ShAmt);
2920  }
2921 
2922  // Calculate the multiplicative inverse, using Newton's method.
2923  APInt t, xn = d;
2924  while ((t = d*xn) != 1)
2925  xn *= APInt(d.getBitWidth(), 2) - t;
2926 
2927  SDValue Op2 = DAG.getConstant(xn, dl, Op1.getValueType());
2928  SDValue Mul = DAG.getNode(ISD::MUL, dl, Op1.getValueType(), Op1, Op2);
2929  Created.push_back(Mul.getNode());
2930  return Mul;
2931 }
2932 
2934  SelectionDAG &DAG,
2935  std::vector<SDNode *> *Created) const {
2937  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2938  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
2939  return SDValue(N,0); // Lower SDIV as SDIV
2940  return SDValue();
2941 }
2942 
2943 /// \brief Given an ISD::SDIV node expressing a divide by constant,
2944 /// return a DAG expression to select that will generate the same value by
2945 /// multiplying by a magic number.
2946 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
2948  SelectionDAG &DAG, bool IsAfterLegalization,
2949  std::vector<SDNode *> *Created) const {
2950  assert(Created && "No vector to hold sdiv ops.");
2951 
2952  EVT VT = N->getValueType(0);
2953  SDLoc dl(N);
2954 
2955  // Check to see if we can do this.
2956  // FIXME: We should be more aggressive here.
2957  if (!isTypeLegal(VT))
2958  return SDValue();
2959 
2960  // If the sdiv has an 'exact' bit we can use a simpler lowering.
2961  if (cast<BinaryWithFlagsSDNode>(N)->Flags.hasExact())
2962  return BuildExactSDIV(*this, N->getOperand(0), Divisor, dl, DAG, *Created);
2963 
2964  APInt::ms magics = Divisor.magic();
2965 
2966  // Multiply the numerator (operand 0) by the magic value
2967  // FIXME: We should support doing a MUL in a wider type
2968  SDValue Q;
2969  if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT) :
2971  Q = DAG.getNode(ISD::MULHS, dl, VT, N->getOperand(0),
2972  DAG.getConstant(magics.m, dl, VT));
2973  else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT) :
2975  Q = SDValue(DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT),
2976  N->getOperand(0),
2977  DAG.getConstant(magics.m, dl, VT)).getNode(), 1);
2978  else
2979  return SDValue(); // No mulhs or equvialent
2980  // If d > 0 and m < 0, add the numerator
2981  if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
2982  Q = DAG.getNode(ISD::ADD, dl, VT, Q, N->getOperand(0));
2983  Created->push_back(Q.getNode());
2984  }
2985  // If d < 0 and m > 0, subtract the numerator.
2986  if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
2987  Q = DAG.getNode(ISD::SUB, dl, VT, Q, N->getOperand(0));
2988  Created->push_back(Q.getNode());
2989  }
2990  auto &DL = DAG.getDataLayout();
2991  // Shift right algebraic if shift value is nonzero
2992  if (magics.s > 0) {
2993  Q = DAG.getNode(
2994  ISD::SRA, dl, VT, Q,
2995  DAG.getConstant(magics.s, dl, getShiftAmountTy(Q.getValueType(), DL)));
2996  Created->push_back(Q.getNode());
2997  }
2998  // Extract the sign bit and add it to the quotient
2999  SDValue T =
3000  DAG.getNode(ISD::SRL, dl, VT, Q,
3001  DAG.getConstant(VT.getScalarSizeInBits() - 1, dl,
3002  getShiftAmountTy(Q.getValueType(), DL)));
3003  Created->push_back(T.getNode());
3004  return DAG.getNode(ISD::ADD, dl, VT, Q, T);
3005 }
3006 
3007 /// \brief Given an ISD::UDIV node expressing a divide by constant,
3008 /// return a DAG expression to select that will generate the same value by
3009 /// multiplying by a magic number.
3010 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3012  SelectionDAG &DAG, bool IsAfterLegalization,
3013  std::vector<SDNode *> *Created) const {
3014  assert(Created && "No vector to hold udiv ops.");
3015 
3016  EVT VT = N->getValueType(0);
3017  SDLoc dl(N);
3018  auto &DL = DAG.getDataLayout();
3019 
3020  // Check to see if we can do this.
3021  // FIXME: We should be more aggressive here.
3022  if (!isTypeLegal(VT))
3023  return SDValue();
3024 
3025  // FIXME: We should use a narrower constant when the upper
3026  // bits are known to be zero.
3027  APInt::mu magics = Divisor.magicu();
3028 
3029  SDValue Q = N->getOperand(0);
3030 
3031  // If the divisor is even, we can avoid using the expensive fixup by shifting
3032  // the divided value upfront.
3033  if (magics.a != 0 && !Divisor[0]) {
3034  unsigned Shift = Divisor.countTrailingZeros();
3035  Q = DAG.getNode(
3036  ISD::SRL, dl, VT, Q,
3037  DAG.getConstant(Shift, dl, getShiftAmountTy(Q.getValueType(), DL)));
3038  Created->push_back(Q.getNode());
3039 
3040  // Get magic number for the shifted divisor.
3041  magics = Divisor.lshr(Shift).magicu(Shift);
3042  assert(magics.a == 0 && "Should use cheap fixup now");
3043  }
3044 
3045  // Multiply the numerator (operand 0) by the magic value
3046  // FIXME: We should support doing a MUL in a wider type
3047  if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT) :
3049  Q = DAG.getNode(ISD::MULHU, dl, VT, Q, DAG.getConstant(magics.m, dl, VT));
3050  else if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT) :
3052  Q = SDValue(DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), Q,
3053  DAG.getConstant(magics.m, dl, VT)).getNode(), 1);
3054  else
3055  return SDValue(); // No mulhu or equivalent
3056 
3057  Created->push_back(Q.getNode());
3058 
3059  if (magics.a == 0) {
3060  assert(magics.s < Divisor.getBitWidth() &&
3061  "We shouldn't generate an undefined shift!");
3062  return DAG.getNode(
3063  ISD::SRL, dl, VT, Q,
3064  DAG.getConstant(magics.s, dl, getShiftAmountTy(Q.getValueType(), DL)));
3065  } else {
3066  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N->getOperand(0), Q);
3067  Created->push_back(NPQ.getNode());
3068  NPQ = DAG.getNode(
3069  ISD::SRL, dl, VT, NPQ,
3070  DAG.getConstant(1, dl, getShiftAmountTy(NPQ.getValueType(), DL)));
3071  Created->push_back(NPQ.getNode());
3072  NPQ = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
3073  Created->push_back(NPQ.getNode());
3074  return DAG.getNode(
3075  ISD::SRL, dl, VT, NPQ,
3076  DAG.getConstant(magics.s - 1, dl,
3077  getShiftAmountTy(NPQ.getValueType(), DL)));
3078  }
3079 }
3080 
3081 bool TargetLowering::
3083  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
3084  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
3085  "be a constant integer");
3086  return true;
3087  }
3088 
3089  return false;
3090 }
3091 
3092 //===----------------------------------------------------------------------===//
3093 // Legalization Utilities
3094 //===----------------------------------------------------------------------===//
3095 
3096 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
3097  SDValue LHS, SDValue RHS,
3098  SmallVectorImpl<SDValue> &Result,
3099  EVT HiLoVT, SelectionDAG &DAG,
3101  SDValue LH, SDValue RL, SDValue RH) const {
3102  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
3103  Opcode == ISD::SMUL_LOHI);
3104 
3105  bool HasMULHS = (Kind == MulExpansionKind::Always) ||
3107  bool HasMULHU = (Kind == MulExpansionKind::Always) ||
3109  bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
3111  bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
3113 
3114  if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
3115  return false;
3116 
3117  unsigned OuterBitSize = VT.getScalarSizeInBits();
3118  unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
3119  unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
3120  unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
3121 
3122  // LL, LH, RL, and RH must be either all NULL or all set to a value.
3123  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
3124  (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
3125 
3126  SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
3127  auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
3128  bool Signed) -> bool {
3129  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
3130  Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
3131  Hi = SDValue(Lo.getNode(), 1);
3132  return true;
3133  }
3134  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
3135  Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
3136  Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
3137  return true;
3138  }
3139  return false;
3140  };
3141 
3142  SDValue Lo, Hi;
3143 
3144  if (!LL.getNode() && !RL.getNode() &&
3146  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
3147  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
3148  }
3149 
3150  if (!LL.getNode())
3151  return false;
3152 
3153  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
3154  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
3155  DAG.MaskedValueIsZero(RHS, HighMask)) {
3156  // The inputs are both zero-extended.
3157  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
3158  Result.push_back(Lo);
3159  Result.push_back(Hi);
3160  if (Opcode != ISD::MUL) {
3161  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
3162  Result.push_back(Zero);
3163  Result.push_back(Zero);
3164  }
3165  return true;
3166  }
3167  }
3168 
3169  if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
3170  RHSSB > InnerBitSize) {
3171  // The input values are both sign-extended.
3172  // TODO non-MUL case?
3173  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
3174  Result.push_back(Lo);
3175  Result.push_back(Hi);
3176  return true;
3177  }
3178  }
3179 
3180  unsigned ShiftAmount = OuterBitSize - InnerBitSize;
3181  EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
3182  if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
3183  // FIXME getShiftAmountTy does not always return a sensible result when VT
3184  // is an illegal type, and so the type may be too small to fit the shift
3185  // amount. Override it with i32. The shift will have to be legalized.
3186  ShiftAmountTy = MVT::i32;
3187  }
3188  SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
3189 
3190  if (!LH.getNode() && !RH.getNode() &&
3193  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
3194  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
3195  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
3196  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
3197  }
3198 
3199  if (!LH.getNode())
3200  return false;
3201 
3202  if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
3203  return false;
3204 
3205  Result.push_back(Lo);
3206 
3207  if (Opcode == ISD::MUL) {
3208  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
3209  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
3210  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
3211  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
3212  Result.push_back(Hi);
3213  return true;
3214  }
3215 
3216  // Compute the full width result.
3217  auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
3218  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3219  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
3220  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3221  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
3222  };
3223 
3224  SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
3225  if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
3226  return false;
3227 
3228  // This is effectively the add part of a multiply-add of half-sized operands,
3229  // so it cannot overflow.
3230  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
3231 
3232  if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
3233  return false;
3234 
3235  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
3236  Merge(Lo, Hi));
3237 
3238  SDValue Carry = Next.getValue(1);
3239  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3240  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
3241 
3242  if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
3243  return false;
3244 
3245  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
3246  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
3247  Carry);
3248  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
3249 
3250  if (Opcode == ISD::SMUL_LOHI) {
3251  SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
3252  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
3253  Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
3254 
3255  NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
3256  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
3257  Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
3258  }
3259 
3260  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3261  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
3262  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3263  return true;
3264 }
3265 
3268  SDValue LL, SDValue LH, SDValue RL,
3269  SDValue RH) const {
3270  SmallVector<SDValue, 2> Result;
3271  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
3272  N->getOperand(0), N->getOperand(1), Result, HiLoVT,
3273  DAG, Kind, LL, LH, RL, RH);
3274  if (Ok) {
3275  assert(Result.size() == 2);
3276  Lo = Result[0];
3277  Hi = Result[1];
3278  }
3279  return Ok;
3280 }
3281 
3283  SelectionDAG &DAG) const {
3284  EVT VT = Node->getOperand(0).getValueType();
3285  EVT NVT = Node->getValueType(0);
3286  SDLoc dl(SDValue(Node, 0));
3287 
3288  // FIXME: Only f32 to i64 conversions are supported.
3289  if (VT != MVT::f32 || NVT != MVT::i64)
3290  return false;
3291 
3292  // Expand f32 -> i64 conversion
3293  // This algorithm comes from compiler-rt's implementation of fixsfdi:
3294  // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c
3295  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(),
3296  VT.getSizeInBits());
3297  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
3298  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
3299  SDValue Bias = DAG.getConstant(127, dl, IntVT);
3300  SDValue SignMask = DAG.getConstant(APInt::getSignBit(VT.getSizeInBits()), dl,
3301  IntVT);
3302  SDValue SignLowBit = DAG.getConstant(VT.getSizeInBits() - 1, dl, IntVT);
3303  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
3304 
3305  SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Node->getOperand(0));
3306 
3307  auto &DL = DAG.getDataLayout();
3308  SDValue ExponentBits = DAG.getNode(
3309  ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
3310  DAG.getZExtOrTrunc(ExponentLoBit, dl, getShiftAmountTy(IntVT, DL)));
3311  SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
3312 
3313  SDValue Sign = DAG.getNode(
3314  ISD::SRA, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
3315  DAG.getZExtOrTrunc(SignLowBit, dl, getShiftAmountTy(IntVT, DL)));
3316  Sign = DAG.getSExtOrTrunc(Sign, dl, NVT);
3317 
3318  SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
3319  DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
3320  DAG.getConstant(0x00800000, dl, IntVT));
3321 
3322  R = DAG.getZExtOrTrunc(R, dl, NVT);
3323 
3324  R = DAG.getSelectCC(
3325  dl, Exponent, ExponentLoBit,
3326  DAG.getNode(ISD::SHL, dl, NVT, R,
3327  DAG.getZExtOrTrunc(
3328  DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
3329  dl, getShiftAmountTy(IntVT, DL))),
3330  DAG.getNode(ISD::SRL, dl, NVT, R,
3331  DAG.getZExtOrTrunc(
3332  DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
3333  dl, getShiftAmountTy(IntVT, DL))),
3334  ISD::SETGT);
3335 
3336  SDValue Ret = DAG.getNode(ISD::SUB, dl, NVT,
3337  DAG.getNode(ISD::XOR, dl, NVT, R, Sign),
3338  Sign);
3339 
3340  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
3341  DAG.getConstant(0, dl, NVT), Ret, ISD::SETLT);
3342  return true;
3343 }
3344 
3346  SelectionDAG &DAG) const {
3347  SDLoc SL(LD);
3348  SDValue Chain = LD->getChain();
3349  SDValue BasePTR = LD->getBasePtr();
3350  EVT SrcVT = LD->getMemoryVT();
3351  ISD::LoadExtType ExtType = LD->getExtensionType();
3352 
3353  unsigned NumElem = SrcVT.getVectorNumElements();
3354 
3355  EVT SrcEltVT = SrcVT.getScalarType();
3356  EVT DstEltVT = LD->getValueType(0).getScalarType();
3357 
3358  unsigned Stride = SrcEltVT.getSizeInBits() / 8;
3359  assert(SrcEltVT.isByteSized());
3360 
3361  EVT PtrVT = BasePTR.getValueType();
3362 
3364  SmallVector<SDValue, 8> LoadChains;
3365 
3366  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
3367  SDValue ScalarLoad =
3368  DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
3369  LD->getPointerInfo().getWithOffset(Idx * Stride),
3370  SrcEltVT, MinAlign(LD->getAlignment(), Idx * Stride),
3371  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3372 
3373  BasePTR = DAG.getNode(ISD::ADD, SL, PtrVT, BasePTR,
3374  DAG.getConstant(Stride, SL, PtrVT));
3375 
3376  Vals.push_back(ScalarLoad.getValue(0));
3377  LoadChains.push_back(ScalarLoad.getValue(1));
3378  }
3379 
3380  SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
3381  SDValue Value = DAG.getBuildVector(LD->getValueType(0), SL, Vals);
3382 
3383  return DAG.getMergeValues({ Value, NewChain }, SL);
3384 }
3385 
3386 // FIXME: This relies on each element having a byte size, otherwise the stride
3387 // is 0 and just overwrites the same location. ExpandStore currently expects
3388 // this broken behavior.
3390  SelectionDAG &DAG) const {
3391  SDLoc SL(ST);
3392 
3393  SDValue Chain = ST->getChain();
3394  SDValue BasePtr = ST->getBasePtr();
3395  SDValue Value = ST->getValue();
3396  EVT StVT = ST->getMemoryVT();
3397 
3398  // The type of the data we want to save
3399  EVT RegVT = Value.getValueType();
3400  EVT RegSclVT = RegVT.getScalarType();
3401 
3402  // The type of data as saved in memory.
3403  EVT MemSclVT = StVT.getScalarType();
3404 
3405  EVT PtrVT = BasePtr.getValueType();
3406 
3407  // Store Stride in bytes
3408  unsigned Stride = MemSclVT.getSizeInBits() / 8;
3409  EVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
3410  unsigned NumElem = StVT.getVectorNumElements();
3411 
3412  // Extract each of the elements from the original vector and save them into
3413  // memory individually.
3414  SmallVector<SDValue, 8> Stores;
3415  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
3416  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
3417  DAG.getConstant(Idx, SL, IdxVT));
3418 
3419  SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
3420  DAG.getConstant(Idx * Stride, SL, PtrVT));
3421 
3422  // This scalar TruncStore may be illegal, but we legalize it later.
3423  SDValue Store = DAG.getTruncStore(
3424  Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
3425  MemSclVT, MinAlign(ST->getAlignment(), Idx * Stride),
3426  ST->getMemOperand()->getFlags(), ST->getAAInfo());
3427 
3428  Stores.push_back(Store);
3429  }
3430 
3431  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
3432 }
3433 
3434 std::pair<SDValue, SDValue>
3437  "unaligned indexed loads not implemented!");
3438  SDValue Chain = LD->getChain();
3439  SDValue Ptr = LD->getBasePtr();
3440  EVT VT = LD->getValueType(0);
3441  EVT LoadedVT = LD->getMemoryVT();
3442  SDLoc dl(LD);
3443  if (VT.isFloatingPoint() || VT.isVector()) {
3444  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
3445  if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
3446  if (!isOperationLegalOrCustom(ISD::LOAD, intVT)) {
3447  // Scalarize the load and let the individual components be handled.
3448  SDValue Scalarized = scalarizeVectorLoad(LD, DAG);
3449  return std::make_pair(Scalarized.getValue(0), Scalarized.getValue(1));
3450  }
3451 
3452  // Expand to a (misaligned) integer load of the same size,
3453  // then bitconvert to floating point or vector.
3454  SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
3455  LD->getMemOperand());
3456  SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
3457  if (LoadedVT != VT)
3458  Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
3459  ISD::ANY_EXTEND, dl, VT, Result);
3460 
3461  return std::make_pair(Result, newLoad.getValue(1));
3462  }
3463 
3464  // Copy the value to a (aligned) stack slot using (unaligned) integer
3465  // loads and stores, then do a (aligned) load from the stack slot.
3466  MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
3467  unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8;
3468  unsigned RegBytes = RegVT.getSizeInBits() / 8;
3469  unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
3470 
3471  // Make sure the stack slot is also aligned for the register type.
3472  SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
3473 
3474  SmallVector<SDValue, 8> Stores;
3475  SDValue StackPtr = StackBase;
3476  unsigned Offset = 0;
3477 
3478  EVT PtrVT = Ptr.getValueType();
3479  EVT StackPtrVT = StackPtr.getValueType();
3480 
3481  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
3482  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
3483 
3484  // Do all but one copies using the full register width.
3485  for (unsigned i = 1; i < NumRegs; i++) {
3486  // Load one integer register's worth from the original location.
3487  SDValue Load = DAG.getLoad(
3488  RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
3490  LD->getAAInfo());
3491  // Follow the load with a store to the stack slot. Remember the store.
3492  Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr,
3493  MachinePointerInfo()));
3494  // Increment the pointers.
3495  Offset += RegBytes;
3496  Ptr = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr, PtrIncrement);
3497  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtrVT, StackPtr,
3498  StackPtrIncrement);
3499  }
3500 
3501  // The last copy may be partial. Do an extending load.
3502  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
3503  8 * (LoadedBytes - Offset));
3504  SDValue Load =
3505  DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
3506  LD->getPointerInfo().getWithOffset(Offset), MemVT,
3507  MinAlign(LD->getAlignment(), Offset),
3508  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3509  // Follow the load with a store to the stack slot. Remember the store.
3510  // On big-endian machines this requires a truncating store to ensure
3511  // that the bits end up in the right place.
3512  Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr,
3513  MachinePointerInfo(), MemVT));
3514 
3515  // The order of the stores doesn't matter - say it with a TokenFactor.
3516  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
3517 
3518  // Finally, perform the original load only redirected to the stack slot.
3519  Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
3520  MachinePointerInfo(), LoadedVT);
3521 
3522  // Callers expect a MERGE_VALUES node.
3523  return std::make_pair(Load, TF);
3524  }
3525 
3526  assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
3527  "Unaligned load of unsupported type.");
3528 
3529  // Compute the new VT that is half the size of the old one. This is an
3530  // integer MVT.
3531  unsigned NumBits = LoadedVT.getSizeInBits();
3532  EVT NewLoadedVT;
3533  NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
3534  NumBits >>= 1;
3535 
3536  unsigned Alignment = LD->getAlignment();
3537  unsigned IncrementSize = NumBits / 8;
3538  ISD::LoadExtType HiExtType = LD->getExtensionType();
3539 
3540  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
3541  if (HiExtType == ISD::NON_EXTLOAD)
3542  HiExtType = ISD::ZEXTLOAD;
3543 
3544  // Load the value in two parts
3545  SDValue Lo, Hi;
3546  if (DAG.getDataLayout().isLittleEndian()) {
3547  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
3548  NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
3549  LD->getAAInfo());
3550  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3551  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3552  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
3553  LD->getPointerInfo().getWithOffset(IncrementSize),
3554  NewLoadedVT, MinAlign(Alignment, IncrementSize),
3555  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3556  } else {
3557  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
3558  NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
3559  LD->getAAInfo());
3560  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3561  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3562  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
3563  LD->getPointerInfo().getWithOffset(IncrementSize),
3564  NewLoadedVT, MinAlign(Alignment, IncrementSize),
3565  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3566  }
3567 
3568  // aggregate the two parts
3569  SDValue ShiftAmount =
3570  DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
3571  DAG.getDataLayout()));
3572  SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
3573  Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
3574 
3575  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3576  Hi.getValue(1));
3577 
3578  return std::make_pair(Result, TF);
3579 }
3580 
3582  SelectionDAG &DAG) const {
3584  "unaligned indexed stores not implemented!");
3585  SDValue Chain = ST->getChain();
3586  SDValue Ptr = ST->getBasePtr();
3587  SDValue Val = ST->getValue();
3588  EVT VT = Val.getValueType();
3589  int Alignment = ST->getAlignment();
3590 
3591  SDLoc dl(ST);
3592  if (ST->getMemoryVT().isFloatingPoint() ||
3593  ST->getMemoryVT().isVector()) {
3594  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
3595  if (isTypeLegal(intVT)) {
3596  if (!isOperationLegalOrCustom(ISD::STORE, intVT)) {
3597  // Scalarize the store and let the individual components be handled.
3598  SDValue Result = scalarizeVectorStore(ST, DAG);
3599 
3600  return Result;
3601  }
3602  // Expand to a bitconvert of the value to the integer type of the
3603  // same size, then a (misaligned) int store.
3604  // FIXME: Does not handle truncating floating point stores!
3605  SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
3606  Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
3607  Alignment, ST->getMemOperand()->getFlags());
3608  return Result;
3609  }
3610  // Do a (aligned) store to a stack slot, then copy from the stack slot
3611  // to the final destination using (unaligned) integer loads and stores.
3612  EVT StoredVT = ST->getMemoryVT();
3613  MVT RegVT =
3614  getRegisterType(*DAG.getContext(),
3616  StoredVT.getSizeInBits()));
3617  EVT PtrVT = Ptr.getValueType();
3618  unsigned StoredBytes = StoredVT.getSizeInBits() / 8;
3619  unsigned RegBytes = RegVT.getSizeInBits() / 8;
3620  unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
3621 
3622  // Make sure the stack slot is also aligned for the register type.
3623  SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT);
3624 
3625  // Perform the original store, only redirected to the stack slot.
3626  SDValue Store = DAG.getTruncStore(Chain, dl, Val, StackPtr,
3627  MachinePointerInfo(), StoredVT);
3628 
3629  EVT StackPtrVT = StackPtr.getValueType();
3630 
3631  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
3632  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
3633  SmallVector<SDValue, 8> Stores;
3634  unsigned Offset = 0;
3635 
3636  // Do all but one copies using the full register width.
3637  for (unsigned i = 1; i < NumRegs; i++) {
3638  // Load one integer register's worth from the stack slot.
3639  SDValue Load =
3640  DAG.getLoad(RegVT, dl, Store, StackPtr, MachinePointerInfo());
3641  // Store it to the final location. Remember the store.
3642  Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
3643  ST->getPointerInfo().getWithOffset(Offset),
3644  MinAlign(ST->getAlignment(), Offset),
3645  ST->getMemOperand()->getFlags()));
3646  // Increment the pointers.
3647  Offset += RegBytes;
3648  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtrVT,
3649  StackPtr, StackPtrIncrement);
3650  Ptr = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr, PtrIncrement);
3651  }
3652 
3653  // The last store may be partial. Do a truncating store. On big-endian
3654  // machines this requires an extending load from the stack slot to ensure
3655  // that the bits are in the right place.
3656  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
3657  8 * (StoredBytes - Offset));
3658 
3659  // Load from the stack slot.
3660  SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
3661  MachinePointerInfo(), MemVT);
3662 
3663  Stores.push_back(
3664  DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
3665  ST->getPointerInfo().getWithOffset(Offset), MemVT,
3666  MinAlign(ST->getAlignment(), Offset),
3667  ST->getMemOperand()->getFlags(), ST->getAAInfo()));
3668  // The order of the stores doesn't matter - say it with a TokenFactor.
3669  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
3670  return Result;
3671  }
3672 
3673  assert(ST->getMemoryVT().isInteger() &&
3674  !ST->getMemoryVT().isVector() &&
3675  "Unaligned store of unknown type.");
3676  // Get the half-size VT
3677  EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext());
3678  int NumBits = NewStoredVT.getSizeInBits();
3679  int IncrementSize = NumBits / 8;
3680 
3681  // Divide the stored value in two parts.
3682  SDValue ShiftAmount =
3683  DAG.getConstant(NumBits, dl, getShiftAmountTy(Val.getValueType(),
3684  DAG.getDataLayout()));
3685  SDValue Lo = Val;
3686  SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
3687 
3688  // Store the two parts
3689  SDValue Store1, Store2;
3690  Store1 = DAG.getTruncStore(Chain, dl,
3691  DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
3692  Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
3693  ST->getMemOperand()->getFlags());
3694 
3695  EVT PtrVT = Ptr.getValueType();
3696  Ptr = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3697  DAG.getConstant(IncrementSize, dl, PtrVT));
3698  Alignment = MinAlign(Alignment, IncrementSize);
3699  Store2 = DAG.getTruncStore(
3700  Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
3701  ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
3702  ST->getMemOperand()->getFlags(), ST->getAAInfo());
3703 
3704  SDValue Result =
3705  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
3706  return Result;
3707 }
3708 
3709 SDValue
3711  const SDLoc &DL, EVT DataVT,
3712  SelectionDAG &DAG,
3713  bool IsCompressedMemory) const {
3714  SDValue Increment;
3715  EVT AddrVT = Addr.getValueType();
3716  EVT MaskVT = Mask.getValueType();
3717  assert(DataVT.getVectorNumElements() == MaskVT.getVectorNumElements() &&
3718  "Incompatible types of Data and Mask");
3719  if (IsCompressedMemory) {
3720  // Incrementing the pointer according to number of '1's in the mask.
3721  EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
3722  SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
3723  if (MaskIntVT.getSizeInBits() < 32) {
3724  MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
3725  MaskIntVT = MVT::i32;
3726  }
3727 
3728  // Count '1's with POPCNT.
3729  Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
3730  Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
3731  // Scale is an element size in bytes.
3732  SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
3733  AddrVT);
3734  Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
3735  } else
3736  Increment = DAG.getConstant(DataVT.getSizeInBits() / 8, DL, AddrVT);
3737 
3738  return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
3739 }
3740 
3742  SDValue Idx,
3743  EVT VecVT,
3744  const SDLoc &dl) {
3745  if (isa<ConstantSDNode>(Idx))
3746  return Idx;
3747 
3748  EVT IdxVT = Idx.getValueType();
3749  unsigned NElts = VecVT.getVectorNumElements();
3750  if (isPowerOf2_32(NElts)) {
3752  Log2_32(NElts));
3753  return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
3754  DAG.getConstant(Imm, dl, IdxVT));
3755  }
3756 
3757  return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
3758  DAG.getConstant(NElts - 1, dl, IdxVT));
3759 }
3760 
3762  SDValue VecPtr, EVT VecVT,
3763  SDValue Index) const {
3764  SDLoc dl(Index);
3765  // Make sure the index type is big enough to compute in.
3766  Index = DAG.getZExtOrTrunc(Index, dl, getPointerTy(DAG.getDataLayout()));
3767 
3768  EVT EltVT = VecVT.getVectorElementType();
3769 
3770  // Calculate the element offset and add it to the pointer.
3771  unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
3772  assert(EltSize * 8 == EltVT.getSizeInBits() &&
3773  "Converting bits to bytes lost precision");
3774 
3775  Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl);
3776 
3777  EVT IdxVT = Index.getValueType();
3778 
3779  Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
3780  DAG.getConstant(EltSize, dl, IdxVT));
3781  return DAG.getNode(ISD::ADD, dl, IdxVT, Index, VecPtr);
3782 }
3783 
3784 //===----------------------------------------------------------------------===//
3785 // Implementation of Emulated TLS Model
3786 //===----------------------------------------------------------------------===//
3787 
3789  SelectionDAG &DAG) const {
3790  // Access to address of TLS varialbe xyz is lowered to a function call:
3791  // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
3792  EVT PtrVT = getPointerTy(DAG.getDataLayout());
3793  PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
3794  SDLoc dl(GA);
3795 
3796  ArgListTy Args;
3797  ArgListEntry Entry;
3798  std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
3799  Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
3800  StringRef EmuTlsVarName(NameString);
3801  GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
3802  assert(EmuTlsVar && "Cannot find EmuTlsVar ");
3803  Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
3804  Entry.Ty = VoidPtrType;
3805  Args.push_back(Entry);
3806 
3807  SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
3808 
3810  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
3811  CLI.setCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
3812  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3813 
3814  // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
3815  // At last for X86 targets, maybe good for other targets too?
3817  MFI.setAdjustsStack(true); // Is this only for X86 target?
3818  MFI.setHasCalls(true);
3819 
3820  assert((GA->getOffset() == 0) &&
3821  "Emulated TLS must have zero offset in GlobalAddressSDNode");
3822  return CallResult.first;
3823 }
3824 
3826  SelectionDAG &DAG) const {
3827  assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
3828  if (!isCtlzFast())
3829  return SDValue();
3830  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
3831  SDLoc dl(Op);
3832  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
3833  if (C->isNullValue() && CC == ISD::SETEQ) {
3834  EVT VT = Op.getOperand(0).getValueType();
3835  SDValue Zext = Op.getOperand(0);
3836  if (VT.bitsLT(MVT::i32)) {
3837  VT = MVT::i32;
3838  Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
3839  }
3840  unsigned Log2b = Log2_32(VT.getSizeInBits());
3841  SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
3842  SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
3843  DAG.getConstant(Log2b, dl, MVT::i32));
3844  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
3845  }
3846  }
3847  return SDValue();
3848 }
MachineLoop * L
bool hasType(MVT vt) const
Return true if this TargetRegisterClass has the ValueType vt.
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
bool hasNoUnsignedWrap() const
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:500
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
APInt ashr(unsigned shiftAmt) const
Arithmetic right-shift function.
Definition: APInt.cpp:1035
static MVT getIntegerVT(unsigned BitWidth)
unsigned Log2_32_Ceil(uint32_t Value)
Log2_32_Ceil - This function returns the ceil log base 2 of the specified value, 32 if the value is z...
Definition: MathExtras.h:526
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:762
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
mu magicu(unsigned LeadingZeros=0) const
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
Definition: APInt.cpp:1441
SDValue getValue(unsigned R) const
static APInt getSignBit(unsigned BitWidth)
Get the SignBit for a specific bit width.
Definition: APInt.h:451
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
Flags getFlags() const
Return the raw flags of the source value,.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:458
LLVMContext * getContext() const
Definition: SelectionDAG.h:333
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:804
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:298
size_t i
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
virtual StringRef getRegAsmName(unsigned Reg) const
Return the assembly name for Reg.
Various leaf nodes.
Definition: ISDOpcodes.h:60
bool hasOneUse() const
Return true if there is exactly one use of this node.
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) != Y —> (~X & Y) !=...
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool ShrinkDemandedConstant(SDValue Op, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
const TargetMachine & getTargetMachine() const
SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:219
bool isConstTrueVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
unsigned s
shift amount
Definition: APInt.h:1712
bool isKnownToBeAPowerOfTwo(SDValue Val) const
Test if the given value is known to have exactly one bit set.
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the MVT corresponding to this LLVM type. See getValueType.
bool hasExact() const
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:536
const GlobalValue * getGlobal() const
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
unsigned getSizeInBits() const
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDValue Op1, APInt d, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDNode * > &Created)
Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the con...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
regclass_iterator regclass_end() const
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
bool isBitwiseNot(SDValue V)
Returns true if V is a bitwise not operation.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
The two locations do not alias at all.
Definition: AliasAnalysis.h:79
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
ms magic() const
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
Definition: APInt.cpp:1397
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
static bool isCommutativeBinOp(unsigned Opcode)
Returns true if the opcode is a commutative binary operation.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ConstantSDNode * getConstantSplatNode(BitVector *UndefElements=nullptr) const
Returns the splatted constant or null if this is not a constant splat.
Type * getElementType() const
Definition: DerivedTypes.h:462
SDValue scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
Definition: Attributes.cpp:994
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal...
Definition: ISDOpcodes.h:888
const SDValue & getBasePtr() const
CallLoweringInfo & setDiscardResult(bool Value=true)
SDValue BuildUDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector< SDNode * > *Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
Magic data for optimising unsigned division by a constant.
Definition: APInt.h:1716
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:369
SDValue getConstTrueVal(SelectionDAG &DAG, EVT VT, const SDLoc &DL) const
Return a constant of type VT that contains a true value that respects getBooleanContents() ...
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:431
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:212
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:263
bool isRegLoc() const
bool isAllOnesValue() const
SDValue getExternalSymbol(const char *Sym, EVT VT)
ConstraintCodeVector Codes
Code - The constraint code, either the register name (in braces) or the constraint letter/number...
Definition: InlineAsm.h:148
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition: Type.h:239
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:324
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors...
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:133
virtual const char * LowerXConstraint(EVT ConstraintVT) const
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
void setNoSignedWrap(bool b)
lazy value info
void setAttributes(ImmutableCallSite *CS, unsigned AttrIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
bool isRound() const
isRound - Return true if the size is a power-of-two number of bytes.
Definition: ValueTypes.h:188
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &dl)
struct fuzzer::@269 Flags
Shift and rotation operations.
Definition: ISDOpcodes.h:344
static F t[256]
Class to represent struct types.
Definition: DerivedTypes.h:199
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:327
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
Definition: CallSite.h:102
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
virtual void computeKnownBitsForTargetNode(const SDValue Op, APInt &KnownZero, APInt &KnownOne, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.cpp:1122
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:34
iterator begin() const
begin/end - Return all of the registers in this class.
Reg
All possible values of the reg field in the ModR/M byte.
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:233
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
bool isPositionIndependent() const
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:123
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:239
Context object for machine code objects.
Definition: MCContext.h:51
void emitError(unsigned LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:850
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getLocReg() const
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
#define F(x, y, z)
Definition: MD5.cpp:51
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
static SDNodeIterator begin(const SDNode *N)
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool Signed) const
Return if N is a True value when extended to VT.
This contains information for each constraint that we are lowering.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
EK_BlockAddress - Each entry is a plain address of block, e.g.
CallLoweringInfo & setZExtResult(bool Value=true)
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:220
signed char MatchingInput
MatchingInput - If this is not -1, this is an output constraint where an input constraint is required...
Definition: InlineAsm.h:130
const SDValue & getBasePtr() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
getHalfSizedIntegerVT - Finds the smallest simple value type that is greater than or equal to half th...
Definition: ValueTypes.h:293
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:737
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:842
const APInt & getAPIntValue() const
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:487
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber.
Definition: InlineAsm.h:120
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:135
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:875
unsigned getMinSignedBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1298
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:151
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Idx) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:328
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
Definition: ValueTypes.h:218
bool isLegalRC(const TargetRegisterClass *RC) const
Return true if the value types that can be represented by the specified register class are all legal...
Class to represent pointers.
Definition: DerivedTypes.h:443
This class is used to represent ISD::STORE nodes.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:274
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:518
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL) const
Soften the operands of a comparison.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:254
SDNode * getNode() const
get the SDNode which holds the desired result
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:589
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, std::vector< SDNode * > *Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:262
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool isMatchingInputConstraint() const
Return true of this is an input operand that is a matching constraint like "4".
EK_GPRel64BlockAddress - Each entry is an address of block, encoded with a relocation as gp-relative...
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:166
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors...
unsigned const MachineRegisterInfo * MRI
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1147
regclass_iterator regclass_begin() const
Register class iterators.
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:916
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:399
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
bool isIndirect
isIndirect - True if this operand is an indirect operand.
Definition: InlineAsm.h:144
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:818
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1321
unsigned getMatchedOperand() const
If this is an input matching constraint, this method returns the output operand it matches...
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:228
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Definition: APInt.h:1952
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:587
static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo, const TargetLowering &TLI, SDValue Op, SelectionDAG *DAG)
If there are multiple different constraints that we could pick for this operand (e.g.
unsigned getScalarValueSizeInBits() const
SubConstraintInfoVector multipleAlternatives
multipleAlternatives - If there are multiple alternative constraints, this array will contain them...
Definition: InlineAsm.h:155
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
Definition: APInt.cpp:654
uint16_t getParamAlignment(uint16_t i) const
Extract the alignment for a call or parameter (0=unknown).
Definition: CallSite.h:383
uint32_t Offset
bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively, each computing an n/2-bit part of the result.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
bool isPositionIndependent() const
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1255
bool CombineTo(SDValue O, SDValue N)
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
unsigned getOpcode() const
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
Definition: ISDOpcodes.h:895
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:281
bool SimplifyDemandedBits(SDNode *User, unsigned OpIdx, const APInt &Demanded, DAGCombinerInfo &DCI)
Helper for SimplifyDemandedBits that can simplify an operation with multiple uses.
bool isVolatile() const
const SDValue & getValue() const
bool isConstFalseVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, APInt &KnownZero, APInt &KnownOne, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue BuildSDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector< SDNode * > *Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:485
EVT - Extended Value Type.
Definition: ValueTypes.h:31
bool isIntN(unsigned N, int64_t x)
isIntN - Checks if an signed integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:366
Type * getType() const
getType - Return the type of the instruction that generated this call site
Definition: CallSite.h:258
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
Definition: MathExtras.h:619
std::vector< ArgListEntry > ArgListTy
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This structure contains all information that is necessary for lowering calls.
bool shouldAssumeDSOLocal(const Module &M, const GlobalValue *GV) const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:213
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
This class contains a discriminated union of information about pointers in memory operands...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::string ConstraintCode
This contains the actual string for the code, like "m".
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
const MachinePointerInfo & getPointerInfo() const
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const
Get the CondCode that's to be used to test the result of the comparison libcall against zero...
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
Definition: ValueTypes.h:200
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.cpp:703
std::vector< AsmOperandInfo > AsmOperandInfoVector
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:234
std::vector< std::string > ConstraintCodeVector
Definition: InlineAsm.h:98
uint64_t * Vals
static void Merge(const std::string &Input, const std::vector< std::string > Result, size_t NumNewFeatures)
void setNoUnsignedWrap(bool b)
SmallVectorImpl< T >::const_pointer c_str(SmallVectorImpl< T > &str)
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
Definition: APInt.cpp:649
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:285
double Log2(double Value)
Log2 - This function returns the log base 2 of the specified value.
Definition: MathExtras.h:502
ValTy * getArgument(unsigned ArgNo) const
Definition: CallSite.h:178
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:166
APInt m
magic number
Definition: APInt.h:1717
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:639
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
CCValAssign - Represent assignment of one arg/retval to a location.
GlobalVariable * getNamedGlobal(StringRef Name)
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:357
const SDValue & getChain() const
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
bool hasMatchingInput() const
hasMatchingInput - Return true if this is an output constraint that has a matching input constraint...
Definition: InlineAsm.h:134
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
Definition: APInt.h:337
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:438
CallLoweringInfo & setSExtResult(bool Value=true)
virtual const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase, but as an MCExpr.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset...
bool hasNoSignedWrap() const
Represents one node in the SelectionDAG.
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:513
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
AttributeSet getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:176
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MCSymbol * getJTISymbol(unsigned JTI, MCContext &Ctx, bool isLinkerPrivate=false) const
getJTISymbol - Return the MCSymbol for the specified non-empty jump table.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Definition: APInt.h:77
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:354
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:195
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:400
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:403
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:130
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:426
virtual bool isIntDivCheap(EVT VT, AttributeSet Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts, adds, and multiplies for this target.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1942
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition: APInt.h:503
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
bool isAllOnesValue() const
Determine if all bits are set.
Definition: APInt.h:342
These are IR-level optimization flags that may be propagated to SDNodes.
Magic data for optimising signed division by a constant.
Definition: APInt.h:1710
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
unsigned s
shift amount
Definition: APInt.h:1719
bool isUndef() const
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
APInt m
magic number
Definition: APInt.h:1711
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:418
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:536
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:665
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
ArrayRef< SDUse > ops() const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
bool paramHasAttr(unsigned i, Attribute::AttrKind Kind) const
Return true if the call or the callee has the given attribute.
Definition: CallSite.h:359
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:349
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
static volatile int Zero
static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT)
Return an integer indicating how general CT is.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &Demanded, const SDLoc &dl)
Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
SDValue getGLOBAL_OFFSET_TABLE(EVT VT)
Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
Definition: SelectionDAG.h:742
const unsigned Kind
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:183
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:118
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:441
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:107
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
TargetLowering(const TargetLowering &)=delete
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:537
LLVM Value Representation.
Definition: Value.h:71
static SDNodeIterator end(const SDNode *N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:830
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:239
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
iterator end() const
Definition: StringRef.h:105
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:533
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:331
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:125
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
Definition: APInt.h:1343
Primary interface to the complete machine description for the target machine.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:980
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:377
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
EK_GPRel32BlockAddress - Each entry is an address of block, encoded with a relocation as gp-relative...
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml","ocaml 3.10-compatible collector")
MVT ConstraintVT
The ValueType for the operand value.
Conversion operators.
Definition: ISDOpcodes.h:397
BooleanContent
Enum that describes how the target represents true/false values.
int * Ptr
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:381
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:406
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
unsigned getAlignment() const
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
const TargetRegisterClass *const * regclass_iterator
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, ImmutableCallSite CS) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:226
const SDNodeFlags * getFlags() const
This could be defined as a virtual function and implemented more simply and directly, but it is not to avoid creating a vtable for this class.
static ConstraintInfoVector ParseConstraints(StringRef ConstraintString)
ParseConstraints - Split up the constraint string into the specific constraints and their prefixes...
Definition: InlineAsm.cpp:211
bool a
add indicator
Definition: APInt.h:1718
This file describes how to lower LLVM code to machine code.
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
unsigned getLiveInPhysReg(unsigned VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:529
void selectAlternative(unsigned index)
selectAlternative - Point this constraint to the alternative constraint indicated by the index...
Definition: InlineAsm.cpp:200
uint64_t getZExtValue() const
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:326
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:248
This class is used to represent ISD::LOAD nodes.