LLVM  7.0.0svn
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the TargetLowering class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/BitVector.h"
16 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/GlobalVariable.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCExpr.h"
32 #include "llvm/Support/KnownBits.h"
36 #include <cctype>
37 using namespace llvm;
38 
39 /// NOTE: The TargetMachine owns TLOF.
41  : TargetLoweringBase(tm) {}
42 
43 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
44  return nullptr;
45 }
46 
49 }
50 
51 /// Check whether a given call node is in tail position within its function. If
52 /// so, it sets Chain to the input chain of the tail call.
54  SDValue &Chain) const {
55  const Function &F = DAG.getMachineFunction().getFunction();
56 
57  // Conservatively require the attributes of the call to match those of
58  // the return. Ignore noalias because it doesn't affect the call sequence.
59  AttributeList CallerAttrs = F.getAttributes();
60  if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
61  .removeAttribute(Attribute::NoAlias)
62  .hasAttributes())
63  return false;
64 
65  // It's not safe to eliminate the sign / zero extension of the return value.
66  if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
67  CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
68  return false;
69 
70  // Check if the only use is a function return node.
71  return isUsedByReturnOnly(Node, Chain);
72 }
73 
75  const uint32_t *CallerPreservedMask,
76  const SmallVectorImpl<CCValAssign> &ArgLocs,
77  const SmallVectorImpl<SDValue> &OutVals) const {
78  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
79  const CCValAssign &ArgLoc = ArgLocs[I];
80  if (!ArgLoc.isRegLoc())
81  continue;
82  unsigned Reg = ArgLoc.getLocReg();
83  // Only look at callee saved registers.
84  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
85  continue;
86  // Check that we pass the value used for the caller.
87  // (We look for a CopyFromReg reading a virtual register that is used
88  // for the function live-in value of register Reg)
89  SDValue Value = OutVals[I];
90  if (Value->getOpcode() != ISD::CopyFromReg)
91  return false;
92  unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
93  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
94  return false;
95  }
96  return true;
97 }
98 
99 /// Set CallLoweringInfo attribute flags based on a call instruction
100 /// and called function attributes.
102  unsigned ArgIdx) {
103  IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt);
104  IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt);
105  IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg);
106  IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet);
107  IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest);
108  IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal);
109  IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca);
110  IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned);
111  IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
112  IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError);
113  Alignment = CS->getParamAlignment(ArgIdx);
114 }
115 
116 /// Generate a libcall taking the given operands as arguments and returning a
117 /// result of type RetVT.
118 std::pair<SDValue, SDValue>
120  ArrayRef<SDValue> Ops, bool isSigned,
121  const SDLoc &dl, bool doesNotReturn,
122  bool isReturnValueUsed) const {
124  Args.reserve(Ops.size());
125 
127  for (SDValue Op : Ops) {
128  Entry.Node = Op;
129  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
130  Entry.IsSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
131  Entry.IsZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
132  Args.push_back(Entry);
133  }
134 
135  if (LC == RTLIB::UNKNOWN_LIBCALL)
136  report_fatal_error("Unsupported library call operation!");
138  getPointerTy(DAG.getDataLayout()));
139 
140  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
142  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, isSigned);
143  CLI.setDebugLoc(dl)
144  .setChain(DAG.getEntryNode())
145  .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
146  .setNoReturn(doesNotReturn)
147  .setDiscardResult(!isReturnValueUsed)
148  .setSExtResult(signExtend)
149  .setZExtResult(!signExtend);
150  return LowerCallTo(CLI);
151 }
152 
153 /// Soften the operands of a comparison. This code is shared among BR_CC,
154 /// SELECT_CC, and SETCC handlers.
156  SDValue &NewLHS, SDValue &NewRHS,
157  ISD::CondCode &CCCode,
158  const SDLoc &dl) const {
159  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
160  && "Unsupported setcc type!");
161 
162  // Expand into one or more soft-fp libcall(s).
163  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
164  bool ShouldInvertCC = false;
165  switch (CCCode) {
166  case ISD::SETEQ:
167  case ISD::SETOEQ:
168  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
169  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
170  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
171  break;
172  case ISD::SETNE:
173  case ISD::SETUNE:
174  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
175  (VT == MVT::f64) ? RTLIB::UNE_F64 :
176  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
177  break;
178  case ISD::SETGE:
179  case ISD::SETOGE:
180  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
181  (VT == MVT::f64) ? RTLIB::OGE_F64 :
182  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
183  break;
184  case ISD::SETLT:
185  case ISD::SETOLT:
186  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
187  (VT == MVT::f64) ? RTLIB::OLT_F64 :
188  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
189  break;
190  case ISD::SETLE:
191  case ISD::SETOLE:
192  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
193  (VT == MVT::f64) ? RTLIB::OLE_F64 :
194  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
195  break;
196  case ISD::SETGT:
197  case ISD::SETOGT:
198  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
199  (VT == MVT::f64) ? RTLIB::OGT_F64 :
200  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
201  break;
202  case ISD::SETUO:
203  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
204  (VT == MVT::f64) ? RTLIB::UO_F64 :
205  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
206  break;
207  case ISD::SETO:
208  LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
209  (VT == MVT::f64) ? RTLIB::O_F64 :
210  (VT == MVT::f128) ? RTLIB::O_F128 : RTLIB::O_PPCF128;
211  break;
212  case ISD::SETONE:
213  // SETONE = SETOLT | SETOGT
214  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
215  (VT == MVT::f64) ? RTLIB::OLT_F64 :
216  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
217  LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
218  (VT == MVT::f64) ? RTLIB::OGT_F64 :
219  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
220  break;
221  case ISD::SETUEQ:
222  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
223  (VT == MVT::f64) ? RTLIB::UO_F64 :
224  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
225  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
226  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
227  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
228  break;
229  default:
230  // Invert CC for unordered comparisons
231  ShouldInvertCC = true;
232  switch (CCCode) {
233  case ISD::SETULT:
234  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
235  (VT == MVT::f64) ? RTLIB::OGE_F64 :
236  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
237  break;
238  case ISD::SETULE:
239  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
240  (VT == MVT::f64) ? RTLIB::OGT_F64 :
241  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
242  break;
243  case ISD::SETUGT:
244  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
245  (VT == MVT::f64) ? RTLIB::OLE_F64 :
246  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
247  break;
248  case ISD::SETUGE:
249  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
250  (VT == MVT::f64) ? RTLIB::OLT_F64 :
251  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
252  break;
253  default: llvm_unreachable("Do not know how to soften this setcc!");
254  }
255  }
256 
257  // Use the target specific return value for comparions lib calls.
258  EVT RetVT = getCmpLibcallReturnType();
259  SDValue Ops[2] = {NewLHS, NewRHS};
260  NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, false /*sign irrelevant*/,
261  dl).first;
262  NewRHS = DAG.getConstant(0, dl, RetVT);
263 
264  CCCode = getCmpLibcallCC(LC1);
265  if (ShouldInvertCC)
266  CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);
267 
268  if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
269  SDValue Tmp = DAG.getNode(
270  ISD::SETCC, dl,
271  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
272  NewLHS, NewRHS, DAG.getCondCode(CCCode));
273  NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, false/*sign irrelevant*/,
274  dl).first;
275  NewLHS = DAG.getNode(
276  ISD::SETCC, dl,
277  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
278  NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
279  NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
280  NewRHS = SDValue();
281  }
282 }
283 
284 /// Return the entry encoding for a jump table in the current function. The
285 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
287  // In non-pic modes, just use the address of a block.
288  if (!isPositionIndependent())
290 
291  // In PIC mode, if the target supports a GPRel32 directive, use it.
292  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
294 
295  // Otherwise, use a label difference.
297 }
298 
300  SelectionDAG &DAG) const {
301  // If our PIC model is GP relative, use the global offset table as the base.
302  unsigned JTEncoding = getJumpTableEncoding();
303 
304  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
307 
308  return Table;
309 }
310 
311 /// This returns the relocation base for the given PIC jumptable, the same as
312 /// getPICJumpTableRelocBase, but as an MCExpr.
313 const MCExpr *
315  unsigned JTI,MCContext &Ctx) const{
316  // The normal PIC reloc base is the label at the start of the jump table.
317  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
318 }
319 
320 bool
322  const TargetMachine &TM = getTargetMachine();
323  const GlobalValue *GV = GA->getGlobal();
324 
325  // If the address is not even local to this DSO we will have to load it from
326  // a got and then add the offset.
327  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
328  return false;
329 
330  // If the code is position independent we will have to add a base register.
331  if (isPositionIndependent())
332  return false;
333 
334  // Otherwise we can do it.
335  return true;
336 }
337 
338 //===----------------------------------------------------------------------===//
339 // Optimization Methods
340 //===----------------------------------------------------------------------===//
341 
342 /// If the specified instruction has a constant integer operand and there are
343 /// bits set in that constant that are not demanded, then clear those bits and
344 /// return true.
346  TargetLoweringOpt &TLO) const {
347  SelectionDAG &DAG = TLO.DAG;
348  SDLoc DL(Op);
349  unsigned Opcode = Op.getOpcode();
350 
351  // Do target-specific constant optimization.
352  if (targetShrinkDemandedConstant(Op, Demanded, TLO))
353  return TLO.New.getNode();
354 
355  // FIXME: ISD::SELECT, ISD::SELECT_CC
356  switch (Opcode) {
357  default:
358  break;
359  case ISD::XOR:
360  case ISD::AND:
361  case ISD::OR: {
362  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
363  if (!Op1C)
364  return false;
365 
366  // If this is a 'not' op, don't touch it because that's a canonical form.
367  const APInt &C = Op1C->getAPIntValue();
368  if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
369  return false;
370 
371  if (!C.isSubsetOf(Demanded)) {
372  EVT VT = Op.getValueType();
373  SDValue NewC = DAG.getConstant(Demanded & C, DL, VT);
374  SDValue NewOp = DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
375  return TLO.CombineTo(Op, NewOp);
376  }
377 
378  break;
379  }
380  }
381 
382  return false;
383 }
384 
385 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
386 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
387 /// generalized for targets with other types of implicit widening casts.
389  const APInt &Demanded,
390  TargetLoweringOpt &TLO) const {
391  assert(Op.getNumOperands() == 2 &&
392  "ShrinkDemandedOp only supports binary operators!");
393  assert(Op.getNode()->getNumValues() == 1 &&
394  "ShrinkDemandedOp only supports nodes with one result!");
395 
396  SelectionDAG &DAG = TLO.DAG;
397  SDLoc dl(Op);
398 
399  // Early return, as this function cannot handle vector types.
400  if (Op.getValueType().isVector())
401  return false;
402 
403  // Don't do this if the node has another user, which may require the
404  // full value.
405  if (!Op.getNode()->hasOneUse())
406  return false;
407 
408  // Search for the smallest integer type with free casts to and from
409  // Op's type. For expedience, just check power-of-2 integer types.
410  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
411  unsigned DemandedSize = Demanded.getActiveBits();
412  unsigned SmallVTBits = DemandedSize;
413  if (!isPowerOf2_32(SmallVTBits))
414  SmallVTBits = NextPowerOf2(SmallVTBits);
415  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
416  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
417  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
418  TLI.isZExtFree(SmallVT, Op.getValueType())) {
419  // We found a type with free casts.
420  SDValue X = DAG.getNode(
421  Op.getOpcode(), dl, SmallVT,
422  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
423  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
424  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
425  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
426  return TLO.CombineTo(Op, Z);
427  }
428  }
429  return false;
430 }
431 
432 bool
434  const APInt &Demanded,
435  DAGCombinerInfo &DCI,
436  TargetLoweringOpt &TLO) const {
437  SDValue Op = User->getOperand(OpIdx);
438  KnownBits Known;
439 
440  if (!SimplifyDemandedBits(Op, Demanded, Known, TLO, 0, true))
441  return false;
442 
443 
444  // Old will not always be the same as Op. For example:
445  //
446  // Demanded = 0xffffff
447  // Op = i64 truncate (i32 and x, 0xffffff)
448  // In this case simplify demand bits will want to replace the 'and' node
449  // with the value 'x', which will give us:
450  // Old = i32 and x, 0xffffff
451  // New = x
452  if (TLO.Old.hasOneUse()) {
453  // For the one use case, we just commit the change.
454  DCI.CommitTargetLoweringOpt(TLO);
455  return true;
456  }
457 
458  // If Old has more than one use then it must be Op, because the
459  // AssumeSingleUse flag is not propogated to recursive calls of
460  // SimplifyDemanded bits, so the only node with multiple use that
461  // it will attempt to combine will be Op.
462  assert(TLO.Old == Op);
463 
465  for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
466  if (i == OpIdx) {
467  NewOps.push_back(TLO.New);
468  continue;
469  }
470  NewOps.push_back(User->getOperand(i));
471  }
472  User = TLO.DAG.UpdateNodeOperands(User, NewOps);
473  // Op has less users now, so we may be able to perform additional combines
474  // with it.
475  DCI.AddToWorklist(Op.getNode());
476  // User's operands have been updated, so we may be able to do new combines
477  // with it.
478  DCI.AddToWorklist(User);
479  return true;
480 }
481 
483  DAGCombinerInfo &DCI) const {
484 
485  SelectionDAG &DAG = DCI.DAG;
486  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
487  !DCI.isBeforeLegalizeOps());
488  KnownBits Known;
489 
490  bool Simplified = SimplifyDemandedBits(Op, DemandedMask, Known, TLO);
491  if (Simplified)
492  DCI.CommitTargetLoweringOpt(TLO);
493  return Simplified;
494 }
495 
496 /// Look at Op. At this point, we know that only the DemandedMask bits of the
497 /// result of Op are ever used downstream. If we can use this information to
498 /// simplify Op, create a new simplified DAG node and return true, returning the
499 /// original and new nodes in Old and New. Otherwise, analyze the expression and
500 /// return a mask of Known bits for the expression (used to simplify the
501 /// caller). The Known bits may only be accurate for those bits in the
502 /// DemandedMask.
504  const APInt &DemandedMask,
505  KnownBits &Known,
506  TargetLoweringOpt &TLO,
507  unsigned Depth,
508  bool AssumeSingleUse) const {
509  unsigned BitWidth = DemandedMask.getBitWidth();
510  assert(Op.getScalarValueSizeInBits() == BitWidth &&
511  "Mask size mismatches value type size!");
512  APInt NewMask = DemandedMask;
513  SDLoc dl(Op);
514  auto &DL = TLO.DAG.getDataLayout();
515 
516  // Don't know anything.
517  Known = KnownBits(BitWidth);
518 
519  if (Op.getOpcode() == ISD::Constant) {
520  // We know all of the bits for a constant!
521  Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
522  Known.Zero = ~Known.One;
523  return false;
524  }
525 
526  // Other users may use these bits.
527  EVT VT = Op.getValueType();
528  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
529  if (Depth != 0) {
530  // If not at the root, Just compute the Known bits to
531  // simplify things downstream.
532  TLO.DAG.computeKnownBits(Op, Known, Depth);
533  return false;
534  }
535  // If this is the root being simplified, allow it to have multiple uses,
536  // just set the NewMask to all bits.
537  NewMask = APInt::getAllOnesValue(BitWidth);
538  } else if (DemandedMask == 0) {
539  // Not demanding any bits from Op.
540  if (!Op.isUndef())
541  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
542  return false;
543  } else if (Depth == 6) { // Limit search depth.
544  return false;
545  }
546 
547  KnownBits Known2, KnownOut;
548  switch (Op.getOpcode()) {
549  case ISD::BUILD_VECTOR:
550  // Collect the known bits that are shared by every constant vector element.
551  Known.Zero.setAllBits(); Known.One.setAllBits();
552  for (SDValue SrcOp : Op->ops()) {
553  if (!isa<ConstantSDNode>(SrcOp)) {
554  // We can only handle all constant values - bail out with no known bits.
555  Known = KnownBits(BitWidth);
556  return false;
557  }
558  Known2.One = cast<ConstantSDNode>(SrcOp)->getAPIntValue();
559  Known2.Zero = ~Known2.One;
560 
561  // BUILD_VECTOR can implicitly truncate sources, we must handle this.
562  if (Known2.One.getBitWidth() != BitWidth) {
563  assert(Known2.getBitWidth() > BitWidth &&
564  "Expected BUILD_VECTOR implicit truncation");
565  Known2 = Known2.trunc(BitWidth);
566  }
567 
568  // Known bits are the values that are shared by every element.
569  // TODO: support per-element known bits.
570  Known.One &= Known2.One;
571  Known.Zero &= Known2.Zero;
572  }
573  return false; // Don't fall through, will infinitely loop.
574  case ISD::AND:
575  // If the RHS is a constant, check to see if the LHS would be zero without
576  // using the bits from the RHS. Below, we use knowledge about the RHS to
577  // simplify the LHS, here we're using information from the LHS to simplify
578  // the RHS.
579  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op.getOperand(1))) {
580  SDValue Op0 = Op.getOperand(0);
581  KnownBits LHSKnown;
582  // Do not increment Depth here; that can cause an infinite loop.
583  TLO.DAG.computeKnownBits(Op0, LHSKnown, Depth);
584  // If the LHS already has zeros where RHSC does, this 'and' is dead.
585  if ((LHSKnown.Zero & NewMask) == (~RHSC->getAPIntValue() & NewMask))
586  return TLO.CombineTo(Op, Op0);
587 
588  // If any of the set bits in the RHS are known zero on the LHS, shrink
589  // the constant.
590  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & NewMask, TLO))
591  return true;
592 
593  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
594  // constant, but if this 'and' is only clearing bits that were just set by
595  // the xor, then this 'and' can be eliminated by shrinking the mask of
596  // the xor. For example, for a 32-bit X:
597  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
598  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
599  LHSKnown.One == ~RHSC->getAPIntValue()) {
600  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0),
601  Op.getOperand(1));
602  return TLO.CombineTo(Op, Xor);
603  }
604  }
605 
606  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known, TLO, Depth+1))
607  return true;
608  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
609  if (SimplifyDemandedBits(Op.getOperand(0), ~Known.Zero & NewMask,
610  Known2, TLO, Depth+1))
611  return true;
612  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
613 
614  // If all of the demanded bits are known one on one side, return the other.
615  // These bits cannot contribute to the result of the 'and'.
616  if (NewMask.isSubsetOf(Known2.Zero | Known.One))
617  return TLO.CombineTo(Op, Op.getOperand(0));
618  if (NewMask.isSubsetOf(Known.Zero | Known2.One))
619  return TLO.CombineTo(Op, Op.getOperand(1));
620  // If all of the demanded bits in the inputs are known zeros, return zero.
621  if (NewMask.isSubsetOf(Known.Zero | Known2.Zero))
622  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
623  // If the RHS is a constant, see if we can simplify it.
624  if (ShrinkDemandedConstant(Op, ~Known2.Zero & NewMask, TLO))
625  return true;
626  // If the operation can be done in a smaller type, do so.
627  if (ShrinkDemandedOp(Op, BitWidth, NewMask, TLO))
628  return true;
629 
630  // Output known-1 bits are only known if set in both the LHS & RHS.
631  Known.One &= Known2.One;
632  // Output known-0 are known to be clear if zero in either the LHS | RHS.
633  Known.Zero |= Known2.Zero;
634  break;
635  case ISD::OR:
636  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known, TLO, Depth+1))
637  return true;
638  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
639  if (SimplifyDemandedBits(Op.getOperand(0), ~Known.One & NewMask,
640  Known2, TLO, Depth+1))
641  return true;
642  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
643 
644  // If all of the demanded bits are known zero on one side, return the other.
645  // These bits cannot contribute to the result of the 'or'.
646  if (NewMask.isSubsetOf(Known2.One | Known.Zero))
647  return TLO.CombineTo(Op, Op.getOperand(0));
648  if (NewMask.isSubsetOf(Known.One | Known2.Zero))
649  return TLO.CombineTo(Op, Op.getOperand(1));
650  // If the RHS is a constant, see if we can simplify it.
651  if (ShrinkDemandedConstant(Op, NewMask, TLO))
652  return true;
653  // If the operation can be done in a smaller type, do so.
654  if (ShrinkDemandedOp(Op, BitWidth, NewMask, TLO))
655  return true;
656 
657  // Output known-0 bits are only known if clear in both the LHS & RHS.
658  Known.Zero &= Known2.Zero;
659  // Output known-1 are known to be set if set in either the LHS | RHS.
660  Known.One |= Known2.One;
661  break;
662  case ISD::XOR: {
663  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known, TLO, Depth+1))
664  return true;
665  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
666  if (SimplifyDemandedBits(Op.getOperand(0), NewMask, Known2, TLO, Depth+1))
667  return true;
668  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
669 
670  // If all of the demanded bits are known zero on one side, return the other.
671  // These bits cannot contribute to the result of the 'xor'.
672  if (NewMask.isSubsetOf(Known.Zero))
673  return TLO.CombineTo(Op, Op.getOperand(0));
674  if (NewMask.isSubsetOf(Known2.Zero))
675  return TLO.CombineTo(Op, Op.getOperand(1));
676  // If the operation can be done in a smaller type, do so.
677  if (ShrinkDemandedOp(Op, BitWidth, NewMask, TLO))
678  return true;
679 
680  // If all of the unknown bits are known to be zero on one side or the other
681  // (but not both) turn this into an *inclusive* or.
682  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
683  if ((NewMask & ~Known.Zero & ~Known2.Zero) == 0)
684  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT,
685  Op.getOperand(0),
686  Op.getOperand(1)));
687 
688  // Output known-0 bits are known if clear or set in both the LHS & RHS.
689  KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
690  // Output known-1 are known to be set if set in only one of the LHS, RHS.
691  KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
692 
693  // If all of the demanded bits on one side are known, and all of the set
694  // bits on that side are also known to be set on the other side, turn this
695  // into an AND, as we know the bits will be cleared.
696  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
697  // NB: it is okay if more bits are known than are requested
698  if (NewMask.isSubsetOf(Known.Zero|Known.One)) { // all known on one side
699  if (Known.One == Known2.One) { // set bits are the same on both sides
700  SDValue ANDC = TLO.DAG.getConstant(~Known.One & NewMask, dl, VT);
701  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT,
702  Op.getOperand(0), ANDC));
703  }
704  }
705 
706  // If the RHS is a constant, see if we can change it. Don't alter a -1
707  // constant because that's a 'not' op, and that is better for combining and
708  // codegen.
710  if (C && !C->isAllOnesValue()) {
711  if (NewMask.isSubsetOf(C->getAPIntValue())) {
712  // We're flipping all demanded bits. Flip the undemanded bits too.
713  SDValue New = TLO.DAG.getNOT(dl, Op.getOperand(0), VT);
714  return TLO.CombineTo(Op, New);
715  }
716  // If we can't turn this into a 'not', try to shrink the constant.
717  if (ShrinkDemandedConstant(Op, NewMask, TLO))
718  return true;
719  }
720 
721  Known = std::move(KnownOut);
722  break;
723  }
724  case ISD::SELECT:
725  if (SimplifyDemandedBits(Op.getOperand(2), NewMask, Known, TLO, Depth+1))
726  return true;
727  if (SimplifyDemandedBits(Op.getOperand(1), NewMask, Known2, TLO, Depth+1))
728  return true;
729  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
730  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
731 
732  // If the operands are constants, see if we can simplify them.
733  if (ShrinkDemandedConstant(Op, NewMask, TLO))
734  return true;
735 
736  // Only known if known in both the LHS and RHS.
737  Known.One &= Known2.One;
738  Known.Zero &= Known2.Zero;
739  break;
740  case ISD::SELECT_CC:
741  if (SimplifyDemandedBits(Op.getOperand(3), NewMask, Known, TLO, Depth+1))
742  return true;
743  if (SimplifyDemandedBits(Op.getOperand(2), NewMask, Known2, TLO, Depth+1))
744  return true;
745  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
746  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
747 
748  // If the operands are constants, see if we can simplify them.
749  if (ShrinkDemandedConstant(Op, NewMask, TLO))
750  return true;
751 
752  // Only known if known in both the LHS and RHS.
753  Known.One &= Known2.One;
754  Known.Zero &= Known2.Zero;
755  break;
756  case ISD::SETCC: {
757  SDValue Op0 = Op.getOperand(0);
758  SDValue Op1 = Op.getOperand(1);
759  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
760  // If (1) we only need the sign-bit, (2) the setcc operands are the same
761  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
762  // -1, we may be able to bypass the setcc.
763  if (NewMask.isSignMask() && Op0.getScalarValueSizeInBits() == BitWidth &&
764  getBooleanContents(VT) ==
765  BooleanContent::ZeroOrNegativeOneBooleanContent) {
766  // If we're testing X < 0, then this compare isn't needed - just use X!
767  // FIXME: We're limiting to integer types here, but this should also work
768  // if we don't care about FP signed-zero. The use of SETLT with FP means
769  // that we don't care about NaNs.
770  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
772  return TLO.CombineTo(Op, Op0);
773 
774  // TODO: Should we check for other forms of sign-bit comparisons?
775  // Examples: X <= -1, X >= 0
776  }
777  if (getBooleanContents(Op0.getValueType()) ==
779  BitWidth > 1)
780  Known.Zero.setBitsFrom(1);
781  break;
782  }
783  case ISD::SHL:
784  if (ConstantSDNode *SA = isConstOrConstSplat(Op.getOperand(1))) {
785  SDValue InOp = Op.getOperand(0);
786 
787  // If the shift count is an invalid immediate, don't do anything.
788  if (SA->getAPIntValue().uge(BitWidth))
789  break;
790 
791  unsigned ShAmt = SA->getZExtValue();
792 
793  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
794  // single shift. We can do this if the bottom bits (which are shifted
795  // out) are never demanded.
796  if (InOp.getOpcode() == ISD::SRL) {
797  if (ConstantSDNode *SA2 = isConstOrConstSplat(InOp.getOperand(1))) {
798  if (ShAmt && (NewMask & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) {
799  if (SA2->getAPIntValue().ult(BitWidth)) {
800  unsigned C1 = SA2->getZExtValue();
801  unsigned Opc = ISD::SHL;
802  int Diff = ShAmt-C1;
803  if (Diff < 0) {
804  Diff = -Diff;
805  Opc = ISD::SRL;
806  }
807 
808  SDValue NewSA =
809  TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
810  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
811  InOp.getOperand(0),
812  NewSA));
813  }
814  }
815  }
816  }
817 
818  if (SimplifyDemandedBits(InOp, NewMask.lshr(ShAmt), Known, TLO, Depth+1))
819  return true;
820 
821  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
822  // are not demanded. This will likely allow the anyext to be folded away.
823  if (InOp.getNode()->getOpcode() == ISD::ANY_EXTEND) {
824  SDValue InnerOp = InOp.getOperand(0);
825  EVT InnerVT = InnerOp.getValueType();
826  unsigned InnerBits = InnerVT.getScalarSizeInBits();
827  if (ShAmt < InnerBits && NewMask.getActiveBits() <= InnerBits &&
828  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
829  EVT ShTy = getShiftAmountTy(InnerVT, DL);
830  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
831  ShTy = InnerVT;
832  SDValue NarrowShl =
833  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
834  TLO.DAG.getConstant(ShAmt, dl, ShTy));
835  return
836  TLO.CombineTo(Op,
837  TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
838  }
839  // Repeat the SHL optimization above in cases where an extension
840  // intervenes: (shl (anyext (shr x, c1)), c2) to
841  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
842  // aren't demanded (as above) and that the shifted upper c1 bits of
843  // x aren't demanded.
844  if (InOp.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
845  InnerOp.hasOneUse()) {
846  if (ConstantSDNode *SA2 = isConstOrConstSplat(InnerOp.getOperand(1))) {
847  unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
848  if (InnerShAmt < ShAmt &&
849  InnerShAmt < InnerBits &&
850  NewMask.getActiveBits() <= (InnerBits - InnerShAmt + ShAmt) &&
851  NewMask.countTrailingZeros() >= ShAmt) {
852  SDValue NewSA =
853  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
854  Op.getOperand(1).getValueType());
855  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
856  InnerOp.getOperand(0));
857  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT,
858  NewExt, NewSA));
859  }
860  }
861  }
862  }
863 
864  Known.Zero <<= ShAmt;
865  Known.One <<= ShAmt;
866  // low bits known zero.
867  Known.Zero.setLowBits(ShAmt);
868  }
869  break;
870  case ISD::SRL:
871  if (ConstantSDNode *SA = isConstOrConstSplat(Op.getOperand(1))) {
872  SDValue InOp = Op.getOperand(0);
873 
874  // If the shift count is an invalid immediate, don't do anything.
875  if (SA->getAPIntValue().uge(BitWidth))
876  break;
877 
878  unsigned ShAmt = SA->getZExtValue();
879  APInt InDemandedMask = (NewMask << ShAmt);
880 
881  // If the shift is exact, then it does demand the low bits (and knows that
882  // they are zero).
883  if (Op->getFlags().hasExact())
884  InDemandedMask.setLowBits(ShAmt);
885 
886  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
887  // single shift. We can do this if the top bits (which are shifted out)
888  // are never demanded.
889  if (InOp.getOpcode() == ISD::SHL) {
890  if (ConstantSDNode *SA2 = isConstOrConstSplat(InOp.getOperand(1))) {
891  if (ShAmt &&
892  (NewMask & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) {
893  if (SA2->getAPIntValue().ult(BitWidth)) {
894  unsigned C1 = SA2->getZExtValue();
895  unsigned Opc = ISD::SRL;
896  int Diff = ShAmt-C1;
897  if (Diff < 0) {
898  Diff = -Diff;
899  Opc = ISD::SHL;
900  }
901 
902  SDValue NewSA =
903  TLO.DAG.getConstant(Diff, dl, Op.getOperand(1).getValueType());
904  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT,
905  InOp.getOperand(0),
906  NewSA));
907  }
908  }
909  }
910  }
911 
912  // Compute the new bits that are at the top now.
913  if (SimplifyDemandedBits(InOp, InDemandedMask, Known, TLO, Depth+1))
914  return true;
915  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
916  Known.Zero.lshrInPlace(ShAmt);
917  Known.One.lshrInPlace(ShAmt);
918 
919  Known.Zero.setHighBits(ShAmt); // High bits known zero.
920  }
921  break;
922  case ISD::SRA:
923  // If this is an arithmetic shift right and only the low-bit is set, we can
924  // always convert this into a logical shr, even if the shift amount is
925  // variable. The low bit of the shift cannot be an input sign bit unless
926  // the shift amount is >= the size of the datatype, which is undefined.
927  if (NewMask.isOneValue())
928  return TLO.CombineTo(Op,
929  TLO.DAG.getNode(ISD::SRL, dl, VT, Op.getOperand(0),
930  Op.getOperand(1)));
931 
932  if (ConstantSDNode *SA = isConstOrConstSplat(Op.getOperand(1))) {
933  // If the shift count is an invalid immediate, don't do anything.
934  if (SA->getAPIntValue().uge(BitWidth))
935  break;
936 
937  unsigned ShAmt = SA->getZExtValue();
938  APInt InDemandedMask = (NewMask << ShAmt);
939 
940  // If the shift is exact, then it does demand the low bits (and knows that
941  // they are zero).
942  if (Op->getFlags().hasExact())
943  InDemandedMask.setLowBits(ShAmt);
944 
945  // If any of the demanded bits are produced by the sign extension, we also
946  // demand the input sign bit.
947  if (NewMask.countLeadingZeros() < ShAmt)
948  InDemandedMask.setSignBit();
949 
950  if (SimplifyDemandedBits(Op.getOperand(0), InDemandedMask, Known, TLO,
951  Depth+1))
952  return true;
953  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
954  Known.Zero.lshrInPlace(ShAmt);
955  Known.One.lshrInPlace(ShAmt);
956 
957  // If the input sign bit is known to be zero, or if none of the top bits
958  // are demanded, turn this into an unsigned shift right.
959  if (Known.Zero[BitWidth - ShAmt - 1] ||
960  NewMask.countLeadingZeros() >= ShAmt) {
961  SDNodeFlags Flags;
962  Flags.setExact(Op->getFlags().hasExact());
963  return TLO.CombineTo(Op,
964  TLO.DAG.getNode(ISD::SRL, dl, VT, Op.getOperand(0),
965  Op.getOperand(1), Flags));
966  }
967 
968  int Log2 = NewMask.exactLogBase2();
969  if (Log2 >= 0) {
970  // The bit must come from the sign.
971  SDValue NewSA =
972  TLO.DAG.getConstant(BitWidth - 1 - Log2, dl,
973  Op.getOperand(1).getValueType());
974  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT,
975  Op.getOperand(0), NewSA));
976  }
977 
978  if (Known.One[BitWidth - ShAmt - 1])
979  // New bits are known one.
980  Known.One.setHighBits(ShAmt);
981  }
982  break;
983  case ISD::SIGN_EXTEND_INREG: {
984  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
985  unsigned ExVTBits = ExVT.getScalarSizeInBits();
986 
987  // If we only care about the highest bit, don't bother shifting right.
988  if (NewMask.isSignMask()) {
989  SDValue InOp = Op.getOperand(0);
990  bool AlreadySignExtended =
991  TLO.DAG.ComputeNumSignBits(InOp) >= BitWidth-ExVTBits+1;
992  // However if the input is already sign extended we expect the sign
993  // extension to be dropped altogether later and do not simplify.
994  if (!AlreadySignExtended) {
995  // Compute the correct shift amount type, which must be getShiftAmountTy
996  // for scalar types after legalization.
997  EVT ShiftAmtTy = VT;
998  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
999  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1000 
1001  SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ExVTBits, dl,
1002  ShiftAmtTy);
1003  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, InOp,
1004  ShiftAmt));
1005  }
1006  }
1007 
1008  // If none of the extended bits are demanded, eliminate the sextinreg.
1009  if (NewMask.getActiveBits() <= ExVTBits)
1010  return TLO.CombineTo(Op, Op.getOperand(0));
1011 
1012  APInt InputDemandedBits = NewMask.getLoBits(ExVTBits);
1013 
1014  // Since the sign extended bits are demanded, we know that the sign
1015  // bit is demanded.
1016  InputDemandedBits.setBit(ExVTBits - 1);
1017 
1018  if (SimplifyDemandedBits(Op.getOperand(0), InputDemandedBits,
1019  Known, TLO, Depth+1))
1020  return true;
1021  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1022 
1023  // If the sign bit of the input is known set or clear, then we know the
1024  // top bits of the result.
1025 
1026  // If the input sign bit is known zero, convert this into a zero extension.
1027  if (Known.Zero[ExVTBits - 1])
1028  return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(
1029  Op.getOperand(0), dl, ExVT.getScalarType()));
1030 
1031  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1032  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1033  Known.One.setBitsFrom(ExVTBits);
1034  Known.Zero &= Mask;
1035  } else { // Input sign bit unknown
1036  Known.Zero &= Mask;
1037  Known.One &= Mask;
1038  }
1039  break;
1040  }
1041  case ISD::BUILD_PAIR: {
1042  EVT HalfVT = Op.getOperand(0).getValueType();
1043  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1044 
1045  APInt MaskLo = NewMask.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1046  APInt MaskHi = NewMask.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1047 
1048  KnownBits KnownLo, KnownHi;
1049 
1050  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1051  return true;
1052 
1053  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1054  return true;
1055 
1056  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1057  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1058 
1059  Known.One = KnownLo.One.zext(BitWidth) |
1060  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1061  break;
1062  }
1063  case ISD::ZERO_EXTEND: {
1064  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1065 
1066  // If none of the top bits are demanded, convert this into an any_extend.
1067  if (NewMask.getActiveBits() <= OperandBitWidth)
1068  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1069  Op.getOperand(0)));
1070 
1071  APInt InMask = NewMask.trunc(OperandBitWidth);
1072  if (SimplifyDemandedBits(Op.getOperand(0), InMask, Known, TLO, Depth+1))
1073  return true;
1074  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1075  Known = Known.zext(BitWidth);
1076  Known.Zero.setBitsFrom(OperandBitWidth);
1077  break;
1078  }
1079  case ISD::SIGN_EXTEND: {
1080  unsigned InBits = Op.getOperand(0).getValueType().getScalarSizeInBits();
1081 
1082  // If none of the top bits are demanded, convert this into an any_extend.
1083  if (NewMask.getActiveBits() <= InBits)
1084  return TLO.CombineTo(Op,TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1085  Op.getOperand(0)));
1086 
1087  // Since some of the sign extended bits are demanded, we know that the sign
1088  // bit is demanded.
1089  APInt InDemandedBits = NewMask.trunc(InBits);
1090  InDemandedBits.setBit(InBits - 1);
1091 
1092  if (SimplifyDemandedBits(Op.getOperand(0), InDemandedBits, Known, TLO,
1093  Depth+1))
1094  return true;
1095  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1096  // If the sign bit is known one, the top bits match.
1097  Known = Known.sext(BitWidth);
1098 
1099  // If the sign bit is known zero, convert this to a zero extend.
1100  if (Known.isNonNegative())
1101  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT,
1102  Op.getOperand(0)));
1103  break;
1104  }
1105  case ISD::ANY_EXTEND: {
1106  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1107  APInt InMask = NewMask.trunc(OperandBitWidth);
1108  if (SimplifyDemandedBits(Op.getOperand(0), InMask, Known, TLO, Depth+1))
1109  return true;
1110  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1111  Known = Known.zext(BitWidth);
1112  break;
1113  }
1114  case ISD::TRUNCATE: {
1115  // Simplify the input, using demanded bit information, and compute the known
1116  // zero/one bits live out.
1117  unsigned OperandBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
1118  APInt TruncMask = NewMask.zext(OperandBitWidth);
1119  if (SimplifyDemandedBits(Op.getOperand(0), TruncMask, Known, TLO, Depth+1))
1120  return true;
1121  Known = Known.trunc(BitWidth);
1122 
1123  // If the input is only used by this truncate, see if we can shrink it based
1124  // on the known demanded bits.
1125  if (Op.getOperand(0).getNode()->hasOneUse()) {
1126  SDValue In = Op.getOperand(0);
1127  switch (In.getOpcode()) {
1128  default: break;
1129  case ISD::SRL:
1130  // Shrink SRL by a constant if none of the high bits shifted in are
1131  // demanded.
1132  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1133  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1134  // undesirable.
1135  break;
1137  if (!ShAmt)
1138  break;
1139  SDValue Shift = In.getOperand(1);
1140  if (TLO.LegalTypes()) {
1141  uint64_t ShVal = ShAmt->getZExtValue();
1142  Shift = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));
1143  }
1144 
1145  if (ShAmt->getZExtValue() < BitWidth) {
1146  APInt HighBits = APInt::getHighBitsSet(OperandBitWidth,
1147  OperandBitWidth - BitWidth);
1148  HighBits.lshrInPlace(ShAmt->getZExtValue());
1149  HighBits = HighBits.trunc(BitWidth);
1150 
1151  if (!(HighBits & NewMask)) {
1152  // None of the shifted in bits are needed. Add a truncate of the
1153  // shift input, then shift it.
1154  SDValue NewTrunc = TLO.DAG.getNode(ISD::TRUNCATE, dl, VT,
1155  In.getOperand(0));
1156  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc,
1157  Shift));
1158  }
1159  }
1160  break;
1161  }
1162  }
1163 
1164  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1165  break;
1166  }
1167  case ISD::AssertZext: {
1168  // AssertZext demands all of the high bits, plus any of the low bits
1169  // demanded by its users.
1170  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1171  APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
1172  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | NewMask,
1173  Known, TLO, Depth+1))
1174  return true;
1175  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1176 
1177  Known.Zero |= ~InMask;
1178  break;
1179  }
1180  case ISD::BITCAST:
1181  // If this is an FP->Int bitcast and if the sign bit is the only
1182  // thing demanded, turn this into a FGETSIGN.
1183  if (!TLO.LegalOperations() && !VT.isVector() &&
1184  !Op.getOperand(0).getValueType().isVector() &&
1185  NewMask == APInt::getSignMask(Op.getValueSizeInBits()) &&
1187  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
1189  if ((OpVTLegal || i32Legal) && VT.isSimple() &&
1190  Op.getOperand(0).getValueType() != MVT::f128) {
1191  // Cannot eliminate/lower SHL for f128 yet.
1192  EVT Ty = OpVTLegal ? VT : MVT::i32;
1193  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1194  // place. We expect the SHL to be eliminated by other optimizations.
1195  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Op.getOperand(0));
1196  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1197  if (!OpVTLegal && OpVTSizeInBits > 32)
1198  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
1199  unsigned ShVal = Op.getValueSizeInBits() - 1;
1200  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1201  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
1202  }
1203  }
1204  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
1205  // recursive call where Known may be useful to the caller.
1206  if (Depth > 0) {
1207  TLO.DAG.computeKnownBits(Op, Known, Depth);
1208  return false;
1209  }
1210  break;
1211  case ISD::ADD:
1212  case ISD::MUL:
1213  case ISD::SUB: {
1214  // Add, Sub, and Mul don't demand any bits in positions beyond that
1215  // of the highest bit demanded of them.
1216  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
1217  unsigned NewMaskLZ = NewMask.countLeadingZeros();
1218  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - NewMaskLZ);
1219  if (SimplifyDemandedBits(Op0, LoMask, Known2, TLO, Depth + 1) ||
1220  SimplifyDemandedBits(Op1, LoMask, Known2, TLO, Depth + 1) ||
1221  // See if the operation should be performed at a smaller bit width.
1222  ShrinkDemandedOp(Op, BitWidth, NewMask, TLO)) {
1223  SDNodeFlags Flags = Op.getNode()->getFlags();
1224  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1225  // Disable the nsw and nuw flags. We can no longer guarantee that we
1226  // won't wrap after simplification.
1227  Flags.setNoSignedWrap(false);
1228  Flags.setNoUnsignedWrap(false);
1229  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1,
1230  Flags);
1231  return TLO.CombineTo(Op, NewOp);
1232  }
1233  return true;
1234  }
1235 
1236  // If we have a constant operand, we may be able to turn it into -1 if we
1237  // do not demand the high bits. This can make the constant smaller to
1238  // encode, allow more general folding, or match specialized instruction
1239  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
1240  // is probably not useful (and could be detrimental).
1242  APInt HighMask = APInt::getHighBitsSet(NewMask.getBitWidth(), NewMaskLZ);
1243  if (C && !C->isAllOnesValue() && !C->isOne() &&
1244  (C->getAPIntValue() | HighMask).isAllOnesValue()) {
1245  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
1246  // We can't guarantee that the new math op doesn't wrap, so explicitly
1247  // clear those flags to prevent folding with a potential existing node
1248  // that has those flags set.
1249  SDNodeFlags Flags;
1250  Flags.setNoSignedWrap(false);
1251  Flags.setNoUnsignedWrap(false);
1252  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
1253  return TLO.CombineTo(Op, NewOp);
1254  }
1255 
1257  }
1258  default:
1259  // Just use computeKnownBits to compute output bits.
1260  TLO.DAG.computeKnownBits(Op, Known, Depth);
1261  break;
1262  }
1263 
1264  // If we know the value of all of the demanded bits, return this as a
1265  // constant.
1266  if (NewMask.isSubsetOf(Known.Zero|Known.One)) {
1267  // Avoid folding to a constant if any OpaqueConstant is involved.
1268  const SDNode *N = Op.getNode();
1270  E = SDNodeIterator::end(N); I != E; ++I) {
1271  SDNode *Op = *I;
1272  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
1273  if (C->isOpaque())
1274  return false;
1275  }
1276  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
1277  }
1278 
1279  return false;
1280 }
1281 
1283  const APInt &DemandedElts,
1284  APInt &KnownUndef,
1285  APInt &KnownZero,
1286  DAGCombinerInfo &DCI) const {
1287  SelectionDAG &DAG = DCI.DAG;
1288  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
1289  !DCI.isBeforeLegalizeOps());
1290 
1291  bool Simplified =
1292  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
1293  if (Simplified)
1294  DCI.CommitTargetLoweringOpt(TLO);
1295  return Simplified;
1296 }
1297 
1299  SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef,
1300  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
1301  bool AssumeSingleUse) const {
1302  EVT VT = Op.getValueType();
1303  APInt DemandedElts = DemandedEltMask;
1304  unsigned NumElts = DemandedElts.getBitWidth();
1305  assert(VT.isVector() && "Expected vector op");
1306  assert(VT.getVectorNumElements() == NumElts &&
1307  "Mask size mismatches value type element count!");
1308 
1309  KnownUndef = KnownZero = APInt::getNullValue(NumElts);
1310 
1311  // Undef operand.
1312  if (Op.isUndef()) {
1313  KnownUndef.setAllBits();
1314  return false;
1315  }
1316 
1317  // If Op has other users, assume that all elements are needed.
1318  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
1319  DemandedElts.setAllBits();
1320 
1321  // Not demanding any elements from Op.
1322  if (DemandedElts == 0) {
1323  KnownUndef.setAllBits();
1324  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1325  }
1326 
1327  // Limit search depth.
1328  if (Depth >= 6)
1329  return false;
1330 
1331  SDLoc DL(Op);
1332  unsigned EltSizeInBits = VT.getScalarSizeInBits();
1333 
1334  switch (Op.getOpcode()) {
1335  case ISD::SCALAR_TO_VECTOR: {
1336  if (!DemandedElts[0]) {
1337  KnownUndef.setAllBits();
1338  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1339  }
1340  KnownUndef.setHighBits(NumElts - 1);
1341  break;
1342  }
1343  case ISD::BITCAST: {
1344  SDValue Src = Op.getOperand(0);
1345  EVT SrcVT = Src.getValueType();
1346 
1347  // We only handle vectors here.
1348  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
1349  if (!SrcVT.isVector())
1350  break;
1351 
1352  // Fast handling of 'identity' bitcasts.
1353  unsigned NumSrcElts = SrcVT.getVectorNumElements();
1354  if (NumSrcElts == NumElts)
1355  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
1356  KnownZero, TLO, Depth + 1);
1357 
1358  APInt SrcZero, SrcUndef;
1359  APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
1360 
1361  // Bitcast from 'large element' src vector to 'small element' vector, we
1362  // must demand a source element if any DemandedElt maps to it.
1363  if ((NumElts % NumSrcElts) == 0) {
1364  unsigned Scale = NumElts / NumSrcElts;
1365  for (unsigned i = 0; i != NumElts; ++i)
1366  if (DemandedElts[i])
1367  SrcDemandedElts.setBit(i / Scale);
1368 
1369  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1370  TLO, Depth + 1))
1371  return true;
1372 
1373  // If the src element is zero/undef then all the output elements will be -
1374  // only demanded elements are guaranteed to be correct.
1375  for (unsigned i = 0; i != NumSrcElts; ++i) {
1376  if (SrcDemandedElts[i]) {
1377  if (SrcZero[i])
1378  KnownZero.setBits(i * Scale, (i + 1) * Scale);
1379  if (SrcUndef[i])
1380  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
1381  }
1382  }
1383  }
1384 
1385  // Bitcast from 'small element' src vector to 'large element' vector, we
1386  // demand all smaller source elements covered by the larger demanded element
1387  // of this vector.
1388  if ((NumSrcElts % NumElts) == 0) {
1389  unsigned Scale = NumSrcElts / NumElts;
1390  for (unsigned i = 0; i != NumElts; ++i)
1391  if (DemandedElts[i])
1392  SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
1393 
1394  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1395  TLO, Depth + 1))
1396  return true;
1397 
1398  // If all the src elements covering an output element are zero/undef, then
1399  // the output element will be as well, assuming it was demanded.
1400  for (unsigned i = 0; i != NumElts; ++i) {
1401  if (DemandedElts[i]) {
1402  if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
1403  KnownZero.setBit(i);
1404  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
1405  KnownUndef.setBit(i);
1406  }
1407  }
1408  }
1409  break;
1410  }
1411  case ISD::BUILD_VECTOR: {
1412  // Check all elements and simplify any unused elements with UNDEF.
1413  if (!DemandedElts.isAllOnesValue()) {
1414  // Don't simplify BROADCASTS.
1415  if (llvm::any_of(Op->op_values(),
1416  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
1417  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
1418  bool Updated = false;
1419  for (unsigned i = 0; i != NumElts; ++i) {
1420  if (!DemandedElts[i] && !Ops[i].isUndef()) {
1421  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
1422  KnownUndef.setBit(i);
1423  Updated = true;
1424  }
1425  }
1426  if (Updated)
1427  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
1428  }
1429  }
1430  for (unsigned i = 0; i != NumElts; ++i) {
1431  SDValue SrcOp = Op.getOperand(i);
1432  if (SrcOp.isUndef()) {
1433  KnownUndef.setBit(i);
1434  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
1435  (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
1436  KnownZero.setBit(i);
1437  }
1438  }
1439  break;
1440  }
1441  case ISD::CONCAT_VECTORS: {
1442  EVT SubVT = Op.getOperand(0).getValueType();
1443  unsigned NumSubVecs = Op.getNumOperands();
1444  unsigned NumSubElts = SubVT.getVectorNumElements();
1445  for (unsigned i = 0; i != NumSubVecs; ++i) {
1446  SDValue SubOp = Op.getOperand(i);
1447  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1448  APInt SubUndef, SubZero;
1449  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
1450  Depth + 1))
1451  return true;
1452  KnownUndef.insertBits(SubUndef, i * NumSubElts);
1453  KnownZero.insertBits(SubZero, i * NumSubElts);
1454  }
1455  break;
1456  }
1457  case ISD::INSERT_SUBVECTOR: {
1458  if (!isa<ConstantSDNode>(Op.getOperand(2)))
1459  break;
1460  SDValue Base = Op.getOperand(0);
1461  SDValue Sub = Op.getOperand(1);
1462  EVT SubVT = Sub.getValueType();
1463  unsigned NumSubElts = SubVT.getVectorNumElements();
1464  APInt Idx = cast<ConstantSDNode>(Op.getOperand(2))->getAPIntValue();
1465  if (Idx.uge(NumElts - NumSubElts))
1466  break;
1467  unsigned SubIdx = Idx.getZExtValue();
1468  APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
1469  APInt SubUndef, SubZero;
1470  if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
1471  Depth + 1))
1472  return true;
1473  APInt BaseElts = DemandedElts;
1474  BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
1475  if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
1476  Depth + 1))
1477  return true;
1478  KnownUndef.insertBits(SubUndef, SubIdx);
1479  KnownZero.insertBits(SubZero, SubIdx);
1480  break;
1481  }
1482  case ISD::INSERT_VECTOR_ELT: {
1483  SDValue Vec = Op.getOperand(0);
1484  SDValue Scl = Op.getOperand(1);
1485  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1486 
1487  // For a legal, constant insertion index, if we don't need this insertion
1488  // then strip it, else remove it from the demanded elts.
1489  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
1490  unsigned Idx = CIdx->getZExtValue();
1491  if (!DemandedElts[Idx])
1492  return TLO.CombineTo(Op, Vec);
1493  DemandedElts.clearBit(Idx);
1494 
1495  if (SimplifyDemandedVectorElts(Vec, DemandedElts, KnownUndef,
1496  KnownZero, TLO, Depth + 1))
1497  return true;
1498 
1499  KnownUndef.clearBit(Idx);
1500  if (Scl.isUndef())
1501  KnownUndef.setBit(Idx);
1502 
1503  KnownZero.clearBit(Idx);
1504  if (isNullConstant(Scl) || isNullFPConstant(Scl))
1505  KnownZero.setBit(Idx);
1506  break;
1507  }
1508 
1509  APInt VecUndef, VecZero;
1510  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
1511  Depth + 1))
1512  return true;
1513  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
1514  break;
1515  }
1516  case ISD::VSELECT: {
1517  APInt DemandedLHS(DemandedElts);
1518  APInt DemandedRHS(DemandedElts);
1519 
1520  // TODO - add support for constant vselect masks.
1521 
1522  // See if we can simplify either vselect operand.
1523  APInt UndefLHS, ZeroLHS;
1524  APInt UndefRHS, ZeroRHS;
1525  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
1526  ZeroLHS, TLO, Depth + 1))
1527  return true;
1528  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
1529  ZeroRHS, TLO, Depth + 1))
1530  return true;
1531 
1532  KnownUndef = UndefLHS & UndefRHS;
1533  KnownZero = ZeroLHS & ZeroRHS;
1534  break;
1535  }
1536  case ISD::VECTOR_SHUFFLE: {
1537  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1538 
1539  // Collect demanded elements from shuffle operands..
1540  APInt DemandedLHS(NumElts, 0);
1541  APInt DemandedRHS(NumElts, 0);
1542  for (unsigned i = 0; i != NumElts; ++i) {
1543  int M = ShuffleMask[i];
1544  if (M < 0 || !DemandedElts[i])
1545  continue;
1546  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1547  if (M < (int)NumElts)
1548  DemandedLHS.setBit(M);
1549  else
1550  DemandedRHS.setBit(M - NumElts);
1551  }
1552 
1553  // See if we can simplify either shuffle operand.
1554  APInt UndefLHS, ZeroLHS;
1555  APInt UndefRHS, ZeroRHS;
1556  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
1557  ZeroLHS, TLO, Depth + 1))
1558  return true;
1559  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
1560  ZeroRHS, TLO, Depth + 1))
1561  return true;
1562 
1563  // Simplify mask using undef elements from LHS/RHS.
1564  bool Updated = false;
1565  bool IdentityLHS = true, IdentityRHS = true;
1566  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
1567  for (unsigned i = 0; i != NumElts; ++i) {
1568  int &M = NewMask[i];
1569  if (M < 0)
1570  continue;
1571  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
1572  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
1573  Updated = true;
1574  M = -1;
1575  }
1576  IdentityLHS &= (M < 0) || (M == (int)i);
1577  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
1578  }
1579 
1580  // Update legal shuffle masks based on demanded elements if it won't reduce
1581  // to Identity which can cause premature removal of the shuffle mask.
1582  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps &&
1583  isShuffleMaskLegal(NewMask, VT))
1584  return TLO.CombineTo(Op,
1585  TLO.DAG.getVectorShuffle(VT, DL, Op.getOperand(0),
1586  Op.getOperand(1), NewMask));
1587 
1588  // Propagate undef/zero elements from LHS/RHS.
1589  for (unsigned i = 0; i != NumElts; ++i) {
1590  int M = ShuffleMask[i];
1591  if (M < 0) {
1592  KnownUndef.setBit(i);
1593  } else if (M < (int)NumElts) {
1594  if (UndefLHS[M])
1595  KnownUndef.setBit(i);
1596  if (ZeroLHS[M])
1597  KnownZero.setBit(i);
1598  } else {
1599  if (UndefRHS[M - NumElts])
1600  KnownUndef.setBit(i);
1601  if (ZeroRHS[M - NumElts])
1602  KnownZero.setBit(i);
1603  }
1604  }
1605  break;
1606  }
1607  case ISD::ADD:
1608  case ISD::SUB: {
1609  APInt SrcUndef, SrcZero;
1610  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
1611  SrcZero, TLO, Depth + 1))
1612  return true;
1613  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1614  KnownZero, TLO, Depth + 1))
1615  return true;
1616  KnownZero &= SrcZero;
1617  KnownUndef &= SrcUndef;
1618  break;
1619  }
1620  case ISD::TRUNCATE:
1621  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1622  KnownZero, TLO, Depth + 1))
1623  return true;
1624  break;
1625  default: {
1626  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1627  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
1628  KnownZero, TLO, Depth))
1629  return true;
1630  break;
1631  }
1632  }
1633 
1634  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
1635  return false;
1636 }
1637 
1638 /// Determine which of the bits specified in Mask are known to be either zero or
1639 /// one and return them in the Known.
1641  KnownBits &Known,
1642  const APInt &DemandedElts,
1643  const SelectionDAG &DAG,
1644  unsigned Depth) const {
1648  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1649  "Should use MaskedValueIsZero if you don't know whether Op"
1650  " is a target node!");
1651  Known.resetAll();
1652 }
1653 
1655  KnownBits &Known,
1656  const APInt &DemandedElts,
1657  const SelectionDAG &DAG,
1658  unsigned Depth) const {
1659  assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex");
1660 
1661  if (unsigned Align = DAG.InferPtrAlignment(Op)) {
1662  // The low bits are known zero if the pointer is aligned.
1663  Known.Zero.setLowBits(Log2_32(Align));
1664  }
1665 }
1666 
1667 /// This method can be implemented by targets that want to expose additional
1668 /// information about sign bits to the DAG Combiner.
1670  const APInt &,
1671  const SelectionDAG &,
1672  unsigned Depth) const {
1676  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1677  "Should use ComputeNumSignBits if you don't know whether Op"
1678  " is a target node!");
1679  return 1;
1680 }
1681 
1683  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
1684  TargetLoweringOpt &TLO, unsigned Depth) const {
1688  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1689  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
1690  " is a target node!");
1691  return false;
1692 }
1693 
1694 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
1695 // work with truncating build vectors and vectors with elements of less than
1696 // 8 bits.
1698  if (!N)
1699  return false;
1700 
1701  APInt CVal;
1702  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
1703  CVal = CN->getAPIntValue();
1704  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
1705  auto *CN = BV->getConstantSplatNode();
1706  if (!CN)
1707  return false;
1708 
1709  // If this is a truncating build vector, truncate the splat value.
1710  // Otherwise, we may fail to match the expected values below.
1711  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
1712  CVal = CN->getAPIntValue();
1713  if (BVEltWidth < CVal.getBitWidth())
1714  CVal = CVal.trunc(BVEltWidth);
1715  } else {
1716  return false;
1717  }
1718 
1719  switch (getBooleanContents(N->getValueType(0))) {
1721  return CVal[0];
1723  return CVal.isOneValue();
1725  return CVal.isAllOnesValue();
1726  }
1727 
1728  llvm_unreachable("Invalid boolean contents");
1729 }
1730 
1732  if (!N)
1733  return false;
1734 
1735  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
1736  if (!CN) {
1738  if (!BV)
1739  return false;
1740 
1741  // Only interested in constant splats, we don't care about undef
1742  // elements in identifying boolean constants and getConstantSplatNode
1743  // returns NULL if all ops are undef;
1744  CN = BV->getConstantSplatNode();
1745  if (!CN)
1746  return false;
1747  }
1748 
1750  return !CN->getAPIntValue()[0];
1751 
1752  return CN->isNullValue();
1753 }
1754 
1756  bool SExt) const {
1757  if (VT == MVT::i1)
1758  return N->isOne();
1759 
1761  switch (Cnt) {
1763  // An extended value of 1 is always true, unless its original type is i1,
1764  // in which case it will be sign extended to -1.
1765  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
1768  return N->isAllOnesValue() && SExt;
1769  }
1770  llvm_unreachable("Unexpected enumeration.");
1771 }
1772 
1773 /// This helper function of SimplifySetCC tries to optimize the comparison when
1774 /// either operand of the SetCC node is a bitwise-and instruction.
1775 SDValue TargetLowering::simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
1776  ISD::CondCode Cond,
1777  DAGCombinerInfo &DCI,
1778  const SDLoc &DL) const {
1779  // Match these patterns in any of their permutations:
1780  // (X & Y) == Y
1781  // (X & Y) != Y
1782  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
1783  std::swap(N0, N1);
1784 
1785  EVT OpVT = N0.getValueType();
1786  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
1787  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
1788  return SDValue();
1789 
1790  SDValue X, Y;
1791  if (N0.getOperand(0) == N1) {
1792  X = N0.getOperand(1);
1793  Y = N0.getOperand(0);
1794  } else if (N0.getOperand(1) == N1) {
1795  X = N0.getOperand(0);
1796  Y = N0.getOperand(1);
1797  } else {
1798  return SDValue();
1799  }
1800 
1801  SelectionDAG &DAG = DCI.DAG;
1802  SDValue Zero = DAG.getConstant(0, DL, OpVT);
1803  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
1804  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
1805  // Note that where Y is variable and is known to have at most one bit set
1806  // (for example, if it is Z & 1) we cannot do this; the expressions are not
1807  // equivalent when Y == 0.
1808  Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
1809  if (DCI.isBeforeLegalizeOps() ||
1810  isCondCodeLegal(Cond, N0.getSimpleValueType()))
1811  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
1812  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
1813  // If the target supports an 'and-not' or 'and-complement' logic operation,
1814  // try to use that to make a comparison operation more efficient.
1815  // But don't do this transform if the mask is a single bit because there are
1816  // more efficient ways to deal with that case (for example, 'bt' on x86 or
1817  // 'rlwinm' on PPC).
1818 
1819  // Bail out if the compare operand that we want to turn into a zero is
1820  // already a zero (otherwise, infinite loop).
1821  auto *YConst = dyn_cast<ConstantSDNode>(Y);
1822  if (YConst && YConst->isNullValue())
1823  return SDValue();
1824 
1825  // Transform this into: ~X & Y == 0.
1826  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
1827  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
1828  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
1829  }
1830 
1831  return SDValue();
1832 }
1833 
1834 /// Try to simplify a setcc built with the specified operands and cc. If it is
1835 /// unable to simplify it, return a null SDValue.
1837  ISD::CondCode Cond, bool foldBooleans,
1838  DAGCombinerInfo &DCI,
1839  const SDLoc &dl) const {
1840  SelectionDAG &DAG = DCI.DAG;
1841  EVT OpVT = N0.getValueType();
1842 
1843  // These setcc operations always fold.
1844  switch (Cond) {
1845  default: break;
1846  case ISD::SETFALSE:
1847  case ISD::SETFALSE2: return DAG.getBoolConstant(false, dl, VT, OpVT);
1848  case ISD::SETTRUE:
1849  case ISD::SETTRUE2: return DAG.getBoolConstant(true, dl, VT, OpVT);
1850  }
1851 
1852  // Ensure that the constant occurs on the RHS and fold constant comparisons.
1853  // TODO: Handle non-splat vector constants. All undef causes trouble.
1854  ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
1855  if (isConstOrConstSplat(N0) &&
1856  (DCI.isBeforeLegalizeOps() ||
1857  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
1858  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
1859 
1860  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
1861  const APInt &C1 = N1C->getAPIntValue();
1862 
1863  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
1864  // equality comparison, then we're just comparing whether X itself is
1865  // zero.
1866  if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
1867  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
1868  N0.getOperand(1).getOpcode() == ISD::Constant) {
1869  const APInt &ShAmt
1870  = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
1871  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
1872  ShAmt == Log2_32(N0.getValueSizeInBits())) {
1873  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
1874  // (srl (ctlz x), 5) == 0 -> X != 0
1875  // (srl (ctlz x), 5) != 1 -> X != 0
1876  Cond = ISD::SETNE;
1877  } else {
1878  // (srl (ctlz x), 5) != 0 -> X == 0
1879  // (srl (ctlz x), 5) == 1 -> X == 0
1880  Cond = ISD::SETEQ;
1881  }
1882  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
1883  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
1884  Zero, Cond);
1885  }
1886  }
1887 
1888  SDValue CTPOP = N0;
1889  // Look through truncs that don't change the value of a ctpop.
1890  if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
1891  CTPOP = N0.getOperand(0);
1892 
1893  if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
1894  (N0 == CTPOP ||
1896  EVT CTVT = CTPOP.getValueType();
1897  SDValue CTOp = CTPOP.getOperand(0);
1898 
1899  // (ctpop x) u< 2 -> (x & x-1) == 0
1900  // (ctpop x) u> 1 -> (x & x-1) != 0
1901  if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
1902  SDValue Sub = DAG.getNode(ISD::SUB, dl, CTVT, CTOp,
1903  DAG.getConstant(1, dl, CTVT));
1904  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Sub);
1906  return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
1907  }
1908 
1909  // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal.
1910  }
1911 
1912  // (zext x) == C --> x == (trunc C)
1913  // (sext x) == C --> x == (trunc C)
1914  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
1915  DCI.isBeforeLegalize() && N0->hasOneUse()) {
1916  unsigned MinBits = N0.getValueSizeInBits();
1917  SDValue PreExt;
1918  bool Signed = false;
1919  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
1920  // ZExt
1921  MinBits = N0->getOperand(0).getValueSizeInBits();
1922  PreExt = N0->getOperand(0);
1923  } else if (N0->getOpcode() == ISD::AND) {
1924  // DAGCombine turns costly ZExts into ANDs
1925  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
1926  if ((C->getAPIntValue()+1).isPowerOf2()) {
1927  MinBits = C->getAPIntValue().countTrailingOnes();
1928  PreExt = N0->getOperand(0);
1929  }
1930  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
1931  // SExt
1932  MinBits = N0->getOperand(0).getValueSizeInBits();
1933  PreExt = N0->getOperand(0);
1934  Signed = true;
1935  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
1936  // ZEXTLOAD / SEXTLOAD
1937  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
1938  MinBits = LN0->getMemoryVT().getSizeInBits();
1939  PreExt = N0;
1940  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
1941  Signed = true;
1942  MinBits = LN0->getMemoryVT().getSizeInBits();
1943  PreExt = N0;
1944  }
1945  }
1946 
1947  // Figure out how many bits we need to preserve this constant.
1948  unsigned ReqdBits = Signed ?
1949  C1.getBitWidth() - C1.getNumSignBits() + 1 :
1950  C1.getActiveBits();
1951 
1952  // Make sure we're not losing bits from the constant.
1953  if (MinBits > 0 &&
1954  MinBits < C1.getBitWidth() &&
1955  MinBits >= ReqdBits) {
1956  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
1957  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
1958  // Will get folded away.
1959  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
1960  if (MinBits == 1 && C1 == 1)
1961  // Invert the condition.
1962  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
1963  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
1964  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
1965  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
1966  }
1967 
1968  // If truncating the setcc operands is not desirable, we can still
1969  // simplify the expression in some cases:
1970  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
1971  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
1972  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
1973  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
1974  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
1975  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
1976  SDValue TopSetCC = N0->getOperand(0);
1977  unsigned N0Opc = N0->getOpcode();
1978  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
1979  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
1980  TopSetCC.getOpcode() == ISD::SETCC &&
1981  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
1982  (isConstFalseVal(N1C) ||
1983  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
1984 
1985  bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
1986  (!N1C->isNullValue() && Cond == ISD::SETNE);
1987 
1988  if (!Inverse)
1989  return TopSetCC;
1990 
1992  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
1993  TopSetCC.getOperand(0).getValueType().isInteger());
1994  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
1995  TopSetCC.getOperand(1),
1996  InvCond);
1997  }
1998  }
1999  }
2000 
2001  // If the LHS is '(and load, const)', the RHS is 0, the test is for
2002  // equality or unsigned, and all 1 bits of the const are in the same
2003  // partial word, see if we can shorten the load.
2004  if (DCI.isBeforeLegalize() &&
2005  !ISD::isSignedIntSetCC(Cond) &&
2006  N0.getOpcode() == ISD::AND && C1 == 0 &&
2007  N0.getNode()->hasOneUse() &&
2008  isa<LoadSDNode>(N0.getOperand(0)) &&
2009  N0.getOperand(0).getNode()->hasOneUse() &&
2010  isa<ConstantSDNode>(N0.getOperand(1))) {
2011  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
2012  APInt bestMask;
2013  unsigned bestWidth = 0, bestOffset = 0;
2014  if (!Lod->isVolatile() && Lod->isUnindexed()) {
2015  unsigned origWidth = N0.getValueSizeInBits();
2016  unsigned maskWidth = origWidth;
2017  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
2018  // 8 bits, but have to be careful...
2019  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
2020  origWidth = Lod->getMemoryVT().getSizeInBits();
2021  const APInt &Mask =
2022  cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
2023  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
2024  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
2025  for (unsigned offset=0; offset<origWidth/width; offset++) {
2026  if (Mask.isSubsetOf(newMask)) {
2027  if (DAG.getDataLayout().isLittleEndian())
2028  bestOffset = (uint64_t)offset * (width/8);
2029  else
2030  bestOffset = (origWidth/width - offset - 1) * (width/8);
2031  bestMask = Mask.lshr(offset * (width/8) * 8);
2032  bestWidth = width;
2033  break;
2034  }
2035  newMask <<= width;
2036  }
2037  }
2038  }
2039  if (bestWidth) {
2040  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
2041  if (newVT.isRound()) {
2042  EVT PtrType = Lod->getOperand(1).getValueType();
2043  SDValue Ptr = Lod->getBasePtr();
2044  if (bestOffset != 0)
2045  Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
2046  DAG.getConstant(bestOffset, dl, PtrType));
2047  unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
2048  SDValue NewLoad = DAG.getLoad(
2049  newVT, dl, Lod->getChain(), Ptr,
2050  Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
2051  return DAG.getSetCC(dl, VT,
2052  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
2053  DAG.getConstant(bestMask.trunc(bestWidth),
2054  dl, newVT)),
2055  DAG.getConstant(0LL, dl, newVT), Cond);
2056  }
2057  }
2058  }
2059 
2060  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
2061  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
2062  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
2063 
2064  // If the comparison constant has bits in the upper part, the
2065  // zero-extended value could never match.
2066  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
2067  C1.getBitWidth() - InSize))) {
2068  switch (Cond) {
2069  case ISD::SETUGT:
2070  case ISD::SETUGE:
2071  case ISD::SETEQ:
2072  return DAG.getConstant(0, dl, VT);
2073  case ISD::SETULT:
2074  case ISD::SETULE:
2075  case ISD::SETNE:
2076  return DAG.getConstant(1, dl, VT);
2077  case ISD::SETGT:
2078  case ISD::SETGE:
2079  // True if the sign bit of C1 is set.
2080  return DAG.getConstant(C1.isNegative(), dl, VT);
2081  case ISD::SETLT:
2082  case ISD::SETLE:
2083  // True if the sign bit of C1 isn't set.
2084  return DAG.getConstant(C1.isNonNegative(), dl, VT);
2085  default:
2086  break;
2087  }
2088  }
2089 
2090  // Otherwise, we can perform the comparison with the low bits.
2091  switch (Cond) {
2092  case ISD::SETEQ:
2093  case ISD::SETNE:
2094  case ISD::SETUGT:
2095  case ISD::SETUGE:
2096  case ISD::SETULT:
2097  case ISD::SETULE: {
2098  EVT newVT = N0.getOperand(0).getValueType();
2099  if (DCI.isBeforeLegalizeOps() ||
2100  (isOperationLegal(ISD::SETCC, newVT) &&
2101  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
2102  EVT NewSetCCVT =
2103  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), newVT);
2104  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
2105 
2106  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
2107  NewConst, Cond);
2108  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
2109  }
2110  break;
2111  }
2112  default:
2113  break; // todo, be more careful with signed comparisons
2114  }
2115  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2116  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2117  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
2118  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
2119  EVT ExtDstTy = N0.getValueType();
2120  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
2121 
2122  // If the constant doesn't fit into the number of bits for the source of
2123  // the sign extension, it is impossible for both sides to be equal.
2124  if (C1.getMinSignedBits() > ExtSrcTyBits)
2125  return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
2126 
2127  SDValue ZextOp;
2128  EVT Op0Ty = N0.getOperand(0).getValueType();
2129  if (Op0Ty == ExtSrcTy) {
2130  ZextOp = N0.getOperand(0);
2131  } else {
2132  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
2133  ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
2134  DAG.getConstant(Imm, dl, Op0Ty));
2135  }
2136  if (!DCI.isCalledByLegalizer())
2137  DCI.AddToWorklist(ZextOp.getNode());
2138  // Otherwise, make this a use of a zext.
2139  return DAG.getSetCC(dl, VT, ZextOp,
2141  ExtDstTyBits,
2142  ExtSrcTyBits),
2143  dl, ExtDstTy),
2144  Cond);
2145  } else if ((N1C->isNullValue() || N1C->isOne()) &&
2146  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2147  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
2148  if (N0.getOpcode() == ISD::SETCC &&
2149  isTypeLegal(VT) && VT.bitsLE(N0.getValueType())) {
2150  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
2151  if (TrueWhenTrue)
2152  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
2153  // Invert the condition.
2154  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
2155  CC = ISD::getSetCCInverse(CC,
2156  N0.getOperand(0).getValueType().isInteger());
2157  if (DCI.isBeforeLegalizeOps() ||
2158  isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
2159  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
2160  }
2161 
2162  if ((N0.getOpcode() == ISD::XOR ||
2163  (N0.getOpcode() == ISD::AND &&
2164  N0.getOperand(0).getOpcode() == ISD::XOR &&
2165  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
2166  isa<ConstantSDNode>(N0.getOperand(1)) &&
2167  cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
2168  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
2169  // can only do this if the top bits are known zero.
2170  unsigned BitWidth = N0.getValueSizeInBits();
2171  if (DAG.MaskedValueIsZero(N0,
2172  APInt::getHighBitsSet(BitWidth,
2173  BitWidth-1))) {
2174  // Okay, get the un-inverted input value.
2175  SDValue Val;
2176  if (N0.getOpcode() == ISD::XOR) {
2177  Val = N0.getOperand(0);
2178  } else {
2179  assert(N0.getOpcode() == ISD::AND &&
2180  N0.getOperand(0).getOpcode() == ISD::XOR);
2181  // ((X^1)&1)^1 -> X & 1
2182  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
2183  N0.getOperand(0).getOperand(0),
2184  N0.getOperand(1));
2185  }
2186 
2187  return DAG.getSetCC(dl, VT, Val, N1,
2188  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2189  }
2190  } else if (N1C->isOne() &&
2191  (VT == MVT::i1 ||
2192  getBooleanContents(N0->getValueType(0)) ==
2194  SDValue Op0 = N0;
2195  if (Op0.getOpcode() == ISD::TRUNCATE)
2196  Op0 = Op0.getOperand(0);
2197 
2198  if ((Op0.getOpcode() == ISD::XOR) &&
2199  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
2200  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
2201  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
2202  Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
2203  return DAG.getSetCC(dl, VT, Op0.getOperand(0), Op0.getOperand(1),
2204  Cond);
2205  }
2206  if (Op0.getOpcode() == ISD::AND &&
2207  isa<ConstantSDNode>(Op0.getOperand(1)) &&
2208  cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
2209  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
2210  if (Op0.getValueType().bitsGT(VT))
2211  Op0 = DAG.getNode(ISD::AND, dl, VT,
2212  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
2213  DAG.getConstant(1, dl, VT));
2214  else if (Op0.getValueType().bitsLT(VT))
2215  Op0 = DAG.getNode(ISD::AND, dl, VT,
2216  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
2217  DAG.getConstant(1, dl, VT));
2218 
2219  return DAG.getSetCC(dl, VT, Op0,
2220  DAG.getConstant(0, dl, Op0.getValueType()),
2221  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2222  }
2223  if (Op0.getOpcode() == ISD::AssertZext &&
2224  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
2225  return DAG.getSetCC(dl, VT, Op0,
2226  DAG.getConstant(0, dl, Op0.getValueType()),
2227  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2228  }
2229  }
2230  }
2231 
2232  // These simplifications apply to splat vectors as well.
2233  // TODO: Handle more splat vector cases.
2234  if (auto *N1C = isConstOrConstSplat(N1)) {
2235  const APInt &C1 = N1C->getAPIntValue();
2236 
2237  APInt MinVal, MaxVal;
2238  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
2239  if (ISD::isSignedIntSetCC(Cond)) {
2240  MinVal = APInt::getSignedMinValue(OperandBitSize);
2241  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
2242  } else {
2243  MinVal = APInt::getMinValue(OperandBitSize);
2244  MaxVal = APInt::getMaxValue(OperandBitSize);
2245  }
2246 
2247  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
2248  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
2249  // X >= MIN --> true
2250  if (C1 == MinVal)
2251  return DAG.getBoolConstant(true, dl, VT, OpVT);
2252 
2253  if (!VT.isVector()) { // TODO: Support this for vectors.
2254  // X >= C0 --> X > (C0 - 1)
2255  APInt C = C1 - 1;
2256  ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
2257  if ((DCI.isBeforeLegalizeOps() ||
2258  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2259  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2261  return DAG.getSetCC(dl, VT, N0,
2262  DAG.getConstant(C, dl, N1.getValueType()),
2263  NewCC);
2264  }
2265  }
2266  }
2267 
2268  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
2269  // X <= MAX --> true
2270  if (C1 == MaxVal)
2271  return DAG.getBoolConstant(true, dl, VT, OpVT);
2272 
2273  // X <= C0 --> X < (C0 + 1)
2274  if (!VT.isVector()) { // TODO: Support this for vectors.
2275  APInt C = C1 + 1;
2276  ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
2277  if ((DCI.isBeforeLegalizeOps() ||
2278  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2279  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2281  return DAG.getSetCC(dl, VT, N0,
2282  DAG.getConstant(C, dl, N1.getValueType()),
2283  NewCC);
2284  }
2285  }
2286  }
2287 
2288  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
2289  if (C1 == MinVal)
2290  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
2291 
2292  // TODO: Support this for vectors after legalize ops.
2293  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2294  // Canonicalize setlt X, Max --> setne X, Max
2295  if (C1 == MaxVal)
2296  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2297 
2298  // If we have setult X, 1, turn it into seteq X, 0
2299  if (C1 == MinVal+1)
2300  return DAG.getSetCC(dl, VT, N0,
2301  DAG.getConstant(MinVal, dl, N0.getValueType()),
2302  ISD::SETEQ);
2303  }
2304  }
2305 
2306  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
2307  if (C1 == MaxVal)
2308  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
2309 
2310  // TODO: Support this for vectors after legalize ops.
2311  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2312  // Canonicalize setgt X, Min --> setne X, Min
2313  if (C1 == MinVal)
2314  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2315 
2316  // If we have setugt X, Max-1, turn it into seteq X, Max
2317  if (C1 == MaxVal-1)
2318  return DAG.getSetCC(dl, VT, N0,
2319  DAG.getConstant(MaxVal, dl, N0.getValueType()),
2320  ISD::SETEQ);
2321  }
2322  }
2323 
2324  // If we have "setcc X, C0", check to see if we can shrink the immediate
2325  // by changing cc.
2326  // TODO: Support this for vectors after legalize ops.
2327  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2328  // SETUGT X, SINTMAX -> SETLT X, 0
2329  if (Cond == ISD::SETUGT &&
2330  C1 == APInt::getSignedMaxValue(OperandBitSize))
2331  return DAG.getSetCC(dl, VT, N0,
2332  DAG.getConstant(0, dl, N1.getValueType()),
2333  ISD::SETLT);
2334 
2335  // SETULT X, SINTMIN -> SETGT X, -1
2336  if (Cond == ISD::SETULT &&
2337  C1 == APInt::getSignedMinValue(OperandBitSize)) {
2338  SDValue ConstMinusOne =
2339  DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
2340  N1.getValueType());
2341  return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
2342  }
2343  }
2344  }
2345 
2346  // Back to non-vector simplifications.
2347  // TODO: Can we do these for vector splats?
2348  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
2349  const APInt &C1 = N1C->getAPIntValue();
2350 
2351  // Fold bit comparisons when we can.
2352  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2353  (VT == N0.getValueType() ||
2354  (isTypeLegal(VT) && VT.bitsLE(N0.getValueType()))) &&
2355  N0.getOpcode() == ISD::AND) {
2356  auto &DL = DAG.getDataLayout();
2357  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2358  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2359  !DCI.isBeforeLegalize());
2360  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
2361  // Perform the xform if the AND RHS is a single bit.
2362  if (AndRHS->getAPIntValue().isPowerOf2()) {
2363  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2364  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2365  DAG.getConstant(AndRHS->getAPIntValue().logBase2(), dl,
2366  ShiftTy)));
2367  }
2368  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
2369  // (X & 8) == 8 --> (X & 8) >> 3
2370  // Perform the xform if C1 is a single bit.
2371  if (C1.isPowerOf2()) {
2372  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2373  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2374  DAG.getConstant(C1.logBase2(), dl,
2375  ShiftTy)));
2376  }
2377  }
2378  }
2379  }
2380 
2381  if (C1.getMinSignedBits() <= 64 &&
2383  // (X & -256) == 256 -> (X >> 8) == 1
2384  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2385  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
2386  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2387  const APInt &AndRHSC = AndRHS->getAPIntValue();
2388  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
2389  unsigned ShiftBits = AndRHSC.countTrailingZeros();
2390  auto &DL = DAG.getDataLayout();
2391  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2392  !DCI.isBeforeLegalize());
2393  EVT CmpTy = N0.getValueType();
2394  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
2395  DAG.getConstant(ShiftBits, dl,
2396  ShiftTy));
2397  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
2398  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
2399  }
2400  }
2401  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
2402  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
2403  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
2404  // X < 0x100000000 -> (X >> 32) < 1
2405  // X >= 0x100000000 -> (X >> 32) >= 1
2406  // X <= 0x0ffffffff -> (X >> 32) < 1
2407  // X > 0x0ffffffff -> (X >> 32) >= 1
2408  unsigned ShiftBits;
2409  APInt NewC = C1;
2410  ISD::CondCode NewCond = Cond;
2411  if (AdjOne) {
2412  ShiftBits = C1.countTrailingOnes();
2413  NewC = NewC + 1;
2414  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2415  } else {
2416  ShiftBits = C1.countTrailingZeros();
2417  }
2418  NewC.lshrInPlace(ShiftBits);
2419  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
2421  auto &DL = DAG.getDataLayout();
2422  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2423  !DCI.isBeforeLegalize());
2424  EVT CmpTy = N0.getValueType();
2425  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
2426  DAG.getConstant(ShiftBits, dl, ShiftTy));
2427  SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
2428  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
2429  }
2430  }
2431  }
2432  }
2433 
2434  if (isa<ConstantFPSDNode>(N0.getNode())) {
2435  // Constant fold or commute setcc.
2436  SDValue O = DAG.FoldSetCC(VT, N0, N1, Cond, dl);
2437  if (O.getNode()) return O;
2438  } else if (auto *CFP = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
2439  // If the RHS of an FP comparison is a constant, simplify it away in
2440  // some cases.
2441  if (CFP->getValueAPF().isNaN()) {
2442  // If an operand is known to be a nan, we can fold it.
2443  switch (ISD::getUnorderedFlavor(Cond)) {
2444  default: llvm_unreachable("Unknown flavor!");
2445  case 0: // Known false.
2446  return DAG.getBoolConstant(false, dl, VT, OpVT);
2447  case 1: // Known true.
2448  return DAG.getBoolConstant(true, dl, VT, OpVT);
2449  case 2: // Undefined.
2450  return DAG.getUNDEF(VT);
2451  }
2452  }
2453 
2454  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
2455  // constant if knowing that the operand is non-nan is enough. We prefer to
2456  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
2457  // materialize 0.0.
2458  if (Cond == ISD::SETO || Cond == ISD::SETUO)
2459  return DAG.getSetCC(dl, VT, N0, N0, Cond);
2460 
2461  // setcc (fneg x), C -> setcc swap(pred) x, -C
2462  if (N0.getOpcode() == ISD::FNEG) {
2464  if (DCI.isBeforeLegalizeOps() ||
2465  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
2466  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
2467  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
2468  }
2469  }
2470 
2471  // If the condition is not legal, see if we can find an equivalent one
2472  // which is legal.
2473  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
2474  // If the comparison was an awkward floating-point == or != and one of
2475  // the comparison operands is infinity or negative infinity, convert the
2476  // condition to a less-awkward <= or >=.
2477  if (CFP->getValueAPF().isInfinity()) {
2478  if (CFP->getValueAPF().isNegative()) {
2479  if (Cond == ISD::SETOEQ &&
2481  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
2482  if (Cond == ISD::SETUEQ &&
2484  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
2485  if (Cond == ISD::SETUNE &&
2487  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
2488  if (Cond == ISD::SETONE &&
2490  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
2491  } else {
2492  if (Cond == ISD::SETOEQ &&
2494  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
2495  if (Cond == ISD::SETUEQ &&
2497  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
2498  if (Cond == ISD::SETUNE &&
2500  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
2501  if (Cond == ISD::SETONE &&
2503  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
2504  }
2505  }
2506  }
2507  }
2508 
2509  if (N0 == N1) {
2510  // The sext(setcc()) => setcc() optimization relies on the appropriate
2511  // constant being emitted.
2512 
2513  bool EqTrue = ISD::isTrueWhenEqual(Cond);
2514 
2515  // We can always fold X == X for integer setcc's.
2516  if (N0.getValueType().isInteger())
2517  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2518 
2519  unsigned UOF = ISD::getUnorderedFlavor(Cond);
2520  if (UOF == 2) // FP operators that are undefined on NaNs.
2521  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2522  if (UOF == unsigned(EqTrue))
2523  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2524  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
2525  // if it is not already.
2526  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
2527  if (NewCond != Cond &&
2528  (DCI.isBeforeLegalizeOps() ||
2529  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
2530  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
2531  }
2532 
2533  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2534  N0.getValueType().isInteger()) {
2535  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
2536  N0.getOpcode() == ISD::XOR) {
2537  // Simplify (X+Y) == (X+Z) --> Y == Z
2538  if (N0.getOpcode() == N1.getOpcode()) {
2539  if (N0.getOperand(0) == N1.getOperand(0))
2540  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
2541  if (N0.getOperand(1) == N1.getOperand(1))
2542  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
2543  if (isCommutativeBinOp(N0.getOpcode())) {
2544  // If X op Y == Y op X, try other combinations.
2545  if (N0.getOperand(0) == N1.getOperand(1))
2546  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
2547  Cond);
2548  if (N0.getOperand(1) == N1.getOperand(0))
2549  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
2550  Cond);
2551  }
2552  }
2553 
2554  // If RHS is a legal immediate value for a compare instruction, we need
2555  // to be careful about increasing register pressure needlessly.
2556  bool LegalRHSImm = false;
2557 
2558  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
2559  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2560  // Turn (X+C1) == C2 --> X == C2-C1
2561  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
2562  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2563  DAG.getConstant(RHSC->getAPIntValue()-
2564  LHSR->getAPIntValue(),
2565  dl, N0.getValueType()), Cond);
2566  }
2567 
2568  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
2569  if (N0.getOpcode() == ISD::XOR)
2570  // If we know that all of the inverted bits are zero, don't bother
2571  // performing the inversion.
2572  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
2573  return
2574  DAG.getSetCC(dl, VT, N0.getOperand(0),
2575  DAG.getConstant(LHSR->getAPIntValue() ^
2576  RHSC->getAPIntValue(),
2577  dl, N0.getValueType()),
2578  Cond);
2579  }
2580 
2581  // Turn (C1-X) == C2 --> X == C1-C2
2582  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
2583  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
2584  return
2585  DAG.getSetCC(dl, VT, N0.getOperand(1),
2586  DAG.getConstant(SUBC->getAPIntValue() -
2587  RHSC->getAPIntValue(),
2588  dl, N0.getValueType()),
2589  Cond);
2590  }
2591  }
2592 
2593  // Could RHSC fold directly into a compare?
2594  if (RHSC->getValueType(0).getSizeInBits() <= 64)
2595  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
2596  }
2597 
2598  // Simplify (X+Z) == X --> Z == 0
2599  // Don't do this if X is an immediate that can fold into a cmp
2600  // instruction and X+Z has other uses. It could be an induction variable
2601  // chain, and the transform would increase register pressure.
2602  if (!LegalRHSImm || N0.getNode()->hasOneUse()) {
2603  if (N0.getOperand(0) == N1)
2604  return DAG.getSetCC(dl, VT, N0.getOperand(1),
2605  DAG.getConstant(0, dl, N0.getValueType()), Cond);
2606  if (N0.getOperand(1) == N1) {
2607  if (isCommutativeBinOp(N0.getOpcode()))
2608  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2609  DAG.getConstant(0, dl, N0.getValueType()),
2610  Cond);
2611  if (N0.getNode()->hasOneUse()) {
2612  assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
2613  auto &DL = DAG.getDataLayout();
2614  // (Z-X) == X --> Z == X<<1
2615  SDValue SH = DAG.getNode(
2616  ISD::SHL, dl, N1.getValueType(), N1,
2617  DAG.getConstant(1, dl,
2618  getShiftAmountTy(N1.getValueType(), DL,
2619  !DCI.isBeforeLegalize())));
2620  if (!DCI.isCalledByLegalizer())
2621  DCI.AddToWorklist(SH.getNode());
2622  return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond);
2623  }
2624  }
2625  }
2626  }
2627 
2628  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
2629  N1.getOpcode() == ISD::XOR) {
2630  // Simplify X == (X+Z) --> Z == 0
2631  if (N1.getOperand(0) == N0)
2632  return DAG.getSetCC(dl, VT, N1.getOperand(1),
2633  DAG.getConstant(0, dl, N1.getValueType()), Cond);
2634  if (N1.getOperand(1) == N0) {
2635  if (isCommutativeBinOp(N1.getOpcode()))
2636  return DAG.getSetCC(dl, VT, N1.getOperand(0),
2637  DAG.getConstant(0, dl, N1.getValueType()), Cond);
2638  if (N1.getNode()->hasOneUse()) {
2639  assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
2640  auto &DL = DAG.getDataLayout();
2641  // X == (Z-X) --> X<<1 == Z
2642  SDValue SH = DAG.getNode(
2643  ISD::SHL, dl, N1.getValueType(), N0,
2644  DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL,
2645  !DCI.isBeforeLegalize())));
2646  if (!DCI.isCalledByLegalizer())
2647  DCI.AddToWorklist(SH.getNode());
2648  return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond);
2649  }
2650  }
2651  }
2652 
2653  if (SDValue V = simplifySetCCWithAnd(VT, N0, N1, Cond, DCI, dl))
2654  return V;
2655  }
2656 
2657  // Fold away ALL boolean setcc's.
2658  SDValue Temp;
2659  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
2660  EVT OpVT = N0.getValueType();
2661  switch (Cond) {
2662  default: llvm_unreachable("Unknown integer setcc!");
2663  case ISD::SETEQ: // X == Y -> ~(X^Y)
2664  Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
2665  N0 = DAG.getNOT(dl, Temp, OpVT);
2666  if (!DCI.isCalledByLegalizer())
2667  DCI.AddToWorklist(Temp.getNode());
2668  break;
2669  case ISD::SETNE: // X != Y --> (X^Y)
2670  N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
2671  break;
2672  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
2673  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
2674  Temp = DAG.getNOT(dl, N0, OpVT);
2675  N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
2676  if (!DCI.isCalledByLegalizer())
2677  DCI.AddToWorklist(Temp.getNode());
2678  break;
2679  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
2680  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
2681  Temp = DAG.getNOT(dl, N1, OpVT);
2682  N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
2683  if (!DCI.isCalledByLegalizer())
2684  DCI.AddToWorklist(Temp.getNode());
2685  break;
2686  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
2687  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
2688  Temp = DAG.getNOT(dl, N0, OpVT);
2689  N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
2690  if (!DCI.isCalledByLegalizer())
2691  DCI.AddToWorklist(Temp.getNode());
2692  break;
2693  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
2694  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
2695  Temp = DAG.getNOT(dl, N1, OpVT);
2696  N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
2697  break;
2698  }
2699  if (VT.getScalarType() != MVT::i1) {
2700  if (!DCI.isCalledByLegalizer())
2701  DCI.AddToWorklist(N0.getNode());
2702  // FIXME: If running after legalize, we probably can't do this.
2704  N0 = DAG.getNode(ExtendCode, dl, VT, N0);
2705  }
2706  return N0;
2707  }
2708 
2709  // Could not fold it.
2710  return SDValue();
2711 }
2712 
2713 /// Returns true (and the GlobalValue and the offset) if the node is a
2714 /// GlobalAddress + offset.
2716  int64_t &Offset) const {
2717  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
2718  GA = GASD->getGlobal();
2719  Offset += GASD->getOffset();
2720  return true;
2721  }
2722 
2723  if (N->getOpcode() == ISD::ADD) {
2724  SDValue N1 = N->getOperand(0);
2725  SDValue N2 = N->getOperand(1);
2726  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
2727  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
2728  Offset += V->getSExtValue();
2729  return true;
2730  }
2731  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
2732  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
2733  Offset += V->getSExtValue();
2734  return true;
2735  }
2736  }
2737  }
2738 
2739  return false;
2740 }
2741 
2743  DAGCombinerInfo &DCI) const {
2744  // Default implementation: no optimization.
2745  return SDValue();
2746 }
2747 
2748 //===----------------------------------------------------------------------===//
2749 // Inline Assembler Implementation Methods
2750 //===----------------------------------------------------------------------===//
2751 
2754  unsigned S = Constraint.size();
2755 
2756  if (S == 1) {
2757  switch (Constraint[0]) {
2758  default: break;
2759  case 'r': return C_RegisterClass;
2760  case 'm': // memory
2761  case 'o': // offsetable
2762  case 'V': // not offsetable
2763  return C_Memory;
2764  case 'i': // Simple Integer or Relocatable Constant
2765  case 'n': // Simple Integer
2766  case 'E': // Floating Point Constant
2767  case 'F': // Floating Point Constant
2768  case 's': // Relocatable Constant
2769  case 'p': // Address.
2770  case 'X': // Allow ANY value.
2771  case 'I': // Target registers.
2772  case 'J':
2773  case 'K':
2774  case 'L':
2775  case 'M':
2776  case 'N':
2777  case 'O':
2778  case 'P':
2779  case '<':
2780  case '>':
2781  return C_Other;
2782  }
2783  }
2784 
2785  if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') {
2786  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
2787  return C_Memory;
2788  return C_Register;
2789  }
2790  return C_Unknown;
2791 }
2792 
2793 /// Try to replace an X constraint, which matches anything, with another that
2794 /// has more specific requirements based on the type of the corresponding
2795 /// operand.
2796 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const{
2797  if (ConstraintVT.isInteger())
2798  return "r";
2799  if (ConstraintVT.isFloatingPoint())
2800  return "f"; // works for many targets
2801  return nullptr;
2802 }
2803 
2804 /// Lower the specified operand into the Ops vector.
2805 /// If it is invalid, don't add anything to Ops.
2807  std::string &Constraint,
2808  std::vector<SDValue> &Ops,
2809  SelectionDAG &DAG) const {
2810 
2811  if (Constraint.length() > 1) return;
2812 
2813  char ConstraintLetter = Constraint[0];
2814  switch (ConstraintLetter) {
2815  default: break;
2816  case 'X': // Allows any operand; labels (basic block) use this.
2817  if (Op.getOpcode() == ISD::BasicBlock) {
2818  Ops.push_back(Op);
2819  return;
2820  }
2822  case 'i': // Simple Integer or Relocatable Constant
2823  case 'n': // Simple Integer
2824  case 's': { // Relocatable Constant
2825  // These operands are interested in values of the form (GV+C), where C may
2826  // be folded in as an offset of GV, or it may be explicitly added. Also, it
2827  // is possible and fine if either GV or C are missing.
2830 
2831  // If we have "(add GV, C)", pull out GV/C
2832  if (Op.getOpcode() == ISD::ADD) {
2833  C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2835  if (!C || !GA) {
2836  C = dyn_cast<ConstantSDNode>(Op.getOperand(0));
2838  }
2839  if (!C || !GA) {
2840  C = nullptr;
2841  GA = nullptr;
2842  }
2843  }
2844 
2845  // If we find a valid operand, map to the TargetXXX version so that the
2846  // value itself doesn't get selected.
2847  if (GA) { // Either &GV or &GV+C
2848  if (ConstraintLetter != 'n') {
2849  int64_t Offs = GA->getOffset();
2850  if (C) Offs += C->getZExtValue();
2851  Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(),
2852  C ? SDLoc(C) : SDLoc(),
2853  Op.getValueType(), Offs));
2854  }
2855  return;
2856  }
2857  if (C) { // just C, no GV.
2858  // Simple constants are not allowed for 's'.
2859  if (ConstraintLetter != 's') {
2860  // gcc prints these as sign extended. Sign extend value to 64 bits
2861  // now; without this it would get ZExt'd later in
2862  // ScheduleDAGSDNodes::EmitNode, which is very generic.
2863  Ops.push_back(DAG.getTargetConstant(C->getSExtValue(),
2864  SDLoc(C), MVT::i64));
2865  }
2866  return;
2867  }
2868  break;
2869  }
2870  }
2871 }
2872 
2873 std::pair<unsigned, const TargetRegisterClass *>
2875  StringRef Constraint,
2876  MVT VT) const {
2877  if (Constraint.empty() || Constraint[0] != '{')
2878  return std::make_pair(0u, static_cast<TargetRegisterClass*>(nullptr));
2879  assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?");
2880 
2881  // Remove the braces from around the name.
2882  StringRef RegName(Constraint.data()+1, Constraint.size()-2);
2883 
2884  std::pair<unsigned, const TargetRegisterClass*> R =
2885  std::make_pair(0u, static_cast<const TargetRegisterClass*>(nullptr));
2886 
2887  // Figure out which register class contains this reg.
2888  for (const TargetRegisterClass *RC : RI->regclasses()) {
2889  // If none of the value types for this register class are valid, we
2890  // can't use it. For example, 64-bit reg classes on 32-bit targets.
2891  if (!isLegalRC(*RI, *RC))
2892  continue;
2893 
2894  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
2895  I != E; ++I) {
2896  if (RegName.equals_lower(RI->getRegAsmName(*I))) {
2897  std::pair<unsigned, const TargetRegisterClass*> S =
2898  std::make_pair(*I, RC);
2899 
2900  // If this register class has the requested value type, return it,
2901  // otherwise keep searching and return the first class found
2902  // if no other is found which explicitly has the requested type.
2903  if (RI->isTypeLegalForClass(*RC, VT))
2904  return S;
2905  if (!R.second)
2906  R = S;
2907  }
2908  }
2909  }
2910 
2911  return R;
2912 }
2913 
2914 //===----------------------------------------------------------------------===//
2915 // Constraint Selection.
2916 
2917 /// Return true of this is an input operand that is a matching constraint like
2918 /// "4".
2920  assert(!ConstraintCode.empty() && "No known constraint!");
2921  return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
2922 }
2923 
2924 /// If this is an input matching constraint, this method returns the output
2925 /// operand it matches.
2927  assert(!ConstraintCode.empty() && "No known constraint!");
2928  return atoi(ConstraintCode.c_str());
2929 }
2930 
2931 /// Split up the constraint string from the inline assembly value into the
2932 /// specific constraints and their prefixes, and also tie in the associated
2933 /// operand values.
2934 /// If this returns an empty vector, and if the constraint string itself
2935 /// isn't empty, there was an error parsing.
2938  const TargetRegisterInfo *TRI,
2939  ImmutableCallSite CS) const {
2940  /// Information about all of the constraints.
2941  AsmOperandInfoVector ConstraintOperands;
2942  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
2943  unsigned maCount = 0; // Largest number of multiple alternative constraints.
2944 
2945  // Do a prepass over the constraints, canonicalizing them, and building up the
2946  // ConstraintOperands list.
2947  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
2948  unsigned ResNo = 0; // ResNo - The result number of the next output.
2949 
2950  for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
2951  ConstraintOperands.emplace_back(std::move(CI));
2952  AsmOperandInfo &OpInfo = ConstraintOperands.back();
2953 
2954  // Update multiple alternative constraint count.
2955  if (OpInfo.multipleAlternatives.size() > maCount)
2956  maCount = OpInfo.multipleAlternatives.size();
2957 
2958  OpInfo.ConstraintVT = MVT::Other;
2959 
2960  // Compute the value type for each operand.
2961  switch (OpInfo.Type) {
2962  case InlineAsm::isOutput:
2963  // Indirect outputs just consume an argument.
2964  if (OpInfo.isIndirect) {
2965  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
2966  break;
2967  }
2968 
2969  // The return value of the call is this value. As such, there is no
2970  // corresponding argument.
2971  assert(!CS.getType()->isVoidTy() &&
2972  "Bad inline asm!");
2973  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
2974  OpInfo.ConstraintVT =
2975  getSimpleValueType(DL, STy->getElementType(ResNo));
2976  } else {
2977  assert(ResNo == 0 && "Asm only has one result!");
2978  OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
2979  }
2980  ++ResNo;
2981  break;
2982  case InlineAsm::isInput:
2983  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
2984  break;
2985  case InlineAsm::isClobber:
2986  // Nothing to do.
2987  break;
2988  }
2989 
2990  if (OpInfo.CallOperandVal) {
2991  llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
2992  if (OpInfo.isIndirect) {
2993  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
2994  if (!PtrTy)
2995  report_fatal_error("Indirect operand for inline asm not a pointer!");
2996  OpTy = PtrTy->getElementType();
2997  }
2998 
2999  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
3000  if (StructType *STy = dyn_cast<StructType>(OpTy))
3001  if (STy->getNumElements() == 1)
3002  OpTy = STy->getElementType(0);
3003 
3004  // If OpTy is not a single value, it may be a struct/union that we
3005  // can tile with integers.
3006  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
3007  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
3008  switch (BitSize) {
3009  default: break;
3010  case 1:
3011  case 8:
3012  case 16:
3013  case 32:
3014  case 64:
3015  case 128:
3016  OpInfo.ConstraintVT =
3017  MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
3018  break;
3019  }
3020  } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
3021  unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
3022  OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
3023  } else {
3024  OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
3025  }
3026  }
3027  }
3028 
3029  // If we have multiple alternative constraints, select the best alternative.
3030  if (!ConstraintOperands.empty()) {
3031  if (maCount) {
3032  unsigned bestMAIndex = 0;
3033  int bestWeight = -1;
3034  // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
3035  int weight = -1;
3036  unsigned maIndex;
3037  // Compute the sums of the weights for each alternative, keeping track
3038  // of the best (highest weight) one so far.
3039  for (maIndex = 0; maIndex < maCount; ++maIndex) {
3040  int weightSum = 0;
3041  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3042  cIndex != eIndex; ++cIndex) {
3043  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
3044  if (OpInfo.Type == InlineAsm::isClobber)
3045  continue;
3046 
3047  // If this is an output operand with a matching input operand,
3048  // look up the matching input. If their types mismatch, e.g. one
3049  // is an integer, the other is floating point, or their sizes are
3050  // different, flag it as an maCantMatch.
3051  if (OpInfo.hasMatchingInput()) {
3052  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3053  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3054  if ((OpInfo.ConstraintVT.isInteger() !=
3055  Input.ConstraintVT.isInteger()) ||
3056  (OpInfo.ConstraintVT.getSizeInBits() !=
3057  Input.ConstraintVT.getSizeInBits())) {
3058  weightSum = -1; // Can't match.
3059  break;
3060  }
3061  }
3062  }
3063  weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
3064  if (weight == -1) {
3065  weightSum = -1;
3066  break;
3067  }
3068  weightSum += weight;
3069  }
3070  // Update best.
3071  if (weightSum > bestWeight) {
3072  bestWeight = weightSum;
3073  bestMAIndex = maIndex;
3074  }
3075  }
3076 
3077  // Now select chosen alternative in each constraint.
3078  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3079  cIndex != eIndex; ++cIndex) {
3080  AsmOperandInfo& cInfo = ConstraintOperands[cIndex];
3081  if (cInfo.Type == InlineAsm::isClobber)
3082  continue;
3083  cInfo.selectAlternative(bestMAIndex);
3084  }
3085  }
3086  }
3087 
3088  // Check and hook up tied operands, choose constraint code to use.
3089  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3090  cIndex != eIndex; ++cIndex) {
3091  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
3092 
3093  // If this is an output operand with a matching input operand, look up the
3094  // matching input. If their types mismatch, e.g. one is an integer, the
3095  // other is floating point, or their sizes are different, flag it as an
3096  // error.
3097  if (OpInfo.hasMatchingInput()) {
3098  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3099 
3100  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3101  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
3103  OpInfo.ConstraintVT);
3104  std::pair<unsigned, const TargetRegisterClass *> InputRC =
3106  Input.ConstraintVT);
3107  if ((OpInfo.ConstraintVT.isInteger() !=
3108  Input.ConstraintVT.isInteger()) ||
3109  (MatchRC.second != InputRC.second)) {
3110  report_fatal_error("Unsupported asm: input constraint"
3111  " with a matching output constraint of"
3112  " incompatible type!");
3113  }
3114  }
3115  }
3116  }
3117 
3118  return ConstraintOperands;
3119 }
3120 
3121 /// Return an integer indicating how general CT is.
3123  switch (CT) {
3126  return 0;
3128  return 1;
3130  return 2;
3132  return 3;
3133  }
3134  llvm_unreachable("Invalid constraint type");
3135 }
3136 
3137 /// Examine constraint type and operand type and determine a weight value.
3138 /// This object must already have been set up with the operand type
3139 /// and the current alternative constraint selected.
3142  AsmOperandInfo &info, int maIndex) const {
3144  if (maIndex >= (int)info.multipleAlternatives.size())
3145  rCodes = &info.Codes;
3146  else
3147  rCodes = &info.multipleAlternatives[maIndex].Codes;
3148  ConstraintWeight BestWeight = CW_Invalid;
3149 
3150  // Loop over the options, keeping track of the most general one.
3151  for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
3152  ConstraintWeight weight =
3153  getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
3154  if (weight > BestWeight)
3155  BestWeight = weight;
3156  }
3157 
3158  return BestWeight;
3159 }
3160 
3161 /// Examine constraint type and operand type and determine a weight value.
3162 /// This object must already have been set up with the operand type
3163 /// and the current alternative constraint selected.
3166  AsmOperandInfo &info, const char *constraint) const {
3167  ConstraintWeight weight = CW_Invalid;
3168  Value *CallOperandVal = info.CallOperandVal;
3169  // If we don't have a value, we can't do a match,
3170  // but allow it at the lowest weight.
3171  if (!CallOperandVal)
3172  return CW_Default;
3173  // Look at the constraint type.
3174  switch (*constraint) {
3175  case 'i': // immediate integer.
3176  case 'n': // immediate integer with a known value.
3177  if (isa<ConstantInt>(CallOperandVal))
3178  weight = CW_Constant;
3179  break;
3180  case 's': // non-explicit intregal immediate.
3181  if (isa<GlobalValue>(CallOperandVal))
3182  weight = CW_Constant;
3183  break;
3184  case 'E': // immediate float if host format.
3185  case 'F': // immediate float.
3186  if (isa<ConstantFP>(CallOperandVal))
3187  weight = CW_Constant;
3188  break;
3189  case '<': // memory operand with autodecrement.
3190  case '>': // memory operand with autoincrement.
3191  case 'm': // memory operand.
3192  case 'o': // offsettable memory operand
3193  case 'V': // non-offsettable memory operand
3194  weight = CW_Memory;
3195  break;
3196  case 'r': // general register.
3197  case 'g': // general register, memory operand or immediate integer.
3198  // note: Clang converts "g" to "imr".
3199  if (CallOperandVal->getType()->isIntegerTy())
3200  weight = CW_Register;
3201  break;
3202  case 'X': // any operand.
3203  default:
3204  weight = CW_Default;
3205  break;
3206  }
3207  return weight;
3208 }
3209 
3210 /// If there are multiple different constraints that we could pick for this
3211 /// operand (e.g. "imr") try to pick the 'best' one.
3212 /// This is somewhat tricky: constraints fall into four classes:
3213 /// Other -> immediates and magic values
3214 /// Register -> one specific register
3215 /// RegisterClass -> a group of regs
3216 /// Memory -> memory
3217 /// Ideally, we would pick the most specific constraint possible: if we have
3218 /// something that fits into a register, we would pick it. The problem here
3219 /// is that if we have something that could either be in a register or in
3220 /// memory that use of the register could cause selection of *other*
3221 /// operands to fail: they might only succeed if we pick memory. Because of
3222 /// this the heuristic we use is:
3223 ///
3224 /// 1) If there is an 'other' constraint, and if the operand is valid for
3225 /// that constraint, use it. This makes us take advantage of 'i'
3226 /// constraints when available.
3227 /// 2) Otherwise, pick the most general constraint present. This prefers
3228 /// 'm' over 'r', for example.
3229 ///
3231  const TargetLowering &TLI,
3232  SDValue Op, SelectionDAG *DAG) {
3233  assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
3234  unsigned BestIdx = 0;
3236  int BestGenerality = -1;
3237 
3238  // Loop over the options, keeping track of the most general one.
3239  for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
3241  TLI.getConstraintType(OpInfo.Codes[i]);
3242 
3243  // If this is an 'other' constraint, see if the operand is valid for it.
3244  // For example, on X86 we might have an 'rI' constraint. If the operand
3245  // is an integer in the range [0..31] we want to use I (saving a load
3246  // of a register), otherwise we must use 'r'.
3247  if (CType == TargetLowering::C_Other && Op.getNode()) {
3248  assert(OpInfo.Codes[i].size() == 1 &&
3249  "Unhandled multi-letter 'other' constraint");
3250  std::vector<SDValue> ResultOps;
3251  TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
3252  ResultOps, *DAG);
3253  if (!ResultOps.empty()) {
3254  BestType = CType;
3255  BestIdx = i;
3256  break;
3257  }
3258  }
3259 
3260  // Things with matching constraints can only be registers, per gcc
3261  // documentation. This mainly affects "g" constraints.
3262  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
3263  continue;
3264 
3265  // This constraint letter is more general than the previous one, use it.
3266  int Generality = getConstraintGenerality(CType);
3267  if (Generality > BestGenerality) {
3268  BestType = CType;
3269  BestIdx = i;
3270  BestGenerality = Generality;
3271  }
3272  }
3273 
3274  OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
3275  OpInfo.ConstraintType = BestType;
3276 }
3277 
3278 /// Determines the constraint code and constraint type to use for the specific
3279 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
3281  SDValue Op,
3282  SelectionDAG *DAG) const {
3283  assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
3284 
3285  // Single-letter constraints ('r') are very common.
3286  if (OpInfo.Codes.size() == 1) {
3287  OpInfo.ConstraintCode = OpInfo.Codes[0];
3289  } else {
3290  ChooseConstraint(OpInfo, *this, Op, DAG);
3291  }
3292 
3293  // 'X' matches anything.
3294  if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
3295  // Labels and constants are handled elsewhere ('X' is the only thing
3296  // that matches labels). For Functions, the type here is the type of
3297  // the result, which is not what we want to look at; leave them alone.
3298  Value *v = OpInfo.CallOperandVal;
3299  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
3300  OpInfo.CallOperandVal = v;
3301  return;
3302  }
3303 
3304  // Otherwise, try to resolve it to something we know about by looking at
3305  // the actual operand type.
3306  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
3307  OpInfo.ConstraintCode = Repl;
3309  }
3310  }
3311 }
3312 
3313 /// Given an exact SDIV by a constant, create a multiplication
3314 /// with the multiplicative inverse of the constant.
3316  const SDLoc &dl, SelectionDAG &DAG,
3317  std::vector<SDNode *> &Created) {
3318  assert(d != 0 && "Division by zero!");
3319 
3320  // Shift the value upfront if it is even, so the LSB is one.
3321  unsigned ShAmt = d.countTrailingZeros();
3322  if (ShAmt) {
3323  // TODO: For UDIV use SRL instead of SRA.
3324  SDValue Amt =
3325  DAG.getConstant(ShAmt, dl, TLI.getShiftAmountTy(Op1.getValueType(),
3326  DAG.getDataLayout()));
3327  SDNodeFlags Flags;
3328  Flags.setExact(true);
3329  Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, Flags);
3330  Created.push_back(Op1.getNode());
3331  d.ashrInPlace(ShAmt);
3332  }
3333 
3334  // Calculate the multiplicative inverse, using Newton's method.
3335  APInt t, xn = d;
3336  while ((t = d*xn) != 1)
3337  xn *= APInt(d.getBitWidth(), 2) - t;
3338 
3339  SDValue Op2 = DAG.getConstant(xn, dl, Op1.getValueType());
3340  SDValue Mul = DAG.getNode(ISD::MUL, dl, Op1.getValueType(), Op1, Op2);
3341  Created.push_back(Mul.getNode());
3342  return Mul;
3343 }
3344 
3346  SelectionDAG &DAG,
3347  std::vector<SDNode *> *Created) const {
3349  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3350  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
3351  return SDValue(N,0); // Lower SDIV as SDIV
3352  return SDValue();
3353 }
3354 
3355 /// Given an ISD::SDIV node expressing a divide by constant,
3356 /// return a DAG expression to select that will generate the same value by
3357 /// multiplying by a magic number.
3358 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3360  SelectionDAG &DAG, bool IsAfterLegalization,
3361  std::vector<SDNode *> *Created) const {
3362  assert(Created && "No vector to hold sdiv ops.");
3363 
3364  EVT VT = N->getValueType(0);
3365  SDLoc dl(N);
3366 
3367  // Check to see if we can do this.
3368  // FIXME: We should be more aggressive here.
3369  if (!isTypeLegal(VT))
3370  return SDValue();
3371 
3372  // If the sdiv has an 'exact' bit we can use a simpler lowering.
3373  if (N->getFlags().hasExact())
3374  return BuildExactSDIV(*this, N->getOperand(0), Divisor, dl, DAG, *Created);
3375 
3376  APInt::ms magics = Divisor.magic();
3377 
3378  // Multiply the numerator (operand 0) by the magic value
3379  // FIXME: We should support doing a MUL in a wider type
3380  SDValue Q;
3381  if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT) :
3383  Q = DAG.getNode(ISD::MULHS, dl, VT, N->getOperand(0),
3384  DAG.getConstant(magics.m, dl, VT));
3385  else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT) :
3387  Q = SDValue(DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT),
3388  N->getOperand(0),
3389  DAG.getConstant(magics.m, dl, VT)).getNode(), 1);
3390  else
3391  return SDValue(); // No mulhs or equvialent
3392  // If d > 0 and m < 0, add the numerator
3393  if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
3394  Q = DAG.getNode(ISD::ADD, dl, VT, Q, N->getOperand(0));
3395  Created->push_back(Q.getNode());
3396  }
3397  // If d < 0 and m > 0, subtract the numerator.
3398  if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
3399  Q = DAG.getNode(ISD::SUB, dl, VT, Q, N->getOperand(0));
3400  Created->push_back(Q.getNode());
3401  }
3402  auto &DL = DAG.getDataLayout();
3403  // Shift right algebraic if shift value is nonzero
3404  if (magics.s > 0) {
3405  Q = DAG.getNode(
3406  ISD::SRA, dl, VT, Q,
3407  DAG.getConstant(magics.s, dl, getShiftAmountTy(Q.getValueType(), DL)));
3408  Created->push_back(Q.getNode());
3409  }
3410  // Extract the sign bit and add it to the quotient
3411  SDValue T =
3412  DAG.getNode(ISD::SRL, dl, VT, Q,
3413  DAG.getConstant(VT.getScalarSizeInBits() - 1, dl,
3414  getShiftAmountTy(Q.getValueType(), DL)));
3415  Created->push_back(T.getNode());
3416  return DAG.getNode(ISD::ADD, dl, VT, Q, T);
3417 }
3418 
3419 /// Given an ISD::UDIV node expressing a divide by constant,
3420 /// return a DAG expression to select that will generate the same value by
3421 /// multiplying by a magic number.
3422 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3424  SelectionDAG &DAG, bool IsAfterLegalization,
3425  std::vector<SDNode *> *Created) const {
3426  assert(Created && "No vector to hold udiv ops.");
3427 
3428  EVT VT = N->getValueType(0);
3429  SDLoc dl(N);
3430  auto &DL = DAG.getDataLayout();
3431 
3432  // Check to see if we can do this.
3433  // FIXME: We should be more aggressive here.
3434  if (!isTypeLegal(VT))
3435  return SDValue();
3436 
3437  // FIXME: We should use a narrower constant when the upper
3438  // bits are known to be zero.
3439  APInt::mu magics = Divisor.magicu();
3440 
3441  SDValue Q = N->getOperand(0);
3442 
3443  // If the divisor is even, we can avoid using the expensive fixup by shifting
3444  // the divided value upfront.
3445  if (magics.a != 0 && !Divisor[0]) {
3446  unsigned Shift = Divisor.countTrailingZeros();
3447  Q = DAG.getNode(
3448  ISD::SRL, dl, VT, Q,
3449  DAG.getConstant(Shift, dl, getShiftAmountTy(Q.getValueType(), DL)));
3450  Created->push_back(Q.getNode());
3451 
3452  // Get magic number for the shifted divisor.
3453  magics = Divisor.lshr(Shift).magicu(Shift);
3454  assert(magics.a == 0 && "Should use cheap fixup now");
3455  }
3456 
3457  // Multiply the numerator (operand 0) by the magic value
3458  // FIXME: We should support doing a MUL in a wider type
3459  if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT) :
3461  Q = DAG.getNode(ISD::MULHU, dl, VT, Q, DAG.getConstant(magics.m, dl, VT));
3462  else if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT) :
3464  Q = SDValue(DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), Q,
3465  DAG.getConstant(magics.m, dl, VT)).getNode(), 1);
3466  else
3467  return SDValue(); // No mulhu or equivalent
3468 
3469  Created->push_back(Q.getNode());
3470 
3471  if (magics.a == 0) {
3472  assert(magics.s < Divisor.getBitWidth() &&
3473  "We shouldn't generate an undefined shift!");
3474  return DAG.getNode(
3475  ISD::SRL, dl, VT, Q,
3476  DAG.getConstant(magics.s, dl, getShiftAmountTy(Q.getValueType(), DL)));
3477  } else {
3478  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N->getOperand(0), Q);
3479  Created->push_back(NPQ.getNode());
3480  NPQ = DAG.getNode(
3481  ISD::SRL, dl, VT, NPQ,
3482  DAG.getConstant(1, dl, getShiftAmountTy(NPQ.getValueType(), DL)));
3483  Created->push_back(NPQ.getNode());
3484  NPQ = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
3485  Created->push_back(NPQ.getNode());
3486  return DAG.getNode(
3487  ISD::SRL, dl, VT, NPQ,
3488  DAG.getConstant(magics.s - 1, dl,
3489  getShiftAmountTy(NPQ.getValueType(), DL)));
3490  }
3491 }
3492 
3493 bool TargetLowering::
3495  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
3496  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
3497  "be a constant integer");
3498  return true;
3499  }
3500 
3501  return false;
3502 }
3503 
3504 //===----------------------------------------------------------------------===//
3505 // Legalization Utilities
3506 //===----------------------------------------------------------------------===//
3507 
3508 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
3509  SDValue LHS, SDValue RHS,
3510  SmallVectorImpl<SDValue> &Result,
3511  EVT HiLoVT, SelectionDAG &DAG,
3513  SDValue LH, SDValue RL, SDValue RH) const {
3514  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
3515  Opcode == ISD::SMUL_LOHI);
3516 
3517  bool HasMULHS = (Kind == MulExpansionKind::Always) ||
3519  bool HasMULHU = (Kind == MulExpansionKind::Always) ||
3521  bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
3523  bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
3525 
3526  if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
3527  return false;
3528 
3529  unsigned OuterBitSize = VT.getScalarSizeInBits();
3530  unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
3531  unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
3532  unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
3533 
3534  // LL, LH, RL, and RH must be either all NULL or all set to a value.
3535  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
3536  (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
3537 
3538  SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
3539  auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
3540  bool Signed) -> bool {
3541  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
3542  Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
3543  Hi = SDValue(Lo.getNode(), 1);
3544  return true;
3545  }
3546  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
3547  Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
3548  Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
3549  return true;
3550  }
3551  return false;
3552  };
3553 
3554  SDValue Lo, Hi;
3555 
3556  if (!LL.getNode() && !RL.getNode() &&
3558  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
3559  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
3560  }
3561 
3562  if (!LL.getNode())
3563  return false;
3564 
3565  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
3566  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
3567  DAG.MaskedValueIsZero(RHS, HighMask)) {
3568  // The inputs are both zero-extended.
3569  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
3570  Result.push_back(Lo);
3571  Result.push_back(Hi);
3572  if (Opcode != ISD::MUL) {
3573  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
3574  Result.push_back(Zero);
3575  Result.push_back(Zero);
3576  }
3577  return true;
3578  }
3579  }
3580 
3581  if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
3582  RHSSB > InnerBitSize) {
3583  // The input values are both sign-extended.
3584  // TODO non-MUL case?
3585  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
3586  Result.push_back(Lo);
3587  Result.push_back(Hi);
3588  return true;
3589  }
3590  }
3591 
3592  unsigned ShiftAmount = OuterBitSize - InnerBitSize;
3593  EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
3594  if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
3595  // FIXME getShiftAmountTy does not always return a sensible result when VT
3596  // is an illegal type, and so the type may be too small to fit the shift
3597  // amount. Override it with i32. The shift will have to be legalized.
3598  ShiftAmountTy = MVT::i32;
3599  }
3600  SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
3601 
3602  if (!LH.getNode() && !RH.getNode() &&
3605  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
3606  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
3607  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
3608  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
3609  }
3610 
3611  if (!LH.getNode())
3612  return false;
3613 
3614  if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
3615  return false;
3616 
3617  Result.push_back(Lo);
3618 
3619  if (Opcode == ISD::MUL) {
3620  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
3621  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
3622  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
3623  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
3624  Result.push_back(Hi);
3625  return true;
3626  }
3627 
3628  // Compute the full width result.
3629  auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
3630  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3631  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
3632  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3633  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
3634  };
3635 
3636  SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
3637  if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
3638  return false;
3639 
3640  // This is effectively the add part of a multiply-add of half-sized operands,
3641  // so it cannot overflow.
3642  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
3643 
3644  if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
3645  return false;
3646 
3647  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
3648  Merge(Lo, Hi));
3649 
3650  SDValue Carry = Next.getValue(1);
3651  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3652  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
3653 
3654  if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
3655  return false;
3656 
3657  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
3658  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
3659  Carry);
3660  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
3661 
3662  if (Opcode == ISD::SMUL_LOHI) {
3663  SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
3664  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
3665  Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
3666 
3667  NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
3668  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
3669  Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
3670  }
3671 
3672  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3673  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
3674  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
3675  return true;
3676 }
3677 
3680  SDValue LL, SDValue LH, SDValue RL,
3681  SDValue RH) const {
3682  SmallVector<SDValue, 2> Result;
3683  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
3684  N->getOperand(0), N->getOperand(1), Result, HiLoVT,
3685  DAG, Kind, LL, LH, RL, RH);
3686  if (Ok) {
3687  assert(Result.size() == 2);
3688  Lo = Result[0];
3689  Hi = Result[1];
3690  }
3691  return Ok;
3692 }
3693 
3695  SelectionDAG &DAG) const {
3696  EVT VT = Node->getOperand(0).getValueType();
3697  EVT NVT = Node->getValueType(0);
3698  SDLoc dl(SDValue(Node, 0));
3699 
3700  // FIXME: Only f32 to i64 conversions are supported.
3701  if (VT != MVT::f32 || NVT != MVT::i64)
3702  return false;
3703 
3704  // Expand f32 -> i64 conversion
3705  // This algorithm comes from compiler-rt's implementation of fixsfdi:
3706  // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c
3707  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(),
3708  VT.getSizeInBits());
3709  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
3710  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
3711  SDValue Bias = DAG.getConstant(127, dl, IntVT);
3712  SDValue SignMask = DAG.getConstant(APInt::getSignMask(VT.getSizeInBits()), dl,
3713  IntVT);
3714  SDValue SignLowBit = DAG.getConstant(VT.getSizeInBits() - 1, dl, IntVT);
3715  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
3716 
3717  SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Node->getOperand(0));
3718 
3719  auto &DL = DAG.getDataLayout();
3720  SDValue ExponentBits = DAG.getNode(
3721  ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
3722  DAG.getZExtOrTrunc(ExponentLoBit, dl, getShiftAmountTy(IntVT, DL)));
3723  SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
3724 
3725  SDValue Sign = DAG.getNode(
3726  ISD::SRA, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
3727  DAG.getZExtOrTrunc(SignLowBit, dl, getShiftAmountTy(IntVT, DL)));
3728  Sign = DAG.getSExtOrTrunc(Sign, dl, NVT);
3729 
3730  SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
3731  DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
3732  DAG.getConstant(0x00800000, dl, IntVT));
3733 
3734  R = DAG.getZExtOrTrunc(R, dl, NVT);
3735 
3736  R = DAG.getSelectCC(
3737  dl, Exponent, ExponentLoBit,
3738  DAG.getNode(ISD::SHL, dl, NVT, R,
3739  DAG.getZExtOrTrunc(
3740  DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
3741  dl, getShiftAmountTy(IntVT, DL))),
3742  DAG.getNode(ISD::SRL, dl, NVT, R,
3743  DAG.getZExtOrTrunc(
3744  DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
3745  dl, getShiftAmountTy(IntVT, DL))),
3746  ISD::SETGT);
3747 
3748  SDValue Ret = DAG.getNode(ISD::SUB, dl, NVT,
3749  DAG.getNode(ISD::XOR, dl, NVT, R, Sign),
3750  Sign);
3751 
3752  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
3753  DAG.getConstant(0, dl, NVT), Ret, ISD::SETLT);
3754  return true;
3755 }
3756 
3758  SelectionDAG &DAG) const {
3759  SDLoc SL(LD);
3760  SDValue Chain = LD->getChain();
3761  SDValue BasePTR = LD->getBasePtr();
3762  EVT SrcVT = LD->getMemoryVT();
3763  ISD::LoadExtType ExtType = LD->getExtensionType();
3764 
3765  unsigned NumElem = SrcVT.getVectorNumElements();
3766 
3767  EVT SrcEltVT = SrcVT.getScalarType();
3768  EVT DstEltVT = LD->getValueType(0).getScalarType();
3769 
3770  unsigned Stride = SrcEltVT.getSizeInBits() / 8;
3771  assert(SrcEltVT.isByteSized());
3772 
3773  EVT PtrVT = BasePTR.getValueType();
3774 
3776  SmallVector<SDValue, 8> LoadChains;
3777 
3778  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
3779  SDValue ScalarLoad =
3780  DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
3781  LD->getPointerInfo().getWithOffset(Idx * Stride),
3782  SrcEltVT, MinAlign(LD->getAlignment(), Idx * Stride),
3783  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3784 
3785  BasePTR = DAG.getNode(ISD::ADD, SL, PtrVT, BasePTR,
3786  DAG.getConstant(Stride, SL, PtrVT));
3787 
3788  Vals.push_back(ScalarLoad.getValue(0));
3789  LoadChains.push_back(ScalarLoad.getValue(1));
3790  }
3791 
3792  SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
3793  SDValue Value = DAG.getBuildVector(LD->getValueType(0), SL, Vals);
3794 
3795  return DAG.getMergeValues({ Value, NewChain }, SL);
3796 }
3797 
3799  SelectionDAG &DAG) const {
3800  SDLoc SL(ST);
3801 
3802  SDValue Chain = ST->getChain();
3803  SDValue BasePtr = ST->getBasePtr();
3804  SDValue Value = ST->getValue();
3805  EVT StVT = ST->getMemoryVT();
3806 
3807  // The type of the data we want to save
3808  EVT RegVT = Value.getValueType();
3809  EVT RegSclVT = RegVT.getScalarType();
3810 
3811  // The type of data as saved in memory.
3812  EVT MemSclVT = StVT.getScalarType();
3813 
3814  EVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
3815  unsigned NumElem = StVT.getVectorNumElements();
3816 
3817  // A vector must always be stored in memory as-is, i.e. without any padding
3818  // between the elements, since various code depend on it, e.g. in the
3819  // handling of a bitcast of a vector type to int, which may be done with a
3820  // vector store followed by an integer load. A vector that does not have
3821  // elements that are byte-sized must therefore be stored as an integer
3822  // built out of the extracted vector elements.
3823  if (!MemSclVT.isByteSized()) {
3824  unsigned NumBits = StVT.getSizeInBits();
3825  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
3826 
3827  SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
3828 
3829  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
3830  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
3831  DAG.getConstant(Idx, SL, IdxVT));
3832  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
3833  SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
3834  unsigned ShiftIntoIdx =
3835  (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
3836  SDValue ShiftAmount =
3837  DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
3838  SDValue ShiftedElt =
3839  DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
3840  CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
3841  }
3842 
3843  return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
3844  ST->getAlignment(), ST->getMemOperand()->getFlags(),
3845  ST->getAAInfo());
3846  }
3847 
3848  // Store Stride in bytes
3849  unsigned Stride = MemSclVT.getSizeInBits() / 8;
3850  assert (Stride && "Zero stride!");
3851  // Extract each of the elements from the original vector and save them into
3852  // memory individually.
3853  SmallVector<SDValue, 8> Stores;
3854  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
3855  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
3856  DAG.getConstant(Idx, SL, IdxVT));
3857 
3858  SDValue Ptr = DAG.getObjectPtrOffset(SL, BasePtr, Idx * Stride);
3859 
3860  // This scalar TruncStore may be illegal, but we legalize it later.
3861  SDValue Store = DAG.getTruncStore(
3862  Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
3863  MemSclVT, MinAlign(ST->getAlignment(), Idx * Stride),
3864  ST->getMemOperand()->getFlags(), ST->getAAInfo());
3865 
3866  Stores.push_back(Store);
3867  }
3868 
3869  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
3870 }
3871 
3872 std::pair<SDValue, SDValue>
3875  "unaligned indexed loads not implemented!");
3876  SDValue Chain = LD->getChain();
3877  SDValue Ptr = LD->getBasePtr();
3878  EVT VT = LD->getValueType(0);
3879  EVT LoadedVT = LD->getMemoryVT();
3880  SDLoc dl(LD);
3881  auto &MF = DAG.getMachineFunction();
3882 
3883  if (VT.isFloatingPoint() || VT.isVector()) {
3884  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
3885  if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
3886  if (!isOperationLegalOrCustom(ISD::LOAD, intVT)) {
3887  // Scalarize the load and let the individual components be handled.
3888  SDValue Scalarized = scalarizeVectorLoad(LD, DAG);
3889  return std::make_pair(Scalarized.getValue(0), Scalarized.getValue(1));
3890  }
3891 
3892  // Expand to a (misaligned) integer load of the same size,
3893  // then bitconvert to floating point or vector.
3894  SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
3895  LD->getMemOperand());
3896  SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
3897  if (LoadedVT != VT)
3898  Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
3899  ISD::ANY_EXTEND, dl, VT, Result);
3900 
3901  return std::make_pair(Result, newLoad.getValue(1));
3902  }
3903 
3904  // Copy the value to a (aligned) stack slot using (unaligned) integer
3905  // loads and stores, then do a (aligned) load from the stack slot.
3906  MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
3907  unsigned LoadedBytes = LoadedVT.getStoreSize();
3908  unsigned RegBytes = RegVT.getSizeInBits() / 8;
3909  unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
3910 
3911  // Make sure the stack slot is also aligned for the register type.
3912  SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
3913  auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
3914  SmallVector<SDValue, 8> Stores;
3915  SDValue StackPtr = StackBase;
3916  unsigned Offset = 0;
3917 
3918  EVT PtrVT = Ptr.getValueType();
3919  EVT StackPtrVT = StackPtr.getValueType();
3920 
3921  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
3922  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
3923 
3924  // Do all but one copies using the full register width.
3925  for (unsigned i = 1; i < NumRegs; i++) {
3926  // Load one integer register's worth from the original location.
3927  SDValue Load = DAG.getLoad(
3928  RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
3930  LD->getAAInfo());
3931  // Follow the load with a store to the stack slot. Remember the store.
3932  Stores.push_back(DAG.getStore(
3933  Load.getValue(1), dl, Load, StackPtr,
3935  // Increment the pointers.
3936  Offset += RegBytes;
3937 
3938  Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
3939  StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
3940  }
3941 
3942  // The last copy may be partial. Do an extending load.
3943  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
3944  8 * (LoadedBytes - Offset));
3945  SDValue Load =
3946  DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
3947  LD->getPointerInfo().getWithOffset(Offset), MemVT,
3948  MinAlign(LD->getAlignment(), Offset),
3949  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3950  // Follow the load with a store to the stack slot. Remember the store.
3951  // On big-endian machines this requires a truncating store to ensure
3952  // that the bits end up in the right place.
3953  Stores.push_back(DAG.getTruncStore(
3954  Load.getValue(1), dl, Load, StackPtr,
3955  MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
3956 
3957  // The order of the stores doesn't matter - say it with a TokenFactor.
3958  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
3959 
3960  // Finally, perform the original load only redirected to the stack slot.
3961  Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
3963  LoadedVT);
3964 
3965  // Callers expect a MERGE_VALUES node.
3966  return std::make_pair(Load, TF);
3967  }
3968 
3969  assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
3970  "Unaligned load of unsupported type.");
3971 
3972  // Compute the new VT that is half the size of the old one. This is an
3973  // integer MVT.
3974  unsigned NumBits = LoadedVT.getSizeInBits();
3975  EVT NewLoadedVT;
3976  NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
3977  NumBits >>= 1;
3978 
3979  unsigned Alignment = LD->getAlignment();
3980  unsigned IncrementSize = NumBits / 8;
3981  ISD::LoadExtType HiExtType = LD->getExtensionType();
3982 
3983  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
3984  if (HiExtType == ISD::NON_EXTLOAD)
3985  HiExtType = ISD::ZEXTLOAD;
3986 
3987  // Load the value in two parts
3988  SDValue Lo, Hi;
3989  if (DAG.getDataLayout().isLittleEndian()) {
3990  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
3991  NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
3992  LD->getAAInfo());
3993 
3994  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3995  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
3996  LD->getPointerInfo().getWithOffset(IncrementSize),
3997  NewLoadedVT, MinAlign(Alignment, IncrementSize),
3998  LD->getMemOperand()->getFlags(), LD->getAAInfo());
3999  } else {
4000  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
4001  NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
4002  LD->getAAInfo());
4003 
4004  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4005  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
4006  LD->getPointerInfo().getWithOffset(IncrementSize),
4007  NewLoadedVT, MinAlign(Alignment, IncrementSize),
4008  LD->getMemOperand()->getFlags(), LD->getAAInfo());
4009  }
4010 
4011  // aggregate the two parts
4012  SDValue ShiftAmount =
4013  DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
4014  DAG.getDataLayout()));
4015  SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
4016  Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
4017 
4018  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
4019  Hi.getValue(1));
4020 
4021  return std::make_pair(Result, TF);
4022 }
4023 
4025  SelectionDAG &DAG) const {
4027  "unaligned indexed stores not implemented!");
4028  SDValue Chain = ST->getChain();
4029  SDValue Ptr = ST->getBasePtr();
4030  SDValue Val = ST->getValue();
4031  EVT VT = Val.getValueType();
4032  int Alignment = ST->getAlignment();
4033  auto &MF = DAG.getMachineFunction();
4034 
4035  SDLoc dl(ST);
4036  if (ST->getMemoryVT().isFloatingPoint() ||
4037  ST->getMemoryVT().isVector()) {
4038  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
4039  if (isTypeLegal(intVT)) {
4040  if (!isOperationLegalOrCustom(ISD::STORE, intVT)) {
4041  // Scalarize the store and let the individual components be handled.
4042  SDValue Result = scalarizeVectorStore(ST, DAG);
4043 
4044  return Result;
4045  }
4046  // Expand to a bitconvert of the value to the integer type of the
4047  // same size, then a (misaligned) int store.
4048  // FIXME: Does not handle truncating floating point stores!
4049  SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
4050  Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
4051  Alignment, ST->getMemOperand()->getFlags());
4052  return Result;
4053  }
4054  // Do a (aligned) store to a stack slot, then copy from the stack slot
4055  // to the final destination using (unaligned) integer loads and stores.
4056  EVT StoredVT = ST->getMemoryVT();
4057  MVT RegVT =
4058  getRegisterType(*DAG.getContext(),
4060  StoredVT.getSizeInBits()));
4061  EVT PtrVT = Ptr.getValueType();
4062  unsigned StoredBytes = StoredVT.getStoreSize();
4063  unsigned RegBytes = RegVT.getSizeInBits() / 8;
4064  unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
4065 
4066  // Make sure the stack slot is also aligned for the register type.
4067  SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT);
4068  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4069 
4070  // Perform the original store, only redirected to the stack slot.
4071  SDValue Store = DAG.getTruncStore(
4072  Chain, dl, Val, StackPtr,
4073  MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoredVT);
4074 
4075  EVT StackPtrVT = StackPtr.getValueType();
4076 
4077  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
4078  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
4079  SmallVector<SDValue, 8> Stores;
4080  unsigned Offset = 0;
4081 
4082  // Do all but one copies using the full register width.
4083  for (unsigned i = 1; i < NumRegs; i++) {
4084  // Load one integer register's worth from the stack slot.
4085  SDValue Load = DAG.getLoad(
4086  RegVT, dl, Store, StackPtr,
4088  // Store it to the final location. Remember the store.
4089  Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
4090  ST->getPointerInfo().getWithOffset(Offset),
4091  MinAlign(ST->getAlignment(), Offset),
4092  ST->getMemOperand()->getFlags()));
4093  // Increment the pointers.
4094  Offset += RegBytes;
4095  StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
4096  Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
4097  }
4098 
4099  // The last store may be partial. Do a truncating store. On big-endian
4100  // machines this requires an extending load from the stack slot to ensure
4101  // that the bits are in the right place.
4102  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
4103  8 * (StoredBytes - Offset));
4104 
4105  // Load from the stack slot.
4106  SDValue Load = DAG.getExtLoad(
4107  ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
4108  MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT);
4109 
4110  Stores.push_back(
4111  DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
4112  ST->getPointerInfo().getWithOffset(Offset), MemVT,
4113  MinAlign(ST->getAlignment(), Offset),
4114  ST->getMemOperand()->getFlags(), ST->getAAInfo()));
4115  // The order of the stores doesn't matter - say it with a TokenFactor.
4116  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
4117  return Result;
4118  }
4119 
4120  assert(ST->getMemoryVT().isInteger() &&
4121  !ST->getMemoryVT().isVector() &&
4122  "Unaligned store of unknown type.");
4123  // Get the half-size VT
4124  EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext());
4125  int NumBits = NewStoredVT.getSizeInBits();
4126  int IncrementSize = NumBits / 8;
4127 
4128  // Divide the stored value in two parts.
4129  SDValue ShiftAmount =
4130  DAG.getConstant(NumBits, dl, getShiftAmountTy(Val.getValueType(),
4131  DAG.getDataLayout()));
4132  SDValue Lo = Val;
4133  SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
4134 
4135  // Store the two parts
4136  SDValue Store1, Store2;
4137  Store1 = DAG.getTruncStore(Chain, dl,
4138  DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
4139  Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
4140  ST->getMemOperand()->getFlags());
4141 
4142  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
4143  Alignment = MinAlign(Alignment, IncrementSize);
4144  Store2 = DAG.getTruncStore(
4145  Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
4146  ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
4147  ST->getMemOperand()->getFlags(), ST->getAAInfo());
4148 
4149  SDValue Result =
4150  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
4151  return Result;
4152 }
4153 
4154 SDValue
4156  const SDLoc &DL, EVT DataVT,
4157  SelectionDAG &DAG,
4158  bool IsCompressedMemory) const {
4159  SDValue Increment;
4160  EVT AddrVT = Addr.getValueType();
4161  EVT MaskVT = Mask.getValueType();
4162  assert(DataVT.getVectorNumElements() == MaskVT.getVectorNumElements() &&
4163  "Incompatible types of Data and Mask");
4164  if (IsCompressedMemory) {
4165  // Incrementing the pointer according to number of '1's in the mask.
4166  EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
4167  SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
4168  if (MaskIntVT.getSizeInBits() < 32) {
4169  MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
4170  MaskIntVT = MVT::i32;
4171  }
4172 
4173  // Count '1's with POPCNT.
4174  Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
4175  Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
4176  // Scale is an element size in bytes.
4177  SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
4178  AddrVT);
4179  Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
4180  } else
4181  Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);
4182 
4183  return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
4184 }
4185 
4187  SDValue Idx,
4188  EVT VecVT,
4189  const SDLoc &dl) {
4190  if (isa<ConstantSDNode>(Idx))
4191  return Idx;
4192 
4193  EVT IdxVT = Idx.getValueType();
4194  unsigned NElts = VecVT.getVectorNumElements();
4195  if (isPowerOf2_32(NElts)) {
4197  Log2_32(NElts));
4198  return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
4199  DAG.getConstant(Imm, dl, IdxVT));
4200  }
4201 
4202  return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
4203  DAG.getConstant(NElts - 1, dl, IdxVT));
4204 }
4205 
4207  SDValue VecPtr, EVT VecVT,
4208  SDValue Index) const {
4209  SDLoc dl(Index);
4210  // Make sure the index type is big enough to compute in.
4211  Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
4212 
4213  EVT EltVT = VecVT.getVectorElementType();
4214 
4215  // Calculate the element offset and add it to the pointer.
4216  unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
4217  assert(EltSize * 8 == EltVT.getSizeInBits() &&
4218  "Converting bits to bytes lost precision");
4219 
4220  Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl);
4221 
4222  EVT IdxVT = Index.getValueType();
4223 
4224  Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
4225  DAG.getConstant(EltSize, dl, IdxVT));
4226  return DAG.getNode(ISD::ADD, dl, IdxVT, VecPtr, Index);
4227 }
4228 
4229 //===----------------------------------------------------------------------===//
4230 // Implementation of Emulated TLS Model
4231 //===----------------------------------------------------------------------===//
4232 
4234  SelectionDAG &DAG) const {
4235  // Access to address of TLS varialbe xyz is lowered to a function call:
4236  // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
4237  EVT PtrVT = getPointerTy(DAG.getDataLayout());
4238  PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
4239  SDLoc dl(GA);
4240 
4241  ArgListTy Args;
4242  ArgListEntry Entry;
4243  std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
4244  Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
4245  StringRef EmuTlsVarName(NameString);
4246  GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
4247  assert(EmuTlsVar && "Cannot find EmuTlsVar ");
4248  Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
4249  Entry.Ty = VoidPtrType;
4250  Args.push_back(Entry);
4251 
4252  SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
4253 
4255  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
4256  CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
4257  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
4258 
4259  // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
4260  // At last for X86 targets, maybe good for other targets too?
4262  MFI.setAdjustsStack(true); // Is this only for X86 target?
4263  MFI.setHasCalls(true);
4264 
4265  assert((GA->getOffset() == 0) &&
4266  "Emulated TLS must have zero offset in GlobalAddressSDNode");
4267  return CallResult.first;
4268 }
4269 
4271  SelectionDAG &DAG) const {
4272  assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
4273  if (!isCtlzFast())
4274  return SDValue();
4275  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
4276  SDLoc dl(Op);
4277  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
4278  if (C->isNullValue() && CC == ISD::SETEQ) {
4279  EVT VT = Op.getOperand(0).getValueType();
4280  SDValue Zext = Op.getOperand(0);
4281  if (VT.bitsLT(MVT::i32)) {
4282  VT = MVT::i32;
4283  Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
4284  }
4285  unsigned Log2b = Log2_32(VT.getSizeInBits());
4286  SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
4287  SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
4288  DAG.getConstant(Log2b, dl, MVT::i32));
4289  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
4290  }
4291  }
4292  return SDValue();
4293 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:539
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
static MVT getIntegerVT(unsigned BitWidth)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:544
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:829
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isInteger() const
Return true if this is an integer or a vector integer type.
void setSignBit()
Set the sign bit to 1.
Definition: APInt.h:1397
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
bool isUndef() const
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
const GlobalValue * getGlobal() const
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1547
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:555
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1402
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:951
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:321
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:115
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
unsigned getParamAlignment(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Definition: CallSite.h:406
Various leaf nodes.
Definition: ISDOpcodes.h:60
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
virtual MVT::SimpleValueType getCmpLibcallReturnType() const
Return the ValueType for comparison libcalls.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:63
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:262
iterator begin() const
Definition: ArrayRef.h:137
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:137
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:223
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
unsigned s
shift amount
Definition: APInt.h:1941
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:138
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:864
const SDValue & getValue() const
unsigned Reg
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:641
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
int MatchingInput
MatchingInput - If this is not -1, this is an output constraint where an input constraint is required...
Definition: InlineAsm.h:131
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
The two locations do not alias at all.
Definition: AliasAnalysis.h:85
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1313
static ISD::NodeType getExtendForContent(BooleanContent Content)
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:388
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
unsigned getAlignment() const
bool isBitwiseNot(SDValue V)
Returns true if V is a bitwise not operation.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:817
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1374
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:360
unsigned const TargetRegisterInfo * TRI
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, ImmutableCallSite CS) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
F(f)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL) const
Soften the operands of a comparison.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is &#39;desirable&#39; to us...
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
Value * CallOperandVal
If this is the result output operand or a clobber, this is null, otherwise it is the incoming operand...
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal...
Definition: ISDOpcodes.h:957
bool isConstTrueVal(const SDNode *N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:329
CallLoweringInfo & setDiscardResult(bool Value=true)
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Magic data for optimising unsigned division by a constant.
Definition: APInt.h:1945
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:405
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:128
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:528
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:292
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1493
SDValue getExternalSymbol(const char *Sym, EVT VT)
bool hasMatchingInput() const
hasMatchingInput - Return true if this is an output constraint that has a matching input constraint...
Definition: InlineAsm.h:135
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
ConstraintCodeVector Codes
Code - The constraint code, either the register name (in braces) or the constraint letter/number...
Definition: InlineAsm.h:149
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:130
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.h:1616
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
void setNoSignedWrap(bool b)
ms magic() const
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
Definition: APInt.cpp:1151
std::vector< std::string > ConstraintCodeVector
Definition: InlineAsm.h:97
bool hasOneUse() const
Return true if there is exactly one use of this node.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
Definition: APInt.cpp:514
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset...
void setBit(unsigned BitPosition)
Set a given bit to 1.
Definition: APInt.h:1387
static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &dl)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:986
Shift and rotation operations.
Definition: ISDOpcodes.h:380
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
Class to represent struct types.
Definition: DerivedTypes.h:201
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef...
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:191
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:36
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDValue Op1, APInt d, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDNode *> &Created)
Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the con...
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:197
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
iterator_range< regclass_iterator > regclasses() const
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:457
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:398
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
Definition: APInt.cpp:509
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:971
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
zlib-gnu style compression
bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &Demanded, TargetLoweringOpt &TLO) const
Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
unsigned getLiveInPhysReg(unsigned VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it&#39;s free to truncate a value of type FromTy to type ToTy.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
unsigned getSizeInBits() const
Context object for machine code objects.
Definition: MCContext.h:63
void emitError(unsigned LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
ValTy * getCalledValue() const
Return the pointer to function that is being called.
Definition: CallSite.h:100
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1517
int64_t getSExtValue() const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:947
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1559
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
static SDNodeIterator begin(const SDNode *N)
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:399
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1446
This contains information for each constraint that we are lowering.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:403
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:133
bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool Signed) const
Return if N is a True value when extended to VT.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Return true if the call or the callee has the given attribute.
Definition: CallSite.h:377
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
EK_BlockAddress - Each entry is a plain address of block, e.g.
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:221
CallLoweringInfo & setZExtResult(bool Value=true)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:865
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:210
virtual bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:911
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
op_iterator op_begin() const
Definition: