LLVM  9.0.0svn
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "llvm/ADT/BitVector.h"
15 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/DerivedTypes.h"
26 #include "llvm/IR/GlobalVariable.h"
27 #include "llvm/IR/LLVMContext.h"
28 #include "llvm/MC/MCAsmInfo.h"
29 #include "llvm/MC/MCExpr.h"
31 #include "llvm/Support/KnownBits.h"
35 #include <cctype>
36 using namespace llvm;
37 
38 /// NOTE: The TargetMachine owns TLOF.
40  : TargetLoweringBase(tm) {}
41 
42 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43  return nullptr;
44 }
45 
48 }
49 
50 /// Check whether a given call node is in tail position within its function. If
51 /// so, it sets Chain to the input chain of the tail call.
53  SDValue &Chain) const {
54  const Function &F = DAG.getMachineFunction().getFunction();
55 
56  // Conservatively require the attributes of the call to match those of
57  // the return. Ignore NoAlias and NonNull because they don't affect the
58  // call sequence.
59  AttributeList CallerAttrs = F.getAttributes();
60  if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
61  .removeAttribute(Attribute::NoAlias)
62  .removeAttribute(Attribute::NonNull)
63  .hasAttributes())
64  return false;
65 
66  // It's not safe to eliminate the sign / zero extension of the return value.
67  if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
68  CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
69  return false;
70 
71  // Check if the only use is a function return node.
72  return isUsedByReturnOnly(Node, Chain);
73 }
74 
76  const uint32_t *CallerPreservedMask,
77  const SmallVectorImpl<CCValAssign> &ArgLocs,
78  const SmallVectorImpl<SDValue> &OutVals) const {
79  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
80  const CCValAssign &ArgLoc = ArgLocs[I];
81  if (!ArgLoc.isRegLoc())
82  continue;
83  unsigned Reg = ArgLoc.getLocReg();
84  // Only look at callee saved registers.
85  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
86  continue;
87  // Check that we pass the value used for the caller.
88  // (We look for a CopyFromReg reading a virtual register that is used
89  // for the function live-in value of register Reg)
90  SDValue Value = OutVals[I];
91  if (Value->getOpcode() != ISD::CopyFromReg)
92  return false;
93  unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
94  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
95  return false;
96  }
97  return true;
98 }
99 
100 /// Set CallLoweringInfo attribute flags based on a call instruction
101 /// and called function attributes.
103  unsigned ArgIdx) {
104  IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
105  IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
106  IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
107  IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
108  IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
109  IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
110  IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
111  IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
112  IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
113  IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
114  Alignment = Call->getParamAlignment(ArgIdx);
115 }
116 
117 /// Generate a libcall taking the given operands as arguments and returning a
118 /// result of type RetVT.
119 std::pair<SDValue, SDValue>
121  ArrayRef<SDValue> Ops, bool isSigned,
122  const SDLoc &dl, bool doesNotReturn,
123  bool isReturnValueUsed,
124  bool isPostTypeLegalization) const {
126  Args.reserve(Ops.size());
127 
129  for (SDValue Op : Ops) {
130  Entry.Node = Op;
131  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
132  Entry.IsSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
133  Entry.IsZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
134  Args.push_back(Entry);
135  }
136 
137  if (LC == RTLIB::UNKNOWN_LIBCALL)
138  report_fatal_error("Unsupported library call operation!");
140  getPointerTy(DAG.getDataLayout()));
141 
142  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
144  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, isSigned);
145  CLI.setDebugLoc(dl)
146  .setChain(DAG.getEntryNode())
147  .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
148  .setNoReturn(doesNotReturn)
149  .setDiscardResult(!isReturnValueUsed)
150  .setIsPostTypeLegalization(isPostTypeLegalization)
151  .setSExtResult(signExtend)
152  .setZExtResult(!signExtend);
153  return LowerCallTo(CLI);
154 }
155 
156 /// Soften the operands of a comparison. This code is shared among BR_CC,
157 /// SELECT_CC, and SETCC handlers.
159  SDValue &NewLHS, SDValue &NewRHS,
160  ISD::CondCode &CCCode,
161  const SDLoc &dl) const {
162  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
163  && "Unsupported setcc type!");
164 
165  // Expand into one or more soft-fp libcall(s).
166  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
167  bool ShouldInvertCC = false;
168  switch (CCCode) {
169  case ISD::SETEQ:
170  case ISD::SETOEQ:
171  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
172  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
173  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
174  break;
175  case ISD::SETNE:
176  case ISD::SETUNE:
177  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
178  (VT == MVT::f64) ? RTLIB::UNE_F64 :
179  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
180  break;
181  case ISD::SETGE:
182  case ISD::SETOGE:
183  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
184  (VT == MVT::f64) ? RTLIB::OGE_F64 :
185  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
186  break;
187  case ISD::SETLT:
188  case ISD::SETOLT:
189  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
190  (VT == MVT::f64) ? RTLIB::OLT_F64 :
191  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
192  break;
193  case ISD::SETLE:
194  case ISD::SETOLE:
195  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
196  (VT == MVT::f64) ? RTLIB::OLE_F64 :
197  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
198  break;
199  case ISD::SETGT:
200  case ISD::SETOGT:
201  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
202  (VT == MVT::f64) ? RTLIB::OGT_F64 :
203  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
204  break;
205  case ISD::SETUO:
206  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
207  (VT == MVT::f64) ? RTLIB::UO_F64 :
208  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
209  break;
210  case ISD::SETO:
211  LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
212  (VT == MVT::f64) ? RTLIB::O_F64 :
213  (VT == MVT::f128) ? RTLIB::O_F128 : RTLIB::O_PPCF128;
214  break;
215  case ISD::SETONE:
216  // SETONE = SETOLT | SETOGT
217  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
218  (VT == MVT::f64) ? RTLIB::OLT_F64 :
219  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
220  LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
221  (VT == MVT::f64) ? RTLIB::OGT_F64 :
222  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
223  break;
224  case ISD::SETUEQ:
225  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
226  (VT == MVT::f64) ? RTLIB::UO_F64 :
227  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
228  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
229  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
230  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
231  break;
232  default:
233  // Invert CC for unordered comparisons
234  ShouldInvertCC = true;
235  switch (CCCode) {
236  case ISD::SETULT:
237  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
238  (VT == MVT::f64) ? RTLIB::OGE_F64 :
239  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
240  break;
241  case ISD::SETULE:
242  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
243  (VT == MVT::f64) ? RTLIB::OGT_F64 :
244  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
245  break;
246  case ISD::SETUGT:
247  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
248  (VT == MVT::f64) ? RTLIB::OLE_F64 :
249  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
250  break;
251  case ISD::SETUGE:
252  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
253  (VT == MVT::f64) ? RTLIB::OLT_F64 :
254  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
255  break;
256  default: llvm_unreachable("Do not know how to soften this setcc!");
257  }
258  }
259 
260  // Use the target specific return value for comparions lib calls.
261  EVT RetVT = getCmpLibcallReturnType();
262  SDValue Ops[2] = {NewLHS, NewRHS};
263  NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, false /*sign irrelevant*/,
264  dl).first;
265  NewRHS = DAG.getConstant(0, dl, RetVT);
266 
267  CCCode = getCmpLibcallCC(LC1);
268  if (ShouldInvertCC)
269  CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);
270 
271  if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
272  SDValue Tmp = DAG.getNode(
273  ISD::SETCC, dl,
274  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
275  NewLHS, NewRHS, DAG.getCondCode(CCCode));
276  NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, false/*sign irrelevant*/,
277  dl).first;
278  NewLHS = DAG.getNode(
279  ISD::SETCC, dl,
280  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
281  NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
282  NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
283  NewRHS = SDValue();
284  }
285 }
286 
287 /// Return the entry encoding for a jump table in the current function. The
288 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
290  // In non-pic modes, just use the address of a block.
291  if (!isPositionIndependent())
293 
294  // In PIC mode, if the target supports a GPRel32 directive, use it.
295  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
297 
298  // Otherwise, use a label difference.
300 }
301 
303  SelectionDAG &DAG) const {
304  // If our PIC model is GP relative, use the global offset table as the base.
305  unsigned JTEncoding = getJumpTableEncoding();
306 
307  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
310 
311  return Table;
312 }
313 
314 /// This returns the relocation base for the given PIC jumptable, the same as
315 /// getPICJumpTableRelocBase, but as an MCExpr.
316 const MCExpr *
318  unsigned JTI,MCContext &Ctx) const{
319  // The normal PIC reloc base is the label at the start of the jump table.
320  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
321 }
322 
323 bool
325  const TargetMachine &TM = getTargetMachine();
326  const GlobalValue *GV = GA->getGlobal();
327 
328  // If the address is not even local to this DSO we will have to load it from
329  // a got and then add the offset.
330  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
331  return false;
332 
333  // If the code is position independent we will have to add a base register.
334  if (isPositionIndependent())
335  return false;
336 
337  // Otherwise we can do it.
338  return true;
339 }
340 
341 //===----------------------------------------------------------------------===//
342 // Optimization Methods
343 //===----------------------------------------------------------------------===//
344 
345 /// If the specified instruction has a constant integer operand and there are
346 /// bits set in that constant that are not demanded, then clear those bits and
347 /// return true.
349  TargetLoweringOpt &TLO) const {
350  SelectionDAG &DAG = TLO.DAG;
351  SDLoc DL(Op);
352  unsigned Opcode = Op.getOpcode();
353 
354  // Do target-specific constant optimization.
355  if (targetShrinkDemandedConstant(Op, Demanded, TLO))
356  return TLO.New.getNode();
357 
358  // FIXME: ISD::SELECT, ISD::SELECT_CC
359  switch (Opcode) {
360  default:
361  break;
362  case ISD::XOR:
363  case ISD::AND:
364  case ISD::OR: {
365  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
366  if (!Op1C)
367  return false;
368 
369  // If this is a 'not' op, don't touch it because that's a canonical form.
370  const APInt &C = Op1C->getAPIntValue();
371  if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
372  return false;
373 
374  if (!C.isSubsetOf(Demanded)) {
375  EVT VT = Op.getValueType();
376  SDValue NewC = DAG.getConstant(Demanded & C, DL, VT);
377  SDValue NewOp = DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
378  return TLO.CombineTo(Op, NewOp);
379  }
380 
381  break;
382  }
383  }
384 
385  return false;
386 }
387 
388 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
389 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
390 /// generalized for targets with other types of implicit widening casts.
392  const APInt &Demanded,
393  TargetLoweringOpt &TLO) const {
394  assert(Op.getNumOperands() == 2 &&
395  "ShrinkDemandedOp only supports binary operators!");
396  assert(Op.getNode()->getNumValues() == 1 &&
397  "ShrinkDemandedOp only supports nodes with one result!");
398 
399  SelectionDAG &DAG = TLO.DAG;
400  SDLoc dl(Op);
401 
402  // Early return, as this function cannot handle vector types.
403  if (Op.getValueType().isVector())
404  return false;
405 
406  // Don't do this if the node has another user, which may require the
407  // full value.
408  if (!Op.getNode()->hasOneUse())
409  return false;
410 
411  // Search for the smallest integer type with free casts to and from
412  // Op's type. For expedience, just check power-of-2 integer types.
413  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
414  unsigned DemandedSize = Demanded.getActiveBits();
415  unsigned SmallVTBits = DemandedSize;
416  if (!isPowerOf2_32(SmallVTBits))
417  SmallVTBits = NextPowerOf2(SmallVTBits);
418  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
419  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
420  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
421  TLI.isZExtFree(SmallVT, Op.getValueType())) {
422  // We found a type with free casts.
423  SDValue X = DAG.getNode(
424  Op.getOpcode(), dl, SmallVT,
425  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
426  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
427  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
428  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
429  return TLO.CombineTo(Op, Z);
430  }
431  }
432  return false;
433 }
434 
436  DAGCombinerInfo &DCI) const {
437  SelectionDAG &DAG = DCI.DAG;
438  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
439  !DCI.isBeforeLegalizeOps());
440  KnownBits Known;
441 
442  bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
443  if (Simplified) {
444  DCI.AddToWorklist(Op.getNode());
445  DCI.CommitTargetLoweringOpt(TLO);
446  }
447  return Simplified;
448 }
449 
451  KnownBits &Known,
452  TargetLoweringOpt &TLO,
453  unsigned Depth,
454  bool AssumeSingleUse) const {
455  EVT VT = Op.getValueType();
456  APInt DemandedElts = VT.isVector()
458  : APInt(1, 1);
459  return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
460  AssumeSingleUse);
461 }
462 
463 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
464 /// result of Op are ever used downstream. If we can use this information to
465 /// simplify Op, create a new simplified DAG node and return true, returning the
466 /// original and new nodes in Old and New. Otherwise, analyze the expression and
467 /// return a mask of Known bits for the expression (used to simplify the
468 /// caller). The Known bits may only be accurate for those bits in the
469 /// OriginalDemandedBits and OriginalDemandedElts.
471  SDValue Op, const APInt &OriginalDemandedBits,
472  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
473  unsigned Depth, bool AssumeSingleUse) const {
474  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
475  assert(Op.getScalarValueSizeInBits() == BitWidth &&
476  "Mask size mismatches value type size!");
477 
478  unsigned NumElts = OriginalDemandedElts.getBitWidth();
479  assert((!Op.getValueType().isVector() ||
480  NumElts == Op.getValueType().getVectorNumElements()) &&
481  "Unexpected vector size");
482 
483  APInt DemandedBits = OriginalDemandedBits;
484  APInt DemandedElts = OriginalDemandedElts;
485  SDLoc dl(Op);
486  auto &DL = TLO.DAG.getDataLayout();
487 
488  // Don't know anything.
489  Known = KnownBits(BitWidth);
490 
491  if (Op.getOpcode() == ISD::Constant) {
492  // We know all of the bits for a constant!
493  Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
494  Known.Zero = ~Known.One;
495  return false;
496  }
497 
498  // Other users may use these bits.
499  EVT VT = Op.getValueType();
500  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
501  if (Depth != 0) {
502  // If not at the root, Just compute the Known bits to
503  // simplify things downstream.
504  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
505  return false;
506  }
507  // If this is the root being simplified, allow it to have multiple uses,
508  // just set the DemandedBits/Elts to all bits.
509  DemandedBits = APInt::getAllOnesValue(BitWidth);
510  DemandedElts = APInt::getAllOnesValue(NumElts);
511  } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
512  // Not demanding any bits/elts from Op.
513  if (!Op.isUndef())
514  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
515  return false;
516  } else if (Depth == 6) { // Limit search depth.
517  return false;
518  }
519 
520  KnownBits Known2, KnownOut;
521  switch (Op.getOpcode()) {
522  case ISD::SCALAR_TO_VECTOR: {
523  if (!DemandedElts[0])
524  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
525 
526  KnownBits SrcKnown;
527  SDValue Src = Op.getOperand(0);
528  unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
529  APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
530  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
531  return true;
532  Known = SrcKnown.zextOrTrunc(BitWidth, false);
533  break;
534  }
535  case ISD::BUILD_VECTOR:
536  // Collect the known bits that are shared by every constant vector element.
537  Known.Zero.setAllBits(); Known.One.setAllBits();
538  for (SDValue SrcOp : Op->ops()) {
539  if (!isa<ConstantSDNode>(SrcOp)) {
540  // We can only handle all constant values - bail out with no known bits.
541  Known = KnownBits(BitWidth);
542  return false;
543  }
544  Known2.One = cast<ConstantSDNode>(SrcOp)->getAPIntValue();
545  Known2.Zero = ~Known2.One;
546 
547  // BUILD_VECTOR can implicitly truncate sources, we must handle this.
548  if (Known2.One.getBitWidth() != BitWidth) {
549  assert(Known2.getBitWidth() > BitWidth &&
550  "Expected BUILD_VECTOR implicit truncation");
551  Known2 = Known2.trunc(BitWidth);
552  }
553 
554  // Known bits are the values that are shared by every element.
555  // TODO: support per-element known bits.
556  Known.One &= Known2.One;
557  Known.Zero &= Known2.Zero;
558  }
559  return false; // Don't fall through, will infinitely loop.
560  case ISD::INSERT_VECTOR_ELT: {
561  SDValue Vec = Op.getOperand(0);
562  SDValue Scl = Op.getOperand(1);
563  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
564  EVT VecVT = Vec.getValueType();
565 
566  // If index isn't constant, assume we need all vector elements AND the
567  // inserted element.
568  APInt DemandedVecElts(OriginalDemandedElts);
569  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
570  unsigned Idx = CIdx->getZExtValue();
571  DemandedVecElts.clearBit(Idx);
572 
573  // Inserted element is not required.
574  if (!OriginalDemandedElts[Idx])
575  return TLO.CombineTo(Op, Vec);
576  }
577 
578  KnownBits KnownScl;
579  unsigned NumSclBits = Scl.getScalarValueSizeInBits();
580  APInt DemandedSclBits = OriginalDemandedBits.zextOrTrunc(NumSclBits);
581  if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
582  return true;
583 
584  Known = KnownScl.zextOrTrunc(BitWidth, false);
585 
586  KnownBits KnownVec;
587  if (SimplifyDemandedBits(Vec, OriginalDemandedBits, DemandedVecElts,
588  KnownVec, TLO, Depth + 1))
589  return true;
590 
591  if (!!DemandedVecElts) {
592  Known.One &= KnownVec.One;
593  Known.Zero &= KnownVec.Zero;
594  }
595 
596  return false;
597  }
598  case ISD::INSERT_SUBVECTOR: {
599  SDValue Base = Op.getOperand(0);
600  SDValue Sub = Op.getOperand(1);
601  EVT SubVT = Sub.getValueType();
602  unsigned NumSubElts = SubVT.getVectorNumElements();
603 
604  // If index isn't constant, assume we need the original demanded base
605  // elements and ALL the inserted subvector elements.
606  APInt BaseElts = DemandedElts;
607  APInt SubElts = APInt::getAllOnesValue(NumSubElts);
608  if (isa<ConstantSDNode>(Op.getOperand(2))) {
609  const APInt &Idx = Op.getConstantOperandAPInt(2);
610  if (Idx.ule(NumElts - NumSubElts)) {
611  unsigned SubIdx = Idx.getZExtValue();
612  SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
613  BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
614  }
615  }
616 
617  KnownBits KnownSub, KnownBase;
618  if (SimplifyDemandedBits(Sub, DemandedBits, SubElts, KnownSub, TLO,
619  Depth + 1))
620  return true;
621  if (SimplifyDemandedBits(Base, DemandedBits, BaseElts, KnownBase, TLO,
622  Depth + 1))
623  return true;
624 
625  Known.Zero.setAllBits();
626  Known.One.setAllBits();
627  if (!!SubElts) {
628  Known.One &= KnownSub.One;
629  Known.Zero &= KnownSub.Zero;
630  }
631  if (!!BaseElts) {
632  Known.One &= KnownBase.One;
633  Known.Zero &= KnownBase.Zero;
634  }
635  break;
636  }
637  case ISD::CONCAT_VECTORS: {
638  Known.Zero.setAllBits();
639  Known.One.setAllBits();
640  EVT SubVT = Op.getOperand(0).getValueType();
641  unsigned NumSubVecs = Op.getNumOperands();
642  unsigned NumSubElts = SubVT.getVectorNumElements();
643  for (unsigned i = 0; i != NumSubVecs; ++i) {
644  APInt DemandedSubElts =
645  DemandedElts.extractBits(NumSubElts, i * NumSubElts);
646  if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
647  Known2, TLO, Depth + 1))
648  return true;
649  // Known bits are shared by every demanded subvector element.
650  if (!!DemandedSubElts) {
651  Known.One &= Known2.One;
652  Known.Zero &= Known2.Zero;
653  }
654  }
655  break;
656  }
657  case ISD::VECTOR_SHUFFLE: {
658  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
659 
660  // Collect demanded elements from shuffle operands..
661  APInt DemandedLHS(NumElts, 0);
662  APInt DemandedRHS(NumElts, 0);
663  for (unsigned i = 0; i != NumElts; ++i) {
664  if (!DemandedElts[i])
665  continue;
666  int M = ShuffleMask[i];
667  if (M < 0) {
668  // For UNDEF elements, we don't know anything about the common state of
669  // the shuffle result.
670  DemandedLHS.clearAllBits();
671  DemandedRHS.clearAllBits();
672  break;
673  }
674  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
675  if (M < (int)NumElts)
676  DemandedLHS.setBit(M);
677  else
678  DemandedRHS.setBit(M - NumElts);
679  }
680 
681  if (!!DemandedLHS || !!DemandedRHS) {
682  Known.Zero.setAllBits();
683  Known.One.setAllBits();
684  if (!!DemandedLHS) {
685  if (SimplifyDemandedBits(Op.getOperand(0), DemandedBits, DemandedLHS,
686  Known2, TLO, Depth + 1))
687  return true;
688  Known.One &= Known2.One;
689  Known.Zero &= Known2.Zero;
690  }
691  if (!!DemandedRHS) {
692  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedRHS,
693  Known2, TLO, Depth + 1))
694  return true;
695  Known.One &= Known2.One;
696  Known.Zero &= Known2.Zero;
697  }
698  }
699  break;
700  }
701  case ISD::AND: {
702  SDValue Op0 = Op.getOperand(0);
703  SDValue Op1 = Op.getOperand(1);
704 
705  // If the RHS is a constant, check to see if the LHS would be zero without
706  // using the bits from the RHS. Below, we use knowledge about the RHS to
707  // simplify the LHS, here we're using information from the LHS to simplify
708  // the RHS.
709  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
710  // Do not increment Depth here; that can cause an infinite loop.
711  KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
712  // If the LHS already has zeros where RHSC does, this 'and' is dead.
713  if ((LHSKnown.Zero & DemandedBits) ==
714  (~RHSC->getAPIntValue() & DemandedBits))
715  return TLO.CombineTo(Op, Op0);
716 
717  // If any of the set bits in the RHS are known zero on the LHS, shrink
718  // the constant.
719  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits, TLO))
720  return true;
721 
722  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
723  // constant, but if this 'and' is only clearing bits that were just set by
724  // the xor, then this 'and' can be eliminated by shrinking the mask of
725  // the xor. For example, for a 32-bit X:
726  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
727  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
728  LHSKnown.One == ~RHSC->getAPIntValue()) {
729  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
730  return TLO.CombineTo(Op, Xor);
731  }
732  }
733 
734  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
735  Depth + 1))
736  return true;
737  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
738  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
739  Known2, TLO, Depth + 1))
740  return true;
741  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
742 
743  // If all of the demanded bits are known one on one side, return the other.
744  // These bits cannot contribute to the result of the 'and'.
745  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
746  return TLO.CombineTo(Op, Op0);
747  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
748  return TLO.CombineTo(Op, Op1);
749  // If all of the demanded bits in the inputs are known zeros, return zero.
750  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
751  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
752  // If the RHS is a constant, see if we can simplify it.
753  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, TLO))
754  return true;
755  // If the operation can be done in a smaller type, do so.
756  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
757  return true;
758 
759  // Output known-1 bits are only known if set in both the LHS & RHS.
760  Known.One &= Known2.One;
761  // Output known-0 are known to be clear if zero in either the LHS | RHS.
762  Known.Zero |= Known2.Zero;
763  break;
764  }
765  case ISD::OR: {
766  SDValue Op0 = Op.getOperand(0);
767  SDValue Op1 = Op.getOperand(1);
768 
769  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
770  Depth + 1))
771  return true;
772  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
773  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
774  Known2, TLO, Depth + 1))
775  return true;
776  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
777 
778  // If all of the demanded bits are known zero on one side, return the other.
779  // These bits cannot contribute to the result of the 'or'.
780  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
781  return TLO.CombineTo(Op, Op0);
782  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
783  return TLO.CombineTo(Op, Op1);
784  // If the RHS is a constant, see if we can simplify it.
785  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
786  return true;
787  // If the operation can be done in a smaller type, do so.
788  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
789  return true;
790 
791  // Output known-0 bits are only known if clear in both the LHS & RHS.
792  Known.Zero &= Known2.Zero;
793  // Output known-1 are known to be set if set in either the LHS | RHS.
794  Known.One |= Known2.One;
795  break;
796  }
797  case ISD::XOR: {
798  SDValue Op0 = Op.getOperand(0);
799  SDValue Op1 = Op.getOperand(1);
800 
801  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
802  Depth + 1))
803  return true;
804  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
805  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
806  Depth + 1))
807  return true;
808  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
809 
810  // If all of the demanded bits are known zero on one side, return the other.
811  // These bits cannot contribute to the result of the 'xor'.
812  if (DemandedBits.isSubsetOf(Known.Zero))
813  return TLO.CombineTo(Op, Op0);
814  if (DemandedBits.isSubsetOf(Known2.Zero))
815  return TLO.CombineTo(Op, Op1);
816  // If the operation can be done in a smaller type, do so.
817  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
818  return true;
819 
820  // If all of the unknown bits are known to be zero on one side or the other
821  // (but not both) turn this into an *inclusive* or.
822  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
823  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
824  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
825 
826  // Output known-0 bits are known if clear or set in both the LHS & RHS.
827  KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
828  // Output known-1 are known to be set if set in only one of the LHS, RHS.
829  KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
830 
831  if (ConstantSDNode *C = isConstOrConstSplat(Op1)) {
832  // If one side is a constant, and all of the known set bits on the other
833  // side are also set in the constant, turn this into an AND, as we know
834  // the bits will be cleared.
835  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
836  // NB: it is okay if more bits are known than are requested
837  if (C->getAPIntValue() == Known2.One) {
838  SDValue ANDC =
839  TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
840  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
841  }
842 
843  // If the RHS is a constant, see if we can change it. Don't alter a -1
844  // constant because that's a 'not' op, and that is better for combining
845  // and codegen.
846  if (!C->isAllOnesValue()) {
847  if (DemandedBits.isSubsetOf(C->getAPIntValue())) {
848  // We're flipping all demanded bits. Flip the undemanded bits too.
849  SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
850  return TLO.CombineTo(Op, New);
851  }
852  // If we can't turn this into a 'not', try to shrink the constant.
853  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
854  return true;
855  }
856  }
857 
858  Known = std::move(KnownOut);
859  break;
860  }
861  case ISD::SELECT:
862  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
863  Depth + 1))
864  return true;
865  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
866  Depth + 1))
867  return true;
868  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
869  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
870 
871  // If the operands are constants, see if we can simplify them.
872  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
873  return true;
874 
875  // Only known if known in both the LHS and RHS.
876  Known.One &= Known2.One;
877  Known.Zero &= Known2.Zero;
878  break;
879  case ISD::SELECT_CC:
880  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
881  Depth + 1))
882  return true;
883  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
884  Depth + 1))
885  return true;
886  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
887  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
888 
889  // If the operands are constants, see if we can simplify them.
890  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
891  return true;
892 
893  // Only known if known in both the LHS and RHS.
894  Known.One &= Known2.One;
895  Known.Zero &= Known2.Zero;
896  break;
897  case ISD::SETCC: {
898  SDValue Op0 = Op.getOperand(0);
899  SDValue Op1 = Op.getOperand(1);
900  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
901  // If (1) we only need the sign-bit, (2) the setcc operands are the same
902  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
903  // -1, we may be able to bypass the setcc.
904  if (DemandedBits.isSignMask() &&
905  Op0.getScalarValueSizeInBits() == BitWidth &&
906  getBooleanContents(VT) ==
907  BooleanContent::ZeroOrNegativeOneBooleanContent) {
908  // If we're testing X < 0, then this compare isn't needed - just use X!
909  // FIXME: We're limiting to integer types here, but this should also work
910  // if we don't care about FP signed-zero. The use of SETLT with FP means
911  // that we don't care about NaNs.
912  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
914  return TLO.CombineTo(Op, Op0);
915 
916  // TODO: Should we check for other forms of sign-bit comparisons?
917  // Examples: X <= -1, X >= 0
918  }
919  if (getBooleanContents(Op0.getValueType()) ==
921  BitWidth > 1)
922  Known.Zero.setBitsFrom(1);
923  break;
924  }
925  case ISD::SHL: {
926  SDValue Op0 = Op.getOperand(0);
927  SDValue Op1 = Op.getOperand(1);
928 
929  if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) {
930  // If the shift count is an invalid immediate, don't do anything.
931  if (SA->getAPIntValue().uge(BitWidth))
932  break;
933 
934  unsigned ShAmt = SA->getZExtValue();
935 
936  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
937  // single shift. We can do this if the bottom bits (which are shifted
938  // out) are never demanded.
939  if (Op0.getOpcode() == ISD::SRL) {
940  if (ShAmt &&
941  (DemandedBits & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) {
942  if (ConstantSDNode *SA2 = isConstOrConstSplat(Op0.getOperand(1))) {
943  if (SA2->getAPIntValue().ult(BitWidth)) {
944  unsigned C1 = SA2->getZExtValue();
945  unsigned Opc = ISD::SHL;
946  int Diff = ShAmt - C1;
947  if (Diff < 0) {
948  Diff = -Diff;
949  Opc = ISD::SRL;
950  }
951 
952  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
953  return TLO.CombineTo(
954  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
955  }
956  }
957  }
958  }
959 
960  if (SimplifyDemandedBits(Op0, DemandedBits.lshr(ShAmt), DemandedElts,
961  Known, TLO, Depth + 1))
962  return true;
963 
964  // Try shrinking the operation as long as the shift amount will still be
965  // in range.
966  if ((ShAmt < DemandedBits.getActiveBits()) &&
967  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
968  return true;
969 
970  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
971  // are not demanded. This will likely allow the anyext to be folded away.
972  if (Op0.getOpcode() == ISD::ANY_EXTEND) {
973  SDValue InnerOp = Op0.getOperand(0);
974  EVT InnerVT = InnerOp.getValueType();
975  unsigned InnerBits = InnerVT.getScalarSizeInBits();
976  if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
977  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
978  EVT ShTy = getShiftAmountTy(InnerVT, DL);
979  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
980  ShTy = InnerVT;
981  SDValue NarrowShl =
982  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
983  TLO.DAG.getConstant(ShAmt, dl, ShTy));
984  return TLO.CombineTo(
985  Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
986  }
987  // Repeat the SHL optimization above in cases where an extension
988  // intervenes: (shl (anyext (shr x, c1)), c2) to
989  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
990  // aren't demanded (as above) and that the shifted upper c1 bits of
991  // x aren't demanded.
992  if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
993  InnerOp.hasOneUse()) {
994  if (ConstantSDNode *SA2 =
995  isConstOrConstSplat(InnerOp.getOperand(1))) {
996  unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
997  if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
998  DemandedBits.getActiveBits() <=
999  (InnerBits - InnerShAmt + ShAmt) &&
1000  DemandedBits.countTrailingZeros() >= ShAmt) {
1001  SDValue NewSA = TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
1002  Op1.getValueType());
1003  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1004  InnerOp.getOperand(0));
1005  return TLO.CombineTo(
1006  Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1007  }
1008  }
1009  }
1010  }
1011 
1012  Known.Zero <<= ShAmt;
1013  Known.One <<= ShAmt;
1014  // low bits known zero.
1015  Known.Zero.setLowBits(ShAmt);
1016  }
1017  break;
1018  }
1019  case ISD::SRL: {
1020  SDValue Op0 = Op.getOperand(0);
1021  SDValue Op1 = Op.getOperand(1);
1022 
1023  if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) {
1024  // If the shift count is an invalid immediate, don't do anything.
1025  if (SA->getAPIntValue().uge(BitWidth))
1026  break;
1027 
1028  unsigned ShAmt = SA->getZExtValue();
1029  APInt InDemandedMask = (DemandedBits << ShAmt);
1030 
1031  // If the shift is exact, then it does demand the low bits (and knows that
1032  // they are zero).
1033  if (Op->getFlags().hasExact())
1034  InDemandedMask.setLowBits(ShAmt);
1035 
1036  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1037  // single shift. We can do this if the top bits (which are shifted out)
1038  // are never demanded.
1039  if (Op0.getOpcode() == ISD::SHL) {
1040  if (ConstantSDNode *SA2 = isConstOrConstSplat(Op0.getOperand(1))) {
1041  if (ShAmt &&
1042  (DemandedBits & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) {
1043  if (SA2->getAPIntValue().ult(BitWidth)) {
1044  unsigned C1 = SA2->getZExtValue();
1045  unsigned Opc = ISD::SRL;
1046  int Diff = ShAmt - C1;
1047  if (Diff < 0) {
1048  Diff = -Diff;
1049  Opc = ISD::SHL;
1050  }
1051 
1052  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
1053  return TLO.CombineTo(
1054  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1055  }
1056  }
1057  }
1058  }
1059 
1060  // Compute the new bits that are at the top now.
1061  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1062  Depth + 1))
1063  return true;
1064  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1065  Known.Zero.lshrInPlace(ShAmt);
1066  Known.One.lshrInPlace(ShAmt);
1067 
1068  Known.Zero.setHighBits(ShAmt); // High bits known zero.
1069  }
1070  break;
1071  }
1072  case ISD::SRA: {
1073  SDValue Op0 = Op.getOperand(0);
1074  SDValue Op1 = Op.getOperand(1);
1075 
1076  // If this is an arithmetic shift right and only the low-bit is set, we can
1077  // always convert this into a logical shr, even if the shift amount is
1078  // variable. The low bit of the shift cannot be an input sign bit unless
1079  // the shift amount is >= the size of the datatype, which is undefined.
1080  if (DemandedBits.isOneValue())
1081  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1082 
1083  if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) {
1084  // If the shift count is an invalid immediate, don't do anything.
1085  if (SA->getAPIntValue().uge(BitWidth))
1086  break;
1087 
1088  unsigned ShAmt = SA->getZExtValue();
1089  APInt InDemandedMask = (DemandedBits << ShAmt);
1090 
1091  // If the shift is exact, then it does demand the low bits (and knows that
1092  // they are zero).
1093  if (Op->getFlags().hasExact())
1094  InDemandedMask.setLowBits(ShAmt);
1095 
1096  // If any of the demanded bits are produced by the sign extension, we also
1097  // demand the input sign bit.
1098  if (DemandedBits.countLeadingZeros() < ShAmt)
1099  InDemandedMask.setSignBit();
1100 
1101  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1102  Depth + 1))
1103  return true;
1104  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1105  Known.Zero.lshrInPlace(ShAmt);
1106  Known.One.lshrInPlace(ShAmt);
1107 
1108  // If the input sign bit is known to be zero, or if none of the top bits
1109  // are demanded, turn this into an unsigned shift right.
1110  if (Known.Zero[BitWidth - ShAmt - 1] ||
1111  DemandedBits.countLeadingZeros() >= ShAmt) {
1112  SDNodeFlags Flags;
1113  Flags.setExact(Op->getFlags().hasExact());
1114  return TLO.CombineTo(
1115  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1116  }
1117 
1118  int Log2 = DemandedBits.exactLogBase2();
1119  if (Log2 >= 0) {
1120  // The bit must come from the sign.
1121  SDValue NewSA =
1122  TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, Op1.getValueType());
1123  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1124  }
1125 
1126  if (Known.One[BitWidth - ShAmt - 1])
1127  // New bits are known one.
1128  Known.One.setHighBits(ShAmt);
1129  }
1130  break;
1131  }
1132  case ISD::FSHL:
1133  case ISD::FSHR: {
1134  SDValue Op0 = Op.getOperand(0);
1135  SDValue Op1 = Op.getOperand(1);
1136  SDValue Op2 = Op.getOperand(2);
1137  bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1138 
1139  if (ConstantSDNode *SA = isConstOrConstSplat(Op2)) {
1140  unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1141 
1142  // For fshl, 0-shift returns the 1st arg.
1143  // For fshr, 0-shift returns the 2nd arg.
1144  if (Amt == 0) {
1145  if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1146  Known, TLO, Depth + 1))
1147  return true;
1148  break;
1149  }
1150 
1151  // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1152  // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1153  APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1154  APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1155  if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1156  Depth + 1))
1157  return true;
1158  if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1159  Depth + 1))
1160  return true;
1161 
1162  Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1163  Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1164  Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1165  Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1166  Known.One |= Known2.One;
1167  Known.Zero |= Known2.Zero;
1168  }
1169  break;
1170  }
1171  case ISD::SIGN_EXTEND_INREG: {
1172  SDValue Op0 = Op.getOperand(0);
1173  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1174  unsigned ExVTBits = ExVT.getScalarSizeInBits();
1175 
1176  // If we only care about the highest bit, don't bother shifting right.
1177  if (DemandedBits.isSignMask()) {
1178  unsigned NumSignBits = TLO.DAG.ComputeNumSignBits(Op0);
1179  bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1180  // However if the input is already sign extended we expect the sign
1181  // extension to be dropped altogether later and do not simplify.
1182  if (!AlreadySignExtended) {
1183  // Compute the correct shift amount type, which must be getShiftAmountTy
1184  // for scalar types after legalization.
1185  EVT ShiftAmtTy = VT;
1186  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1187  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1188 
1189  SDValue ShiftAmt =
1190  TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1191  return TLO.CombineTo(Op,
1192  TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1193  }
1194  }
1195 
1196  // If none of the extended bits are demanded, eliminate the sextinreg.
1197  if (DemandedBits.getActiveBits() <= ExVTBits)
1198  return TLO.CombineTo(Op, Op0);
1199 
1200  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1201 
1202  // Since the sign extended bits are demanded, we know that the sign
1203  // bit is demanded.
1204  InputDemandedBits.setBit(ExVTBits - 1);
1205 
1206  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1207  return true;
1208  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1209 
1210  // If the sign bit of the input is known set or clear, then we know the
1211  // top bits of the result.
1212 
1213  // If the input sign bit is known zero, convert this into a zero extension.
1214  if (Known.Zero[ExVTBits - 1])
1215  return TLO.CombineTo(
1216  Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT.getScalarType()));
1217 
1218  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1219  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1220  Known.One.setBitsFrom(ExVTBits);
1221  Known.Zero &= Mask;
1222  } else { // Input sign bit unknown
1223  Known.Zero &= Mask;
1224  Known.One &= Mask;
1225  }
1226  break;
1227  }
1228  case ISD::BUILD_PAIR: {
1229  EVT HalfVT = Op.getOperand(0).getValueType();
1230  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1231 
1232  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1233  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1234 
1235  KnownBits KnownLo, KnownHi;
1236 
1237  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1238  return true;
1239 
1240  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1241  return true;
1242 
1243  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1244  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1245 
1246  Known.One = KnownLo.One.zext(BitWidth) |
1247  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1248  break;
1249  }
1250  case ISD::ZERO_EXTEND: {
1251  SDValue Src = Op.getOperand(0);
1252  unsigned InBits = Src.getScalarValueSizeInBits();
1253 
1254  // If none of the top bits are demanded, convert this into an any_extend.
1255  if (DemandedBits.getActiveBits() <= InBits)
1256  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, Src));
1257 
1258  APInt InDemandedBits = DemandedBits.trunc(InBits);
1259  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1))
1260  return true;
1261  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1262  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1263  Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
1264  break;
1265  }
1266  case ISD::SIGN_EXTEND: {
1267  SDValue Src = Op.getOperand(0);
1268  unsigned InBits = Src.getScalarValueSizeInBits();
1269 
1270  // If none of the top bits are demanded, convert this into an any_extend.
1271  if (DemandedBits.getActiveBits() <= InBits)
1272  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, Src));
1273 
1274  // Since some of the sign extended bits are demanded, we know that the sign
1275  // bit is demanded.
1276  APInt InDemandedBits = DemandedBits.trunc(InBits);
1277  InDemandedBits.setBit(InBits - 1);
1278 
1279  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1))
1280  return true;
1281  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1282  // If the sign bit is known one, the top bits match.
1283  Known = Known.sext(BitWidth);
1284 
1285  // If the sign bit is known zero, convert this to a zero extend.
1286  if (Known.isNonNegative())
1287  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Src));
1288  break;
1289  }
1291  // TODO - merge this with SIGN_EXTEND above?
1292  SDValue Src = Op.getOperand(0);
1293  unsigned InBits = Src.getScalarValueSizeInBits();
1294 
1295  APInt InDemandedBits = DemandedBits.trunc(InBits);
1296 
1297  // If some of the sign extended bits are demanded, we know that the sign
1298  // bit is demanded.
1299  if (InBits < DemandedBits.getActiveBits())
1300  InDemandedBits.setBit(InBits - 1);
1301 
1302  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1))
1303  return true;
1304  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1305  // If the sign bit is known one, the top bits match.
1306  Known = Known.sext(BitWidth);
1307  break;
1308  }
1309  case ISD::ANY_EXTEND: {
1310  SDValue Src = Op.getOperand(0);
1311  unsigned InBits = Src.getScalarValueSizeInBits();
1312  APInt InDemandedBits = DemandedBits.trunc(InBits);
1313  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1))
1314  return true;
1315  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1316  Known = Known.zext(BitWidth, false /* => any extend */);
1317  break;
1318  }
1319  case ISD::TRUNCATE: {
1320  SDValue Src = Op.getOperand(0);
1321 
1322  // Simplify the input, using demanded bit information, and compute the known
1323  // zero/one bits live out.
1324  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1325  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1326  if (SimplifyDemandedBits(Src, TruncMask, Known, TLO, Depth + 1))
1327  return true;
1328  Known = Known.trunc(BitWidth);
1329 
1330  // If the input is only used by this truncate, see if we can shrink it based
1331  // on the known demanded bits.
1332  if (Src.getNode()->hasOneUse()) {
1333  switch (Src.getOpcode()) {
1334  default:
1335  break;
1336  case ISD::SRL:
1337  // Shrink SRL by a constant if none of the high bits shifted in are
1338  // demanded.
1339  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1340  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1341  // undesirable.
1342  break;
1343 
1344  auto *ShAmt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
1345  if (!ShAmt || ShAmt->getAPIntValue().uge(BitWidth))
1346  break;
1347 
1348  SDValue Shift = Src.getOperand(1);
1349  uint64_t ShVal = ShAmt->getZExtValue();
1350 
1351  if (TLO.LegalTypes())
1352  Shift = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));
1353 
1354  APInt HighBits =
1355  APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
1356  HighBits.lshrInPlace(ShVal);
1357  HighBits = HighBits.trunc(BitWidth);
1358 
1359  if (!(HighBits & DemandedBits)) {
1360  // None of the shifted in bits are needed. Add a truncate of the
1361  // shift input, then shift it.
1362  SDValue NewTrunc =
1363  TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
1364  return TLO.CombineTo(
1365  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, Shift));
1366  }
1367  break;
1368  }
1369  }
1370 
1371  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1372  break;
1373  }
1374  case ISD::AssertZext: {
1375  // AssertZext demands all of the high bits, plus any of the low bits
1376  // demanded by its users.
1377  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1378  APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
1379  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
1380  TLO, Depth + 1))
1381  return true;
1382  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1383 
1384  Known.Zero |= ~InMask;
1385  break;
1386  }
1387  case ISD::EXTRACT_VECTOR_ELT: {
1388  SDValue Src = Op.getOperand(0);
1389  SDValue Idx = Op.getOperand(1);
1390  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1391  unsigned EltBitWidth = Src.getScalarValueSizeInBits();
1392 
1393  // Demand the bits from every vector element without a constant index.
1394  APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
1395  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
1396  if (CIdx->getAPIntValue().ult(NumSrcElts))
1397  DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
1398 
1399  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
1400  // anything about the extended bits.
1401  APInt DemandedSrcBits = DemandedBits;
1402  if (BitWidth > EltBitWidth)
1403  DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
1404 
1405  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
1406  Depth + 1))
1407  return true;
1408 
1409  Known = Known2;
1410  if (BitWidth > EltBitWidth)
1411  Known = Known.zext(BitWidth, false /* => any extend */);
1412  break;
1413  }
1414  case ISD::BITCAST: {
1415  SDValue Src = Op.getOperand(0);
1416  EVT SrcVT = Src.getValueType();
1417  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
1418 
1419  // If this is an FP->Int bitcast and if the sign bit is the only
1420  // thing demanded, turn this into a FGETSIGN.
1421  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
1422  DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
1423  SrcVT.isFloatingPoint()) {
1424  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
1426  if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
1427  SrcVT != MVT::f128) {
1428  // Cannot eliminate/lower SHL for f128 yet.
1429  EVT Ty = OpVTLegal ? VT : MVT::i32;
1430  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1431  // place. We expect the SHL to be eliminated by other optimizations.
1432  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
1433  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1434  if (!OpVTLegal && OpVTSizeInBits > 32)
1435  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
1436  unsigned ShVal = Op.getValueSizeInBits() - 1;
1437  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1438  return TLO.CombineTo(Op,
1439  TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
1440  }
1441  }
1442 
1443  // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
1444  // Demand the elt/bit if any of the original elts/bits are demanded.
1445  // TODO - bigendian once we have test coverage.
1446  // TODO - bool vectors once SimplifyDemandedVectorElts has SETCC support.
1447  if (SrcVT.isVector() && NumSrcEltBits > 1 &&
1448  (BitWidth % NumSrcEltBits) == 0 &&
1449  TLO.DAG.getDataLayout().isLittleEndian()) {
1450  unsigned Scale = BitWidth / NumSrcEltBits;
1451  unsigned NumSrcElts = SrcVT.getVectorNumElements();
1452  APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
1453  APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
1454  for (unsigned i = 0; i != Scale; ++i) {
1455  unsigned Offset = i * NumSrcEltBits;
1456  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
1457  if (!Sub.isNullValue()) {
1458  DemandedSrcBits |= Sub;
1459  for (unsigned j = 0; j != NumElts; ++j)
1460  if (DemandedElts[j])
1461  DemandedSrcElts.setBit((j * Scale) + i);
1462  }
1463  }
1464 
1465  APInt KnownSrcUndef, KnownSrcZero;
1466  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
1467  KnownSrcZero, TLO, Depth + 1))
1468  return true;
1469 
1470  KnownBits KnownSrcBits;
1471  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
1472  KnownSrcBits, TLO, Depth + 1))
1473  return true;
1474  }
1475 
1476  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
1477  // recursive call where Known may be useful to the caller.
1478  if (Depth > 0) {
1479  Known = TLO.DAG.computeKnownBits(Op, Depth);
1480  return false;
1481  }
1482  break;
1483  }
1484  case ISD::ADD:
1485  case ISD::MUL:
1486  case ISD::SUB: {
1487  // Add, Sub, and Mul don't demand any bits in positions beyond that
1488  // of the highest bit demanded of them.
1489  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
1490  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
1491  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
1492  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
1493  Depth + 1) ||
1494  SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
1495  Depth + 1) ||
1496  // See if the operation should be performed at a smaller bit width.
1497  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
1498  SDNodeFlags Flags = Op.getNode()->getFlags();
1499  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1500  // Disable the nsw and nuw flags. We can no longer guarantee that we
1501  // won't wrap after simplification.
1502  Flags.setNoSignedWrap(false);
1503  Flags.setNoUnsignedWrap(false);
1504  SDValue NewOp =
1505  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
1506  return TLO.CombineTo(Op, NewOp);
1507  }
1508  return true;
1509  }
1510 
1511  // If we have a constant operand, we may be able to turn it into -1 if we
1512  // do not demand the high bits. This can make the constant smaller to
1513  // encode, allow more general folding, or match specialized instruction
1514  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
1515  // is probably not useful (and could be detrimental).
1517  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
1518  if (C && !C->isAllOnesValue() && !C->isOne() &&
1519  (C->getAPIntValue() | HighMask).isAllOnesValue()) {
1520  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
1521  // We can't guarantee that the new math op doesn't wrap, so explicitly
1522  // clear those flags to prevent folding with a potential existing node
1523  // that has those flags set.
1524  SDNodeFlags Flags;
1525  Flags.setNoSignedWrap(false);
1526  Flags.setNoUnsignedWrap(false);
1527  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
1528  return TLO.CombineTo(Op, NewOp);
1529  }
1530 
1532  }
1533  default:
1534  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
1535  if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
1536  Known, TLO, Depth))
1537  return true;
1538  break;
1539  }
1540 
1541  // Just use computeKnownBits to compute output bits.
1542  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1543  break;
1544  }
1545 
1546  // If we know the value of all of the demanded bits, return this as a
1547  // constant.
1548  if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
1549  // Avoid folding to a constant if any OpaqueConstant is involved.
1550  const SDNode *N = Op.getNode();
1552  E = SDNodeIterator::end(N);
1553  I != E; ++I) {
1554  SDNode *Op = *I;
1555  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
1556  if (C->isOpaque())
1557  return false;
1558  }
1559  // TODO: Handle float bits as well.
1560  if (VT.isInteger())
1561  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
1562  }
1563 
1564  return false;
1565 }
1566 
1568  const APInt &DemandedElts,
1569  APInt &KnownUndef,
1570  APInt &KnownZero,
1571  DAGCombinerInfo &DCI) const {
1572  SelectionDAG &DAG = DCI.DAG;
1573  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
1574  !DCI.isBeforeLegalizeOps());
1575 
1576  bool Simplified =
1577  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
1578  if (Simplified) {
1579  DCI.AddToWorklist(Op.getNode());
1580  DCI.CommitTargetLoweringOpt(TLO);
1581  }
1582  return Simplified;
1583 }
1584 
1585 /// Given a vector binary operation and known undefined elements for each input
1586 /// operand, compute whether each element of the output is undefined.
1588  const APInt &UndefOp0,
1589  const APInt &UndefOp1) {
1590  EVT VT = BO.getValueType();
1591  assert(ISD::isBinaryOp(BO.getNode()) && VT.isVector() && "Vector binop only");
1592 
1593  EVT EltVT = VT.getVectorElementType();
1594  unsigned NumElts = VT.getVectorNumElements();
1595  assert(UndefOp0.getBitWidth() == NumElts &&
1596  UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
1597 
1598  auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
1599  const APInt &UndefVals) {
1600  if (UndefVals[Index])
1601  return DAG.getUNDEF(EltVT);
1602 
1603  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1604  // Try hard to make sure that the getNode() call is not creating temporary
1605  // nodes. Ignore opaque integers because they do not constant fold.
1606  SDValue Elt = BV->getOperand(Index);
1607  auto *C = dyn_cast<ConstantSDNode>(Elt);
1608  if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
1609  return Elt;
1610  }
1611 
1612  return SDValue();
1613  };
1614 
1615  APInt KnownUndef = APInt::getNullValue(NumElts);
1616  for (unsigned i = 0; i != NumElts; ++i) {
1617  // If both inputs for this element are either constant or undef and match
1618  // the element type, compute the constant/undef result for this element of
1619  // the vector.
1620  // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
1621  // not handle FP constants. The code within getNode() should be refactored
1622  // to avoid the danger of creating a bogus temporary node here.
1623  SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
1624  SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
1625  if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
1626  if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
1627  KnownUndef.setBit(i);
1628  }
1629  return KnownUndef;
1630 }
1631 
1633  SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef,
1634  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
1635  bool AssumeSingleUse) const {
1636  EVT VT = Op.getValueType();
1637  APInt DemandedElts = DemandedEltMask;
1638  unsigned NumElts = DemandedElts.getBitWidth();
1639  assert(VT.isVector() && "Expected vector op");
1640  assert(VT.getVectorNumElements() == NumElts &&
1641  "Mask size mismatches value type element count!");
1642 
1643  KnownUndef = KnownZero = APInt::getNullValue(NumElts);
1644 
1645  // Undef operand.
1646  if (Op.isUndef()) {
1647  KnownUndef.setAllBits();
1648  return false;
1649  }
1650 
1651  // If Op has other users, assume that all elements are needed.
1652  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
1653  DemandedElts.setAllBits();
1654 
1655  // Not demanding any elements from Op.
1656  if (DemandedElts == 0) {
1657  KnownUndef.setAllBits();
1658  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1659  }
1660 
1661  // Limit search depth.
1662  if (Depth >= 6)
1663  return false;
1664 
1665  SDLoc DL(Op);
1666  unsigned EltSizeInBits = VT.getScalarSizeInBits();
1667 
1668  switch (Op.getOpcode()) {
1669  case ISD::SCALAR_TO_VECTOR: {
1670  if (!DemandedElts[0]) {
1671  KnownUndef.setAllBits();
1672  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1673  }
1674  KnownUndef.setHighBits(NumElts - 1);
1675  break;
1676  }
1677  case ISD::BITCAST: {
1678  SDValue Src = Op.getOperand(0);
1679  EVT SrcVT = Src.getValueType();
1680 
1681  // We only handle vectors here.
1682  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
1683  if (!SrcVT.isVector())
1684  break;
1685 
1686  // Fast handling of 'identity' bitcasts.
1687  unsigned NumSrcElts = SrcVT.getVectorNumElements();
1688  if (NumSrcElts == NumElts)
1689  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
1690  KnownZero, TLO, Depth + 1);
1691 
1692  APInt SrcZero, SrcUndef;
1693  APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
1694 
1695  // Bitcast from 'large element' src vector to 'small element' vector, we
1696  // must demand a source element if any DemandedElt maps to it.
1697  if ((NumElts % NumSrcElts) == 0) {
1698  unsigned Scale = NumElts / NumSrcElts;
1699  for (unsigned i = 0; i != NumElts; ++i)
1700  if (DemandedElts[i])
1701  SrcDemandedElts.setBit(i / Scale);
1702 
1703  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1704  TLO, Depth + 1))
1705  return true;
1706 
1707  // Try calling SimplifyDemandedBits, converting demanded elts to the bits
1708  // of the large element.
1709  // TODO - bigendian once we have test coverage.
1710  if (TLO.DAG.getDataLayout().isLittleEndian()) {
1711  unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
1712  APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
1713  for (unsigned i = 0; i != NumElts; ++i)
1714  if (DemandedElts[i]) {
1715  unsigned Ofs = (i % Scale) * EltSizeInBits;
1716  SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
1717  }
1718 
1719  KnownBits Known;
1720  if (SimplifyDemandedBits(Src, SrcDemandedBits, Known, TLO, Depth + 1))
1721  return true;
1722  }
1723 
1724  // If the src element is zero/undef then all the output elements will be -
1725  // only demanded elements are guaranteed to be correct.
1726  for (unsigned i = 0; i != NumSrcElts; ++i) {
1727  if (SrcDemandedElts[i]) {
1728  if (SrcZero[i])
1729  KnownZero.setBits(i * Scale, (i + 1) * Scale);
1730  if (SrcUndef[i])
1731  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
1732  }
1733  }
1734  }
1735 
1736  // Bitcast from 'small element' src vector to 'large element' vector, we
1737  // demand all smaller source elements covered by the larger demanded element
1738  // of this vector.
1739  if ((NumSrcElts % NumElts) == 0) {
1740  unsigned Scale = NumSrcElts / NumElts;
1741  for (unsigned i = 0; i != NumElts; ++i)
1742  if (DemandedElts[i])
1743  SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
1744 
1745  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1746  TLO, Depth + 1))
1747  return true;
1748 
1749  // If all the src elements covering an output element are zero/undef, then
1750  // the output element will be as well, assuming it was demanded.
1751  for (unsigned i = 0; i != NumElts; ++i) {
1752  if (DemandedElts[i]) {
1753  if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
1754  KnownZero.setBit(i);
1755  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
1756  KnownUndef.setBit(i);
1757  }
1758  }
1759  }
1760  break;
1761  }
1762  case ISD::BUILD_VECTOR: {
1763  // Check all elements and simplify any unused elements with UNDEF.
1764  if (!DemandedElts.isAllOnesValue()) {
1765  // Don't simplify BROADCASTS.
1766  if (llvm::any_of(Op->op_values(),
1767  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
1768  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
1769  bool Updated = false;
1770  for (unsigned i = 0; i != NumElts; ++i) {
1771  if (!DemandedElts[i] && !Ops[i].isUndef()) {
1772  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
1773  KnownUndef.setBit(i);
1774  Updated = true;
1775  }
1776  }
1777  if (Updated)
1778  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
1779  }
1780  }
1781  for (unsigned i = 0; i != NumElts; ++i) {
1782  SDValue SrcOp = Op.getOperand(i);
1783  if (SrcOp.isUndef()) {
1784  KnownUndef.setBit(i);
1785  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
1786  (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
1787  KnownZero.setBit(i);
1788  }
1789  }
1790  break;
1791  }
1792  case ISD::CONCAT_VECTORS: {
1793  EVT SubVT = Op.getOperand(0).getValueType();
1794  unsigned NumSubVecs = Op.getNumOperands();
1795  unsigned NumSubElts = SubVT.getVectorNumElements();
1796  for (unsigned i = 0; i != NumSubVecs; ++i) {
1797  SDValue SubOp = Op.getOperand(i);
1798  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1799  APInt SubUndef, SubZero;
1800  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
1801  Depth + 1))
1802  return true;
1803  KnownUndef.insertBits(SubUndef, i * NumSubElts);
1804  KnownZero.insertBits(SubZero, i * NumSubElts);
1805  }
1806  break;
1807  }
1808  case ISD::INSERT_SUBVECTOR: {
1809  if (!isa<ConstantSDNode>(Op.getOperand(2)))
1810  break;
1811  SDValue Base = Op.getOperand(0);
1812  SDValue Sub = Op.getOperand(1);
1813  EVT SubVT = Sub.getValueType();
1814  unsigned NumSubElts = SubVT.getVectorNumElements();
1815  const APInt &Idx = Op.getConstantOperandAPInt(2);
1816  if (Idx.ugt(NumElts - NumSubElts))
1817  break;
1818  unsigned SubIdx = Idx.getZExtValue();
1819  APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
1820  APInt SubUndef, SubZero;
1821  if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
1822  Depth + 1))
1823  return true;
1824  APInt BaseElts = DemandedElts;
1825  BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
1826  if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
1827  Depth + 1))
1828  return true;
1829  KnownUndef.insertBits(SubUndef, SubIdx);
1830  KnownZero.insertBits(SubZero, SubIdx);
1831  break;
1832  }
1833  case ISD::EXTRACT_SUBVECTOR: {
1834  SDValue Src = Op.getOperand(0);
1836  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1837  if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
1838  // Offset the demanded elts by the subvector index.
1839  uint64_t Idx = SubIdx->getZExtValue();
1840  APInt SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1841  APInt SrcUndef, SrcZero;
1842  if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
1843  Depth + 1))
1844  return true;
1845  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
1846  KnownZero = SrcZero.extractBits(NumElts, Idx);
1847  }
1848  break;
1849  }
1850  case ISD::INSERT_VECTOR_ELT: {
1851  SDValue Vec = Op.getOperand(0);
1852  SDValue Scl = Op.getOperand(1);
1853  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1854 
1855  // For a legal, constant insertion index, if we don't need this insertion
1856  // then strip it, else remove it from the demanded elts.
1857  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
1858  unsigned Idx = CIdx->getZExtValue();
1859  if (!DemandedElts[Idx])
1860  return TLO.CombineTo(Op, Vec);
1861 
1862  APInt DemandedVecElts(DemandedElts);
1863  DemandedVecElts.clearBit(Idx);
1864  if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
1865  KnownZero, TLO, Depth + 1))
1866  return true;
1867 
1868  KnownUndef.clearBit(Idx);
1869  if (Scl.isUndef())
1870  KnownUndef.setBit(Idx);
1871 
1872  KnownZero.clearBit(Idx);
1873  if (isNullConstant(Scl) || isNullFPConstant(Scl))
1874  KnownZero.setBit(Idx);
1875  break;
1876  }
1877 
1878  APInt VecUndef, VecZero;
1879  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
1880  Depth + 1))
1881  return true;
1882  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
1883  break;
1884  }
1885  case ISD::VSELECT: {
1886  // Try to transform the select condition based on the current demanded
1887  // elements.
1888  // TODO: If a condition element is undef, we can choose from one arm of the
1889  // select (and if one arm is undef, then we can propagate that to the
1890  // result).
1891  // TODO - add support for constant vselect masks (see IR version of this).
1892  APInt UnusedUndef, UnusedZero;
1893  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
1894  UnusedZero, TLO, Depth + 1))
1895  return true;
1896 
1897  // See if we can simplify either vselect operand.
1898  APInt DemandedLHS(DemandedElts);
1899  APInt DemandedRHS(DemandedElts);
1900  APInt UndefLHS, ZeroLHS;
1901  APInt UndefRHS, ZeroRHS;
1902  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
1903  ZeroLHS, TLO, Depth + 1))
1904  return true;
1905  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
1906  ZeroRHS, TLO, Depth + 1))
1907  return true;
1908 
1909  KnownUndef = UndefLHS & UndefRHS;
1910  KnownZero = ZeroLHS & ZeroRHS;
1911  break;
1912  }
1913  case ISD::VECTOR_SHUFFLE: {
1914  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1915 
1916  // Collect demanded elements from shuffle operands..
1917  APInt DemandedLHS(NumElts, 0);
1918  APInt DemandedRHS(NumElts, 0);
1919  for (unsigned i = 0; i != NumElts; ++i) {
1920  int M = ShuffleMask[i];
1921  if (M < 0 || !DemandedElts[i])
1922  continue;
1923  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1924  if (M < (int)NumElts)
1925  DemandedLHS.setBit(M);
1926  else
1927  DemandedRHS.setBit(M - NumElts);
1928  }
1929 
1930  // See if we can simplify either shuffle operand.
1931  APInt UndefLHS, ZeroLHS;
1932  APInt UndefRHS, ZeroRHS;
1933  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
1934  ZeroLHS, TLO, Depth + 1))
1935  return true;
1936  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
1937  ZeroRHS, TLO, Depth + 1))
1938  return true;
1939 
1940  // Simplify mask using undef elements from LHS/RHS.
1941  bool Updated = false;
1942  bool IdentityLHS = true, IdentityRHS = true;
1943  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
1944  for (unsigned i = 0; i != NumElts; ++i) {
1945  int &M = NewMask[i];
1946  if (M < 0)
1947  continue;
1948  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
1949  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
1950  Updated = true;
1951  M = -1;
1952  }
1953  IdentityLHS &= (M < 0) || (M == (int)i);
1954  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
1955  }
1956 
1957  // Update legal shuffle masks based on demanded elements if it won't reduce
1958  // to Identity which can cause premature removal of the shuffle mask.
1959  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps &&
1960  isShuffleMaskLegal(NewMask, VT))
1961  return TLO.CombineTo(Op,
1962  TLO.DAG.getVectorShuffle(VT, DL, Op.getOperand(0),
1963  Op.getOperand(1), NewMask));
1964 
1965  // Propagate undef/zero elements from LHS/RHS.
1966  for (unsigned i = 0; i != NumElts; ++i) {
1967  int M = ShuffleMask[i];
1968  if (M < 0) {
1969  KnownUndef.setBit(i);
1970  } else if (M < (int)NumElts) {
1971  if (UndefLHS[M])
1972  KnownUndef.setBit(i);
1973  if (ZeroLHS[M])
1974  KnownZero.setBit(i);
1975  } else {
1976  if (UndefRHS[M - NumElts])
1977  KnownUndef.setBit(i);
1978  if (ZeroRHS[M - NumElts])
1979  KnownZero.setBit(i);
1980  }
1981  }
1982  break;
1983  }
1986  APInt SrcUndef, SrcZero;
1987  SDValue Src = Op.getOperand(0);
1988  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1989  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
1990  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
1991  Depth + 1))
1992  return true;
1993  KnownZero = SrcZero.zextOrTrunc(NumElts);
1994  KnownUndef = SrcUndef.zextOrTrunc(NumElts);
1995 
1997  // zext(undef) upper bits are guaranteed to be zero.
1998  if (DemandedElts.isSubsetOf(KnownUndef))
1999  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2000  KnownUndef.clearAllBits();
2001  }
2002  break;
2003  }
2004 
2005  // TODO: There are more binop opcodes that could be handled here - MUL, MIN,
2006  // MAX, saturated math, etc.
2007  case ISD::OR:
2008  case ISD::XOR:
2009  case ISD::ADD:
2010  case ISD::SUB:
2011  case ISD::FADD:
2012  case ISD::FSUB:
2013  case ISD::FMUL:
2014  case ISD::FDIV:
2015  case ISD::FREM: {
2016  APInt UndefRHS, ZeroRHS;
2017  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
2018  ZeroRHS, TLO, Depth + 1))
2019  return true;
2020  APInt UndefLHS, ZeroLHS;
2021  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
2022  ZeroLHS, TLO, Depth + 1))
2023  return true;
2024 
2025  KnownZero = ZeroLHS & ZeroRHS;
2026  KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2027  break;
2028  }
2029  case ISD::AND: {
2030  APInt SrcUndef, SrcZero;
2031  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
2032  SrcZero, TLO, Depth + 1))
2033  return true;
2034  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2035  KnownZero, TLO, Depth + 1))
2036  return true;
2037 
2038  // If either side has a zero element, then the result element is zero, even
2039  // if the other is an UNDEF.
2040  // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2041  // and then handle 'and' nodes with the rest of the binop opcodes.
2042  KnownZero |= SrcZero;
2043  KnownUndef &= SrcUndef;
2044  KnownUndef &= ~KnownZero;
2045  break;
2046  }
2047  case ISD::TRUNCATE:
2048  case ISD::SIGN_EXTEND:
2049  case ISD::ZERO_EXTEND:
2050  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2051  KnownZero, TLO, Depth + 1))
2052  return true;
2053 
2054  if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2055  // zext(undef) upper bits are guaranteed to be zero.
2056  if (DemandedElts.isSubsetOf(KnownUndef))
2057  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2058  KnownUndef.clearAllBits();
2059  }
2060  break;
2061  default: {
2062  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2063  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2064  KnownZero, TLO, Depth))
2065  return true;
2066  } else {
2067  KnownBits Known;
2068  APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2069  if (SimplifyDemandedBits(Op, DemandedBits, DemandedEltMask, Known, TLO,
2070  Depth, AssumeSingleUse))
2071  return true;
2072  }
2073  break;
2074  }
2075  }
2076  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
2077 
2078  // Constant fold all undef cases.
2079  // TODO: Handle zero cases as well.
2080  if (DemandedElts.isSubsetOf(KnownUndef))
2081  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2082 
2083  return false;
2084 }
2085 
2086 /// Determine which of the bits specified in Mask are known to be either zero or
2087 /// one and return them in the Known.
2089  KnownBits &Known,
2090  const APInt &DemandedElts,
2091  const SelectionDAG &DAG,
2092  unsigned Depth) const {
2096  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2097  "Should use MaskedValueIsZero if you don't know whether Op"
2098  " is a target node!");
2099  Known.resetAll();
2100 }
2101 
2103  KnownBits &Known,
2104  const APInt &DemandedElts,
2105  const SelectionDAG &DAG,
2106  unsigned Depth) const {
2107  assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex");
2108 
2109  if (unsigned Align = DAG.InferPtrAlignment(Op)) {
2110  // The low bits are known zero if the pointer is aligned.
2111  Known.Zero.setLowBits(Log2_32(Align));
2112  }
2113 }
2114 
2115 /// This method can be implemented by targets that want to expose additional
2116 /// information about sign bits to the DAG Combiner.
2118  const APInt &,
2119  const SelectionDAG &,
2120  unsigned Depth) const {
2124  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2125  "Should use ComputeNumSignBits if you don't know whether Op"
2126  " is a target node!");
2127  return 1;
2128 }
2129 
2131  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2132  TargetLoweringOpt &TLO, unsigned Depth) const {
2136  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2137  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
2138  " is a target node!");
2139  return false;
2140 }
2141 
2143  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2144  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
2148  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2149  "Should use SimplifyDemandedBits if you don't know whether Op"
2150  " is a target node!");
2151  computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
2152  return false;
2153 }
2154 
2156  const SelectionDAG &DAG,
2157  bool SNaN,
2158  unsigned Depth) const {
2162  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2163  "Should use isKnownNeverNaN if you don't know whether Op"
2164  " is a target node!");
2165  return false;
2166 }
2167 
2168 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
2169 // work with truncating build vectors and vectors with elements of less than
2170 // 8 bits.
2172  if (!N)
2173  return false;
2174 
2175  APInt CVal;
2176  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
2177  CVal = CN->getAPIntValue();
2178  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
2179  auto *CN = BV->getConstantSplatNode();
2180  if (!CN)
2181  return false;
2182 
2183  // If this is a truncating build vector, truncate the splat value.
2184  // Otherwise, we may fail to match the expected values below.
2185  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
2186  CVal = CN->getAPIntValue();
2187  if (BVEltWidth < CVal.getBitWidth())
2188  CVal = CVal.trunc(BVEltWidth);
2189  } else {
2190  return false;
2191  }
2192 
2193  switch (getBooleanContents(N->getValueType(0))) {
2195  return CVal[0];
2197  return CVal.isOneValue();
2199  return CVal.isAllOnesValue();
2200  }
2201 
2202  llvm_unreachable("Invalid boolean contents");
2203 }
2204 
2206  if (!N)
2207  return false;
2208 
2209  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
2210  if (!CN) {
2212  if (!BV)
2213  return false;
2214 
2215  // Only interested in constant splats, we don't care about undef
2216  // elements in identifying boolean constants and getConstantSplatNode
2217  // returns NULL if all ops are undef;
2218  CN = BV->getConstantSplatNode();
2219  if (!CN)
2220  return false;
2221  }
2222 
2224  return !CN->getAPIntValue()[0];
2225 
2226  return CN->isNullValue();
2227 }
2228 
2230  bool SExt) const {
2231  if (VT == MVT::i1)
2232  return N->isOne();
2233 
2235  switch (Cnt) {
2237  // An extended value of 1 is always true, unless its original type is i1,
2238  // in which case it will be sign extended to -1.
2239  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
2242  return N->isAllOnesValue() && SExt;
2243  }
2244  llvm_unreachable("Unexpected enumeration.");
2245 }
2246 
2247 /// This helper function of SimplifySetCC tries to optimize the comparison when
2248 /// either operand of the SetCC node is a bitwise-and instruction.
2249 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2250  ISD::CondCode Cond, const SDLoc &DL,
2251  DAGCombinerInfo &DCI) const {
2252  // Match these patterns in any of their permutations:
2253  // (X & Y) == Y
2254  // (X & Y) != Y
2255  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
2256  std::swap(N0, N1);
2257 
2258  EVT OpVT = N0.getValueType();
2259  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
2260  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
2261  return SDValue();
2262 
2263  SDValue X, Y;
2264  if (N0.getOperand(0) == N1) {
2265  X = N0.getOperand(1);
2266  Y = N0.getOperand(0);
2267  } else if (N0.getOperand(1) == N1) {
2268  X = N0.getOperand(0);
2269  Y = N0.getOperand(1);
2270  } else {
2271  return SDValue();
2272  }
2273 
2274  SelectionDAG &DAG = DCI.DAG;
2275  SDValue Zero = DAG.getConstant(0, DL, OpVT);
2276  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
2277  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
2278  // Note that where Y is variable and is known to have at most one bit set
2279  // (for example, if it is Z & 1) we cannot do this; the expressions are not
2280  // equivalent when Y == 0.
2281  Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
2282  if (DCI.isBeforeLegalizeOps() ||
2283  isCondCodeLegal(Cond, N0.getSimpleValueType()))
2284  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
2285  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
2286  // If the target supports an 'and-not' or 'and-complement' logic operation,
2287  // try to use that to make a comparison operation more efficient.
2288  // But don't do this transform if the mask is a single bit because there are
2289  // more efficient ways to deal with that case (for example, 'bt' on x86 or
2290  // 'rlwinm' on PPC).
2291 
2292  // Bail out if the compare operand that we want to turn into a zero is
2293  // already a zero (otherwise, infinite loop).
2294  auto *YConst = dyn_cast<ConstantSDNode>(Y);
2295  if (YConst && YConst->isNullValue())
2296  return SDValue();
2297 
2298  // Transform this into: ~X & Y == 0.
2299  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
2300  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
2301  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
2302  }
2303 
2304  return SDValue();
2305 }
2306 
2307 /// There are multiple IR patterns that could be checking whether certain
2308 /// truncation of a signed number would be lossy or not. The pattern which is
2309 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
2310 /// We are looking for the following pattern: (KeptBits is a constant)
2311 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
2312 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
2313 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
2314 /// We will unfold it into the natural trunc+sext pattern:
2315 /// ((%x << C) a>> C) dstcond %x
2316 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
2317 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
2318  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
2319  const SDLoc &DL) const {
2320  // We must be comparing with a constant.
2321  ConstantSDNode *C1;
2322  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
2323  return SDValue();
2324 
2325  // N0 should be: add %x, (1 << (KeptBits-1))
2326  if (N0->getOpcode() != ISD::ADD)
2327  return SDValue();
2328 
2329  // And we must be 'add'ing a constant.
2330  ConstantSDNode *C01;
2331  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
2332  return SDValue();
2333 
2334  SDValue X = N0->getOperand(0);
2335  EVT XVT = X.getValueType();
2336 
2337  // Validate constants ...
2338 
2339  APInt I1 = C1->getAPIntValue();
2340 
2341  ISD::CondCode NewCond;
2342  if (Cond == ISD::CondCode::SETULT) {
2343  NewCond = ISD::CondCode::SETEQ;
2344  } else if (Cond == ISD::CondCode::SETULE) {
2345  NewCond = ISD::CondCode::SETEQ;
2346  // But need to 'canonicalize' the constant.
2347  I1 += 1;
2348  } else if (Cond == ISD::CondCode::SETUGT) {
2349  NewCond = ISD::CondCode::SETNE;
2350  // But need to 'canonicalize' the constant.
2351  I1 += 1;
2352  } else if (Cond == ISD::CondCode::SETUGE) {
2353  NewCond = ISD::CondCode::SETNE;
2354  } else
2355  return SDValue();
2356 
2357  APInt I01 = C01->getAPIntValue();
2358 
2359  auto checkConstants = [&I1, &I01]() -> bool {
2360  // Both of them must be power-of-two, and the constant from setcc is bigger.
2361  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
2362  };
2363 
2364  if (checkConstants()) {
2365  // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
2366  } else {
2367  // What if we invert constants? (and the target predicate)
2368  I1.negate();
2369  I01.negate();
2370  NewCond = getSetCCInverse(NewCond, /*isInteger=*/true);
2371  if (!checkConstants())
2372  return SDValue();
2373  // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
2374  }
2375 
2376  // They are power-of-two, so which bit is set?
2377  const unsigned KeptBits = I1.logBase2();
2378  const unsigned KeptBitsMinusOne = I01.logBase2();
2379 
2380  // Magic!
2381  if (KeptBits != (KeptBitsMinusOne + 1))
2382  return SDValue();
2383  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
2384 
2385  // We don't want to do this in every single case.
2386  SelectionDAG &DAG = DCI.DAG;
2387  if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
2388  XVT, KeptBits))
2389  return SDValue();
2390 
2391  const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
2392  assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
2393 
2394  // Unfold into: ((%x << C) a>> C) cond %x
2395  // Where 'cond' will be either 'eq' or 'ne'.
2396  SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
2397  SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
2398  SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
2399  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
2400 
2401  return T2;
2402 }
2403 
2404 /// Try to fold an equality comparison with a {add/sub/xor} binary operation as
2405 /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
2406 /// handle the commuted versions of these patterns.
2407 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
2408  ISD::CondCode Cond, const SDLoc &DL,
2409  DAGCombinerInfo &DCI) const {
2410  unsigned BOpcode = N0.getOpcode();
2411  assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
2412  "Unexpected binop");
2413  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
2414 
2415  // (X + Y) == X --> Y == 0
2416  // (X - Y) == X --> Y == 0
2417  // (X ^ Y) == X --> Y == 0
2418  SelectionDAG &DAG = DCI.DAG;
2419  EVT OpVT = N0.getValueType();
2420  SDValue X = N0.getOperand(0);
2421  SDValue Y = N0.getOperand(1);
2422  if (X == N1)
2423  return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
2424 
2425  if (Y != N1)
2426  return SDValue();
2427 
2428  // (X + Y) == Y --> X == 0
2429  // (X ^ Y) == Y --> X == 0
2430  if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
2431  return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
2432 
2433  // The shift would not be valid if the operands are boolean (i1).
2434  if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
2435  return SDValue();
2436 
2437  // (X - Y) == Y --> X == Y << 1
2438  EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
2439  !DCI.isBeforeLegalize());
2440  SDValue One = DAG.getConstant(1, DL, ShiftVT);
2441  SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
2442  if (!DCI.isCalledByLegalizer())
2443  DCI.AddToWorklist(YShl1.getNode());
2444  return DAG.getSetCC(DL, VT, X, YShl1, Cond);
2445 }
2446 
2447 /// Try to simplify a setcc built with the specified operands and cc. If it is
2448 /// unable to simplify it, return a null SDValue.
2450  ISD::CondCode Cond, bool foldBooleans,
2451  DAGCombinerInfo &DCI,
2452  const SDLoc &dl) const {
2453  SelectionDAG &DAG = DCI.DAG;
2454  EVT OpVT = N0.getValueType();
2455 
2456  // Constant fold or commute setcc.
2457  if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
2458  return Fold;
2459 
2460  // Ensure that the constant occurs on the RHS and fold constant comparisons.
2461  // TODO: Handle non-splat vector constants. All undef causes trouble.
2462  ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
2463  if (isConstOrConstSplat(N0) &&
2464  (DCI.isBeforeLegalizeOps() ||
2465  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
2466  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
2467 
2468  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
2469  const APInt &C1 = N1C->getAPIntValue();
2470 
2471  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
2472  // equality comparison, then we're just comparing whether X itself is
2473  // zero.
2474  if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
2475  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
2476  N0.getOperand(1).getOpcode() == ISD::Constant) {
2477  const APInt &ShAmt = N0.getConstantOperandAPInt(1);
2478  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2479  ShAmt == Log2_32(N0.getValueSizeInBits())) {
2480  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
2481  // (srl (ctlz x), 5) == 0 -> X != 0
2482  // (srl (ctlz x), 5) != 1 -> X != 0
2483  Cond = ISD::SETNE;
2484  } else {
2485  // (srl (ctlz x), 5) != 0 -> X == 0
2486  // (srl (ctlz x), 5) == 1 -> X == 0
2487  Cond = ISD::SETEQ;
2488  }
2489  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
2490  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
2491  Zero, Cond);
2492  }
2493  }
2494 
2495  SDValue CTPOP = N0;
2496  // Look through truncs that don't change the value of a ctpop.
2497  if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
2498  CTPOP = N0.getOperand(0);
2499 
2500  if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
2501  (N0 == CTPOP ||
2503  EVT CTVT = CTPOP.getValueType();
2504  SDValue CTOp = CTPOP.getOperand(0);
2505 
2506  // (ctpop x) u< 2 -> (x & x-1) == 0
2507  // (ctpop x) u> 1 -> (x & x-1) != 0
2508  if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
2509  SDValue Sub = DAG.getNode(ISD::SUB, dl, CTVT, CTOp,
2510  DAG.getConstant(1, dl, CTVT));
2511  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Sub);
2513  return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
2514  }
2515 
2516  // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal.
2517  }
2518 
2519  // (zext x) == C --> x == (trunc C)
2520  // (sext x) == C --> x == (trunc C)
2521  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2522  DCI.isBeforeLegalize() && N0->hasOneUse()) {
2523  unsigned MinBits = N0.getValueSizeInBits();
2524  SDValue PreExt;
2525  bool Signed = false;
2526  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
2527  // ZExt
2528  MinBits = N0->getOperand(0).getValueSizeInBits();
2529  PreExt = N0->getOperand(0);
2530  } else if (N0->getOpcode() == ISD::AND) {
2531  // DAGCombine turns costly ZExts into ANDs
2532  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
2533  if ((C->getAPIntValue()+1).isPowerOf2()) {
2534  MinBits = C->getAPIntValue().countTrailingOnes();
2535  PreExt = N0->getOperand(0);
2536  }
2537  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
2538  // SExt
2539  MinBits = N0->getOperand(0).getValueSizeInBits();
2540  PreExt = N0->getOperand(0);
2541  Signed = true;
2542  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
2543  // ZEXTLOAD / SEXTLOAD
2544  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
2545  MinBits = LN0->getMemoryVT().getSizeInBits();
2546  PreExt = N0;
2547  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
2548  Signed = true;
2549  MinBits = LN0->getMemoryVT().getSizeInBits();
2550  PreExt = N0;
2551  }
2552  }
2553 
2554  // Figure out how many bits we need to preserve this constant.
2555  unsigned ReqdBits = Signed ?
2556  C1.getBitWidth() - C1.getNumSignBits() + 1 :
2557  C1.getActiveBits();
2558 
2559  // Make sure we're not losing bits from the constant.
2560  if (MinBits > 0 &&
2561  MinBits < C1.getBitWidth() &&
2562  MinBits >= ReqdBits) {
2563  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
2564  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
2565  // Will get folded away.
2566  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
2567  if (MinBits == 1 && C1 == 1)
2568  // Invert the condition.
2569  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
2570  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2571  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
2572  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
2573  }
2574 
2575  // If truncating the setcc operands is not desirable, we can still
2576  // simplify the expression in some cases:
2577  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
2578  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
2579  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
2580  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
2581  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
2582  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
2583  SDValue TopSetCC = N0->getOperand(0);
2584  unsigned N0Opc = N0->getOpcode();
2585  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
2586  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
2587  TopSetCC.getOpcode() == ISD::SETCC &&
2588  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
2589  (isConstFalseVal(N1C) ||
2590  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
2591 
2592  bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
2593  (!N1C->isNullValue() && Cond == ISD::SETNE);
2594 
2595  if (!Inverse)
2596  return TopSetCC;
2597 
2599  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
2600  TopSetCC.getOperand(0).getValueType().isInteger());
2601  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
2602  TopSetCC.getOperand(1),
2603  InvCond);
2604  }
2605  }
2606  }
2607 
2608  // If the LHS is '(and load, const)', the RHS is 0, the test is for
2609  // equality or unsigned, and all 1 bits of the const are in the same
2610  // partial word, see if we can shorten the load.
2611  if (DCI.isBeforeLegalize() &&
2612  !ISD::isSignedIntSetCC(Cond) &&
2613  N0.getOpcode() == ISD::AND && C1 == 0 &&
2614  N0.getNode()->hasOneUse() &&
2615  isa<LoadSDNode>(N0.getOperand(0)) &&
2616  N0.getOperand(0).getNode()->hasOneUse() &&
2617  isa<ConstantSDNode>(N0.getOperand(1))) {
2618  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
2619  APInt bestMask;
2620  unsigned bestWidth = 0, bestOffset = 0;
2621  if (!Lod->isVolatile() && Lod->isUnindexed()) {
2622  unsigned origWidth = N0.getValueSizeInBits();
2623  unsigned maskWidth = origWidth;
2624  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
2625  // 8 bits, but have to be careful...
2626  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
2627  origWidth = Lod->getMemoryVT().getSizeInBits();
2628  const APInt &Mask = N0.getConstantOperandAPInt(1);
2629  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
2630  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
2631  for (unsigned offset=0; offset<origWidth/width; offset++) {
2632  if (Mask.isSubsetOf(newMask)) {
2633  if (DAG.getDataLayout().isLittleEndian())
2634  bestOffset = (uint64_t)offset * (width/8);
2635  else
2636  bestOffset = (origWidth/width - offset - 1) * (width/8);
2637  bestMask = Mask.lshr(offset * (width/8) * 8);
2638  bestWidth = width;
2639  break;
2640  }
2641  newMask <<= width;
2642  }
2643  }
2644  }
2645  if (bestWidth) {
2646  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
2647  if (newVT.isRound() &&
2648  shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
2649  EVT PtrType = Lod->getOperand(1).getValueType();
2650  SDValue Ptr = Lod->getBasePtr();
2651  if (bestOffset != 0)
2652  Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
2653  DAG.getConstant(bestOffset, dl, PtrType));
2654  unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
2655  SDValue NewLoad = DAG.getLoad(
2656  newVT, dl, Lod->getChain(), Ptr,
2657  Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
2658  return DAG.getSetCC(dl, VT,
2659  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
2660  DAG.getConstant(bestMask.trunc(bestWidth),
2661  dl, newVT)),
2662  DAG.getConstant(0LL, dl, newVT), Cond);
2663  }
2664  }
2665  }
2666 
2667  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
2668  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
2669  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
2670 
2671  // If the comparison constant has bits in the upper part, the
2672  // zero-extended value could never match.
2673  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
2674  C1.getBitWidth() - InSize))) {
2675  switch (Cond) {
2676  case ISD::SETUGT:
2677  case ISD::SETUGE:
2678  case ISD::SETEQ:
2679  return DAG.getConstant(0, dl, VT);
2680  case ISD::SETULT:
2681  case ISD::SETULE:
2682  case ISD::SETNE:
2683  return DAG.getConstant(1, dl, VT);
2684  case ISD::SETGT:
2685  case ISD::SETGE:
2686  // True if the sign bit of C1 is set.
2687  return DAG.getConstant(C1.isNegative(), dl, VT);
2688  case ISD::SETLT:
2689  case ISD::SETLE:
2690  // True if the sign bit of C1 isn't set.
2691  return DAG.getConstant(C1.isNonNegative(), dl, VT);
2692  default:
2693  break;
2694  }
2695  }
2696 
2697  // Otherwise, we can perform the comparison with the low bits.
2698  switch (Cond) {
2699  case ISD::SETEQ:
2700  case ISD::SETNE:
2701  case ISD::SETUGT:
2702  case ISD::SETUGE:
2703  case ISD::SETULT:
2704  case ISD::SETULE: {
2705  EVT newVT = N0.getOperand(0).getValueType();
2706  if (DCI.isBeforeLegalizeOps() ||
2707  (isOperationLegal(ISD::SETCC, newVT) &&
2708  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
2709  EVT NewSetCCVT =
2710  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), newVT);
2711  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
2712 
2713  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
2714  NewConst, Cond);
2715  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
2716  }
2717  break;
2718  }
2719  default:
2720  break; // todo, be more careful with signed comparisons
2721  }
2722  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2723  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2724  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
2725  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
2726  EVT ExtDstTy = N0.getValueType();
2727  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
2728 
2729  // If the constant doesn't fit into the number of bits for the source of
2730  // the sign extension, it is impossible for both sides to be equal.
2731  if (C1.getMinSignedBits() > ExtSrcTyBits)
2732  return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
2733 
2734  SDValue ZextOp;
2735  EVT Op0Ty = N0.getOperand(0).getValueType();
2736  if (Op0Ty == ExtSrcTy) {
2737  ZextOp = N0.getOperand(0);
2738  } else {
2739  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
2740  ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
2741  DAG.getConstant(Imm, dl, Op0Ty));
2742  }
2743  if (!DCI.isCalledByLegalizer())
2744  DCI.AddToWorklist(ZextOp.getNode());
2745  // Otherwise, make this a use of a zext.
2746  return DAG.getSetCC(dl, VT, ZextOp,
2748  ExtDstTyBits,
2749  ExtSrcTyBits),
2750  dl, ExtDstTy),
2751  Cond);
2752  } else if ((N1C->isNullValue() || N1C->isOne()) &&
2753  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2754  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
2755  if (N0.getOpcode() == ISD::SETCC &&
2756  isTypeLegal(VT) && VT.bitsLE(N0.getValueType())) {
2757  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
2758  if (TrueWhenTrue)
2759  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
2760  // Invert the condition.
2761  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
2762  CC = ISD::getSetCCInverse(CC,
2763  N0.getOperand(0).getValueType().isInteger());
2764  if (DCI.isBeforeLegalizeOps() ||
2765  isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
2766  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
2767  }
2768 
2769  if ((N0.getOpcode() == ISD::XOR ||
2770  (N0.getOpcode() == ISD::AND &&
2771  N0.getOperand(0).getOpcode() == ISD::XOR &&
2772  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
2773  isa<ConstantSDNode>(N0.getOperand(1)) &&
2774  cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
2775  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
2776  // can only do this if the top bits are known zero.
2777  unsigned BitWidth = N0.getValueSizeInBits();
2778  if (DAG.MaskedValueIsZero(N0,
2779  APInt::getHighBitsSet(BitWidth,
2780  BitWidth-1))) {
2781  // Okay, get the un-inverted input value.
2782  SDValue Val;
2783  if (N0.getOpcode() == ISD::XOR) {
2784  Val = N0.getOperand(0);
2785  } else {
2786  assert(N0.getOpcode() == ISD::AND &&
2787  N0.getOperand(0).getOpcode() == ISD::XOR);
2788  // ((X^1)&1)^1 -> X & 1
2789  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
2790  N0.getOperand(0).getOperand(0),
2791  N0.getOperand(1));
2792  }
2793 
2794  return DAG.getSetCC(dl, VT, Val, N1,
2795  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2796  }
2797  } else if (N1C->isOne() &&
2798  (VT == MVT::i1 ||
2799  getBooleanContents(N0->getValueType(0)) ==
2801  SDValue Op0 = N0;
2802  if (Op0.getOpcode() == ISD::TRUNCATE)
2803  Op0 = Op0.getOperand(0);
2804 
2805  if ((Op0.getOpcode() == ISD::XOR) &&
2806  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
2807  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
2808  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
2809  Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
2810  return DAG.getSetCC(dl, VT, Op0.getOperand(0), Op0.getOperand(1),
2811  Cond);
2812  }
2813  if (Op0.getOpcode() == ISD::AND &&
2814  isa<ConstantSDNode>(Op0.getOperand(1)) &&
2815  cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
2816  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
2817  if (Op0.getValueType().bitsGT(VT))
2818  Op0 = DAG.getNode(ISD::AND, dl, VT,
2819  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
2820  DAG.getConstant(1, dl, VT));
2821  else if (Op0.getValueType().bitsLT(VT))
2822  Op0 = DAG.getNode(ISD::AND, dl, VT,
2823  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
2824  DAG.getConstant(1, dl, VT));
2825 
2826  return DAG.getSetCC(dl, VT, Op0,
2827  DAG.getConstant(0, dl, Op0.getValueType()),
2828  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2829  }
2830  if (Op0.getOpcode() == ISD::AssertZext &&
2831  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
2832  return DAG.getSetCC(dl, VT, Op0,
2833  DAG.getConstant(0, dl, Op0.getValueType()),
2834  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2835  }
2836  }
2837 
2838  if (SDValue V =
2839  optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
2840  return V;
2841  }
2842 
2843  // These simplifications apply to splat vectors as well.
2844  // TODO: Handle more splat vector cases.
2845  if (auto *N1C = isConstOrConstSplat(N1)) {
2846  const APInt &C1 = N1C->getAPIntValue();
2847 
2848  APInt MinVal, MaxVal;
2849  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
2850  if (ISD::isSignedIntSetCC(Cond)) {
2851  MinVal = APInt::getSignedMinValue(OperandBitSize);
2852  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
2853  } else {
2854  MinVal = APInt::getMinValue(OperandBitSize);
2855  MaxVal = APInt::getMaxValue(OperandBitSize);
2856  }
2857 
2858  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
2859  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
2860  // X >= MIN --> true
2861  if (C1 == MinVal)
2862  return DAG.getBoolConstant(true, dl, VT, OpVT);
2863 
2864  if (!VT.isVector()) { // TODO: Support this for vectors.
2865  // X >= C0 --> X > (C0 - 1)
2866  APInt C = C1 - 1;
2867  ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
2868  if ((DCI.isBeforeLegalizeOps() ||
2869  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2870  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2872  return DAG.getSetCC(dl, VT, N0,
2873  DAG.getConstant(C, dl, N1.getValueType()),
2874  NewCC);
2875  }
2876  }
2877  }
2878 
2879  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
2880  // X <= MAX --> true
2881  if (C1 == MaxVal)
2882  return DAG.getBoolConstant(true, dl, VT, OpVT);
2883 
2884  // X <= C0 --> X < (C0 + 1)
2885  if (!VT.isVector()) { // TODO: Support this for vectors.
2886  APInt C = C1 + 1;
2887  ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
2888  if ((DCI.isBeforeLegalizeOps() ||
2889  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2890  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2892  return DAG.getSetCC(dl, VT, N0,
2893  DAG.getConstant(C, dl, N1.getValueType()),
2894  NewCC);
2895  }
2896  }
2897  }
2898 
2899  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
2900  if (C1 == MinVal)
2901  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
2902 
2903  // TODO: Support this for vectors after legalize ops.
2904  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2905  // Canonicalize setlt X, Max --> setne X, Max
2906  if (C1 == MaxVal)
2907  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2908 
2909  // If we have setult X, 1, turn it into seteq X, 0
2910  if (C1 == MinVal+1)
2911  return DAG.getSetCC(dl, VT, N0,
2912  DAG.getConstant(MinVal, dl, N0.getValueType()),
2913  ISD::SETEQ);
2914  }
2915  }
2916 
2917  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
2918  if (C1 == MaxVal)
2919  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
2920 
2921  // TODO: Support this for vectors after legalize ops.
2922  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2923  // Canonicalize setgt X, Min --> setne X, Min
2924  if (C1 == MinVal)
2925  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2926 
2927  // If we have setugt X, Max-1, turn it into seteq X, Max
2928  if (C1 == MaxVal-1)
2929  return DAG.getSetCC(dl, VT, N0,
2930  DAG.getConstant(MaxVal, dl, N0.getValueType()),
2931  ISD::SETEQ);
2932  }
2933  }
2934 
2935  // If we have "setcc X, C0", check to see if we can shrink the immediate
2936  // by changing cc.
2937  // TODO: Support this for vectors after legalize ops.
2938  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2939  // SETUGT X, SINTMAX -> SETLT X, 0
2940  if (Cond == ISD::SETUGT &&
2941  C1 == APInt::getSignedMaxValue(OperandBitSize))
2942  return DAG.getSetCC(dl, VT, N0,
2943  DAG.getConstant(0, dl, N1.getValueType()),
2944  ISD::SETLT);
2945 
2946  // SETULT X, SINTMIN -> SETGT X, -1
2947  if (Cond == ISD::SETULT &&
2948  C1 == APInt::getSignedMinValue(OperandBitSize)) {
2949  SDValue ConstMinusOne =
2950  DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
2951  N1.getValueType());
2952  return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
2953  }
2954  }
2955  }
2956 
2957  // Back to non-vector simplifications.
2958  // TODO: Can we do these for vector splats?
2959  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
2960  const APInt &C1 = N1C->getAPIntValue();
2961 
2962  // Fold bit comparisons when we can.
2963  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2964  (VT == N0.getValueType() ||
2965  (isTypeLegal(VT) && VT.bitsLE(N0.getValueType()))) &&
2966  N0.getOpcode() == ISD::AND) {
2967  auto &DL = DAG.getDataLayout();
2968  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2969  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2970  !DCI.isBeforeLegalize());
2971  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
2972  // Perform the xform if the AND RHS is a single bit.
2973  if (AndRHS->getAPIntValue().isPowerOf2()) {
2974  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2975  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2976  DAG.getConstant(AndRHS->getAPIntValue().logBase2(), dl,
2977  ShiftTy)));
2978  }
2979  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
2980  // (X & 8) == 8 --> (X & 8) >> 3
2981  // Perform the xform if C1 is a single bit.
2982  if (C1.isPowerOf2()) {
2983  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2984  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2985  DAG.getConstant(C1.logBase2(), dl,
2986  ShiftTy)));
2987  }
2988  }
2989  }
2990  }
2991 
2992  if (C1.getMinSignedBits() <= 64 &&
2994  // (X & -256) == 256 -> (X >> 8) == 1
2995  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2996  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
2997  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2998  const APInt &AndRHSC = AndRHS->getAPIntValue();
2999  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
3000  unsigned ShiftBits = AndRHSC.countTrailingZeros();
3001  auto &DL = DAG.getDataLayout();
3002  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
3003  !DCI.isBeforeLegalize());
3004  EVT CmpTy = N0.getValueType();
3005  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
3006  DAG.getConstant(ShiftBits, dl,
3007  ShiftTy));
3008  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
3009  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
3010  }
3011  }
3012  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
3013  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
3014  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
3015  // X < 0x100000000 -> (X >> 32) < 1
3016  // X >= 0x100000000 -> (X >> 32) >= 1
3017  // X <= 0x0ffffffff -> (X >> 32) < 1
3018  // X > 0x0ffffffff -> (X >> 32) >= 1
3019  unsigned ShiftBits;
3020  APInt NewC = C1;
3021  ISD::CondCode NewCond = Cond;
3022  if (AdjOne) {
3023  ShiftBits = C1.countTrailingOnes();
3024  NewC = NewC + 1;
3025  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
3026  } else {
3027  ShiftBits = C1.countTrailingZeros();
3028  }
3029  NewC.lshrInPlace(ShiftBits);
3030  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
3032  auto &DL = DAG.getDataLayout();
3033  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
3034  !DCI.isBeforeLegalize());
3035  EVT CmpTy = N0.getValueType();
3036  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
3037  DAG.getConstant(ShiftBits, dl, ShiftTy));
3038  SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
3039  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
3040  }
3041  }
3042  }
3043  }
3044 
3045  if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
3046  auto *CFP = cast<ConstantFPSDNode>(N1);
3047  assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
3048 
3049  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
3050  // constant if knowing that the operand is non-nan is enough. We prefer to
3051  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
3052  // materialize 0.0.
3053  if (Cond == ISD::SETO || Cond == ISD::SETUO)
3054  return DAG.getSetCC(dl, VT, N0, N0, Cond);
3055 
3056  // setcc (fneg x), C -> setcc swap(pred) x, -C
3057  if (N0.getOpcode() == ISD::FNEG) {
3059  if (DCI.isBeforeLegalizeOps() ||
3060  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
3061  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
3062  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
3063  }
3064  }
3065 
3066  // If the condition is not legal, see if we can find an equivalent one
3067  // which is legal.
3068  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
3069  // If the comparison was an awkward floating-point == or != and one of
3070  // the comparison operands is infinity or negative infinity, convert the
3071  // condition to a less-awkward <= or >=.
3072  if (CFP->getValueAPF().isInfinity()) {
3073  if (CFP->getValueAPF().isNegative()) {
3074  if (Cond == ISD::SETOEQ &&
3076  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
3077  if (Cond == ISD::SETUEQ &&
3079  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
3080  if (Cond == ISD::SETUNE &&
3082  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
3083  if (Cond == ISD::SETONE &&
3085  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
3086  } else {
3087  if (Cond == ISD::SETOEQ &&
3089  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
3090  if (Cond == ISD::SETUEQ &&
3092  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
3093  if (Cond == ISD::SETUNE &&
3095  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
3096  if (Cond == ISD::SETONE &&
3098  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
3099  }
3100  }
3101  }
3102  }
3103 
3104  if (N0 == N1) {
3105  // The sext(setcc()) => setcc() optimization relies on the appropriate
3106  // constant being emitted.
3107  assert(!N0.getValueType().isInteger() &&
3108  "Integer types should be handled by FoldSetCC");
3109 
3110  bool EqTrue = ISD::isTrueWhenEqual(Cond);
3111  unsigned UOF = ISD::getUnorderedFlavor(Cond);
3112  if (UOF == 2) // FP operators that are undefined on NaNs.
3113  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
3114  if (UOF == unsigned(EqTrue))
3115  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
3116  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
3117  // if it is not already.
3118  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
3119  if (NewCond != Cond &&
3120  (DCI.isBeforeLegalizeOps() ||
3121  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
3122  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
3123  }
3124 
3125  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3126  N0.getValueType().isInteger()) {
3127  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
3128  N0.getOpcode() == ISD::XOR) {
3129  // Simplify (X+Y) == (X+Z) --> Y == Z
3130  if (N0.getOpcode() == N1.getOpcode()) {
3131  if (N0.getOperand(0) == N1.getOperand(0))
3132  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
3133  if (N0.getOperand(1) == N1.getOperand(1))
3134  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
3135  if (isCommutativeBinOp(N0.getOpcode())) {
3136  // If X op Y == Y op X, try other combinations.
3137  if (N0.getOperand(0) == N1.getOperand(1))
3138  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
3139  Cond);
3140  if (N0.getOperand(1) == N1.getOperand(0))
3141  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
3142  Cond);
3143  }
3144  }
3145 
3146  // If RHS is a legal immediate value for a compare instruction, we need
3147  // to be careful about increasing register pressure needlessly.
3148  bool LegalRHSImm = false;
3149 
3150  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
3151  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
3152  // Turn (X+C1) == C2 --> X == C2-C1
3153  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
3154  return DAG.getSetCC(dl, VT, N0.getOperand(0),
3155  DAG.getConstant(RHSC->getAPIntValue()-
3156  LHSR->getAPIntValue(),
3157  dl, N0.getValueType()), Cond);
3158  }
3159 
3160  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
3161  if (N0.getOpcode() == ISD::XOR)
3162  // If we know that all of the inverted bits are zero, don't bother
3163  // performing the inversion.
3164  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
3165  return
3166  DAG.getSetCC(dl, VT, N0.getOperand(0),
3167  DAG.getConstant(LHSR->getAPIntValue() ^
3168  RHSC->getAPIntValue(),
3169  dl, N0.getValueType()),
3170  Cond);
3171  }
3172 
3173  // Turn (C1-X) == C2 --> X == C1-C2
3174  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
3175  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
3176  return
3177  DAG.getSetCC(dl, VT, N0.getOperand(1),
3178  DAG.getConstant(SUBC->getAPIntValue() -
3179  RHSC->getAPIntValue(),
3180  dl, N0.getValueType()),
3181  Cond);
3182  }
3183  }
3184 
3185  // Could RHSC fold directly into a compare?
3186  if (RHSC->getValueType(0).getSizeInBits() <= 64)
3187  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
3188  }
3189 
3190  // (X+Y) == X --> Y == 0 and similar folds.
3191  // Don't do this if X is an immediate that can fold into a cmp
3192  // instruction and X+Y has other uses. It could be an induction variable
3193  // chain, and the transform would increase register pressure.
3194  if (!LegalRHSImm || N0.hasOneUse())
3195  if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
3196  return V;
3197  }
3198 
3199  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
3200  N1.getOpcode() == ISD::XOR)
3201  if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
3202  return V;
3203 
3204  if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
3205  return V;
3206  }
3207 
3208  // Fold away ALL boolean setcc's.
3209  SDValue Temp;
3210  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
3211  EVT OpVT = N0.getValueType();
3212  switch (Cond) {
3213  default: llvm_unreachable("Unknown integer setcc!");
3214  case ISD::SETEQ: // X == Y -> ~(X^Y)
3215  Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
3216  N0 = DAG.getNOT(dl, Temp, OpVT);
3217  if (!DCI.isCalledByLegalizer())
3218  DCI.AddToWorklist(Temp.getNode());
3219  break;
3220  case ISD::SETNE: // X != Y --> (X^Y)
3221  N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
3222  break;
3223  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
3224  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
3225  Temp = DAG.getNOT(dl, N0, OpVT);
3226  N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
3227  if (!DCI.isCalledByLegalizer())
3228  DCI.AddToWorklist(Temp.getNode());
3229  break;
3230  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
3231  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
3232  Temp = DAG.getNOT(dl, N1, OpVT);
3233  N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
3234  if (!DCI.isCalledByLegalizer())
3235  DCI.AddToWorklist(Temp.getNode());
3236  break;
3237  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
3238  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
3239  Temp = DAG.getNOT(dl, N0, OpVT);
3240  N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
3241  if (!DCI.isCalledByLegalizer())
3242  DCI.AddToWorklist(Temp.getNode());
3243  break;
3244  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
3245  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
3246  Temp = DAG.getNOT(dl, N1, OpVT);
3247  N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
3248  break;
3249  }
3250  if (VT.getScalarType() != MVT::i1) {
3251  if (!DCI.isCalledByLegalizer())
3252  DCI.AddToWorklist(N0.getNode());
3253  // FIXME: If running after legalize, we probably can't do this.
3255  N0 = DAG.getNode(ExtendCode, dl, VT, N0);
3256  }
3257  return N0;
3258  }
3259 
3260  // Could not fold it.
3261  return SDValue();
3262 }
3263 
3264 /// Returns true (and the GlobalValue and the offset) if the node is a
3265 /// GlobalAddress + offset.
3267  int64_t &Offset) const {
3268 
3269  SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
3270 
3271  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
3272  GA = GASD->getGlobal();
3273  Offset += GASD->getOffset();
3274  return true;
3275  }
3276 
3277  if (N->getOpcode() == ISD::ADD) {
3278  SDValue N1 = N->getOperand(0);
3279  SDValue N2 = N->getOperand(1);
3280  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
3281  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
3282  Offset += V->getSExtValue();
3283  return true;
3284  }
3285  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
3286  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
3287  Offset += V->getSExtValue();
3288  return true;
3289  }
3290  }
3291  }
3292 
3293  return false;
3294 }
3295 
3297  DAGCombinerInfo &DCI) const {
3298  // Default implementation: no optimization.
3299  return SDValue();
3300 }
3301 
3302 //===----------------------------------------------------------------------===//
3303 // Inline Assembler Implementation Methods
3304 //===----------------------------------------------------------------------===//
3305 
3308  unsigned S = Constraint.size();
3309 
3310  if (S == 1) {
3311  switch (Constraint[0]) {
3312  default: break;
3313  case 'r': return C_RegisterClass;
3314  case 'm': // memory
3315  case 'o': // offsetable
3316  case 'V': // not offsetable
3317  return C_Memory;
3318  case 'i': // Simple Integer or Relocatable Constant
3319  case 'n': // Simple Integer
3320  case 'E': // Floating Point Constant
3321  case 'F': // Floating Point Constant
3322  case 's': // Relocatable Constant
3323  case 'p': // Address.
3324  case 'X': // Allow ANY value.
3325  case 'I': // Target registers.
3326  case 'J':
3327  case 'K':
3328  case 'L':
3329  case 'M':
3330  case 'N':
3331  case 'O':
3332  case 'P':
3333  case '<':
3334  case '>':
3335  return C_Other;
3336  }
3337  }
3338 
3339  if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
3340  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
3341  return C_Memory;
3342  return C_Register;
3343  }
3344  return C_Unknown;
3345 }
3346 
3347 /// Try to replace an X constraint, which matches anything, with another that
3348 /// has more specific requirements based on the type of the corresponding
3349 /// operand.
3350 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
3351  if (ConstraintVT.isInteger())
3352  return "r";
3353  if (ConstraintVT.isFloatingPoint())
3354  return "f"; // works for many targets
3355  return nullptr;
3356 }
3357 
3359  SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
3360  SelectionDAG &DAG) const {
3361  return SDValue();
3362 }
3363 
3364 /// Lower the specified operand into the Ops vector.
3365 /// If it is invalid, don't add anything to Ops.
3367  std::string &Constraint,
3368  std::vector<SDValue> &Ops,
3369  SelectionDAG &DAG) const {
3370 
3371  if (Constraint.length() > 1) return;
3372 
3373  char ConstraintLetter = Constraint[0];
3374  switch (ConstraintLetter) {
3375  default: break;
3376  case 'X': // Allows any operand; labels (basic block) use this.
3377  if (Op.getOpcode() == ISD::BasicBlock ||
3379  Ops.push_back(Op);
3380  return;
3381  }
3383  case 'i': // Simple Integer or Relocatable Constant
3384  case 'n': // Simple Integer
3385  case 's': { // Relocatable Constant
3386  // These operands are interested in values of the form (GV+C), where C may
3387  // be folded in as an offset of GV, or it may be explicitly added. Also, it
3388  // is possible and fine if either GV or C are missing.
3391 
3392  // If we have "(add GV, C)", pull out GV/C
3393  if (Op.getOpcode() == ISD::ADD) {
3394  C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3396  if (!C || !GA) {
3397  C = dyn_cast<ConstantSDNode>(Op.getOperand(0));
3399  }
3400  if (!C || !GA) {
3401  C = nullptr;
3402  GA = nullptr;
3403  }
3404  }
3405 
3406  // If we find a valid operand, map to the TargetXXX version so that the
3407  // value itself doesn't get selected.
3408  if (GA) { // Either &GV or &GV+C
3409  if (ConstraintLetter != 'n') {
3410  int64_t Offs = GA->getOffset();
3411  if (C) Offs += C->getZExtValue();
3412  Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(),
3413  C ? SDLoc(C) : SDLoc(),
3414  Op.getValueType(), Offs));
3415  }
3416  return;
3417  }
3418  if (C) { // just C, no GV.
3419  // Simple constants are not allowed for 's'.
3420  if (ConstraintLetter != 's') {
3421  // gcc prints these as sign extended. Sign extend value to 64 bits
3422  // now; without this it would get ZExt'd later in
3423  // ScheduleDAGSDNodes::EmitNode, which is very generic.
3424  Ops.push_back(DAG.getTargetConstant(C->getSExtValue(),
3425  SDLoc(C), MVT::i64));
3426  }
3427  return;
3428  }
3429  break;
3430  }
3431  }
3432 }
3433 
3434 std::pair<unsigned, const TargetRegisterClass *>
3436  StringRef Constraint,
3437  MVT VT) const {
3438  if (Constraint.empty() || Constraint[0] != '{')
3439  return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
3440  assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
3441 
3442  // Remove the braces from around the name.
3443  StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
3444 
3445  std::pair<unsigned, const TargetRegisterClass *> R =
3446  std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
3447 
3448  // Figure out which register class contains this reg.
3449  for (const TargetRegisterClass *RC : RI->regclasses()) {
3450  // If none of the value types for this register class are valid, we
3451  // can't use it. For example, 64-bit reg classes on 32-bit targets.
3452  if (!isLegalRC(*RI, *RC))
3453  continue;
3454 
3455  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
3456  I != E; ++I) {
3457  if (RegName.equals_lower(RI->getRegAsmName(*I))) {
3458  std::pair<unsigned, const TargetRegisterClass *> S =
3459  std::make_pair(*I, RC);
3460 
3461  // If this register class has the requested value type, return it,
3462  // otherwise keep searching and return the first class found
3463  // if no other is found which explicitly has the requested type.
3464  if (RI->isTypeLegalForClass(*RC, VT))
3465  return S;
3466  if (!R.second)
3467  R = S;
3468  }
3469  }
3470  }
3471 
3472  return R;
3473 }
3474 
3475 //===----------------------------------------------------------------------===//
3476 // Constraint Selection.
3477 
3478 /// Return true of this is an input operand that is a matching constraint like
3479 /// "4".
3481  assert(!ConstraintCode.empty() && "No known constraint!");
3482  return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
3483 }
3484 
3485 /// If this is an input matching constraint, this method returns the output
3486 /// operand it matches.
3488  assert(!ConstraintCode.empty() && "No known constraint!");
3489  return atoi(ConstraintCode.c_str());
3490 }
3491 
3492 /// Split up the constraint string from the inline assembly value into the
3493 /// specific constraints and their prefixes, and also tie in the associated
3494 /// operand values.
3495 /// If this returns an empty vector, and if the constraint string itself
3496 /// isn't empty, there was an error parsing.
3499  const TargetRegisterInfo *TRI,
3500  ImmutableCallSite CS) const {
3501  /// Information about all of the constraints.
3502  AsmOperandInfoVector ConstraintOperands;
3503  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
3504  unsigned maCount = 0; // Largest number of multiple alternative constraints.
3505 
3506  // Do a prepass over the constraints, canonicalizing them, and building up the
3507  // ConstraintOperands list.
3508  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
3509  unsigned ResNo = 0; // ResNo - The result number of the next output.
3510 
3511  for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
3512  ConstraintOperands.emplace_back(std::move(CI));
3513  AsmOperandInfo &OpInfo = ConstraintOperands.back();
3514 
3515  // Update multiple alternative constraint count.
3516  if (OpInfo.multipleAlternatives.size() > maCount)
3517  maCount = OpInfo.multipleAlternatives.size();
3518 
3519  OpInfo.ConstraintVT = MVT::Other;
3520 
3521  // Compute the value type for each operand.
3522  switch (OpInfo.Type) {
3523  case InlineAsm::isOutput:
3524  // Indirect outputs just consume an argument.
3525  if (OpInfo.isIndirect) {
3526  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
3527  break;
3528  }
3529 
3530  // The return value of the call is this value. As such, there is no
3531  // corresponding argument.
3532  assert(!CS.getType()->isVoidTy() &&
3533  "Bad inline asm!");
3534  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
3535  OpInfo.ConstraintVT =
3536  getSimpleValueType(DL, STy->getElementType(ResNo));
3537  } else {
3538  assert(ResNo == 0 && "Asm only has one result!");
3539  OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
3540  }
3541  ++ResNo;
3542  break;
3543  case InlineAsm::isInput:
3544  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
3545  break;
3546  case InlineAsm::isClobber:
3547  // Nothing to do.
3548  break;
3549  }
3550 
3551  if (OpInfo.CallOperandVal) {
3552  llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
3553  if (OpInfo.isIndirect) {
3554  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
3555  if (!PtrTy)
3556  report_fatal_error("Indirect operand for inline asm not a pointer!");
3557  OpTy = PtrTy->getElementType();
3558  }
3559 
3560  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
3561  if (StructType *STy = dyn_cast<StructType>(OpTy))
3562  if (STy->getNumElements() == 1)
3563  OpTy = STy->getElementType(0);
3564 
3565  // If OpTy is not a single value, it may be a struct/union that we
3566  // can tile with integers.
3567  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
3568  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
3569  switch (BitSize) {
3570  default: break;
3571  case 1:
3572  case 8:
3573  case 16:
3574  case 32:
3575  case 64:
3576  case 128:
3577  OpInfo.ConstraintVT =
3578  MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
3579  break;
3580  }
3581  } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
3582  unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
3583  OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
3584  } else {
3585  OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
3586  }
3587  }
3588  }
3589 
3590  // If we have multiple alternative constraints, select the best alternative.
3591  if (!ConstraintOperands.empty()) {
3592  if (maCount) {
3593  unsigned bestMAIndex = 0;
3594  int bestWeight = -1;
3595  // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
3596  int weight = -1;
3597  unsigned maIndex;
3598  // Compute the sums of the weights for each alternative, keeping track
3599  // of the best (highest weight) one so far.
3600  for (maIndex = 0; maIndex < maCount; ++maIndex) {
3601  int weightSum = 0;
3602  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3603  cIndex != eIndex; ++cIndex) {
3604  AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
3605  if (OpInfo.Type == InlineAsm::isClobber)
3606  continue;
3607 
3608  // If this is an output operand with a matching input operand,
3609  // look up the matching input. If their types mismatch, e.g. one
3610  // is an integer, the other is floating point, or their sizes are
3611  // different, flag it as an maCantMatch.
3612  if (OpInfo.hasMatchingInput()) {
3613  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3614  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3615  if ((OpInfo.ConstraintVT.isInteger() !=
3616  Input.ConstraintVT.isInteger()) ||
3617  (OpInfo.ConstraintVT.getSizeInBits() !=
3618  Input.ConstraintVT.getSizeInBits())) {
3619  weightSum = -1; // Can't match.
3620  break;
3621  }
3622  }
3623  }
3624  weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
3625  if (weight == -1) {
3626  weightSum = -1;
3627  break;
3628  }
3629  weightSum += weight;
3630  }
3631  // Update best.
3632  if (weightSum > bestWeight) {
3633  bestWeight = weightSum;
3634  bestMAIndex = maIndex;
3635  }
3636  }
3637 
3638  // Now select chosen alternative in each constraint.
3639  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3640  cIndex != eIndex; ++cIndex) {
3641  AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
3642  if (cInfo.Type == InlineAsm::isClobber)
3643  continue;
3644  cInfo.selectAlternative(bestMAIndex);
3645  }
3646  }
3647  }
3648 
3649  // Check and hook up tied operands, choose constraint code to use.
3650  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3651  cIndex != eIndex; ++cIndex) {
3652  AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
3653 
3654  // If this is an output operand with a matching input operand, look up the
3655  // matching input. If their types mismatch, e.g. one is an integer, the
3656  // other is floating point, or their sizes are different, flag it as an
3657  // error.
3658  if (OpInfo.hasMatchingInput()) {
3659  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3660 
3661  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3662  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
3664  OpInfo.ConstraintVT);
3665  std::pair<unsigned, const TargetRegisterClass *> InputRC =
3667  Input.ConstraintVT);
3668  if ((OpInfo.ConstraintVT.isInteger() !=
3669  Input.ConstraintVT.isInteger()) ||
3670  (MatchRC.second != InputRC.second)) {
3671  report_fatal_error("Unsupported asm: input constraint"
3672  " with a matching output constraint of"
3673  " incompatible type!");
3674  }
3675  }
3676  }
3677  }
3678 
3679  return ConstraintOperands;
3680 }
3681 
3682 /// Return an integer indicating how general CT is.
3684  switch (CT) {
3687  return 0;
3689  return 1;
3691  return 2;
3693  return 3;
3694  }
3695  llvm_unreachable("Invalid constraint type");
3696 }
3697 
3698 /// Examine constraint type and operand type and determine a weight value.
3699 /// This object must already have been set up with the operand type
3700 /// and the current alternative constraint selected.
3703  AsmOperandInfo &info, int maIndex) const {
3705  if (maIndex >= (int)info.multipleAlternatives.size())
3706  rCodes = &info.Codes;
3707  else
3708  rCodes = &info.multipleAlternatives[maIndex].Codes;
3709  ConstraintWeight BestWeight = CW_Invalid;
3710 
3711  // Loop over the options, keeping track of the most general one.
3712  for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
3713  ConstraintWeight weight =
3714  getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
3715  if (weight > BestWeight)
3716  BestWeight = weight;
3717  }
3718 
3719  return BestWeight;
3720 }
3721 
3722 /// Examine constraint type and operand type and determine a weight value.
3723 /// This object must already have been set up with the operand type
3724 /// and the current alternative constraint selected.
3727  AsmOperandInfo &info, const char *constraint) const {
3728  ConstraintWeight weight = CW_Invalid;
3729  Value *CallOperandVal = info.CallOperandVal;
3730  // If we don't have a value, we can't do a match,
3731  // but allow it at the lowest weight.
3732  if (!CallOperandVal)
3733  return CW_Default;
3734  // Look at the constraint type.
3735  switch (*constraint) {
3736  case 'i': // immediate integer.
3737  case 'n': // immediate integer with a known value.
3738  if (isa<ConstantInt>(CallOperandVal))
3739  weight = CW_Constant;
3740  break;
3741  case 's': // non-explicit intregal immediate.
3742  if (isa<GlobalValue>(CallOperandVal))
3743  weight = CW_Constant;
3744  break;
3745  case 'E': // immediate float if host format.
3746  case 'F': // immediate float.
3747  if (isa<ConstantFP>(CallOperandVal))
3748  weight = CW_Constant;
3749  break;
3750  case '<': // memory operand with autodecrement.
3751  case '>': // memory operand with autoincrement.
3752  case 'm': // memory operand.
3753  case 'o': // offsettable memory operand
3754  case 'V': // non-offsettable memory operand
3755  weight = CW_Memory;
3756  break;
3757  case 'r': // general register.
3758  case 'g': // general register, memory operand or immediate integer.
3759  // note: Clang converts "g" to "imr".
3760  if (CallOperandVal->getType()->isIntegerTy())
3761  weight = CW_Register;
3762  break;
3763  case 'X': // any operand.
3764  default:
3765  weight = CW_Default;
3766  break;
3767  }
3768  return weight;
3769 }
3770 
3771 /// If there are multiple different constraints that we could pick for this
3772 /// operand (e.g. "imr") try to pick the 'best' one.
3773 /// This is somewhat tricky: constraints fall into four classes:
3774 /// Other -> immediates and magic values
3775 /// Register -> one specific register
3776 /// RegisterClass -> a group of regs
3777 /// Memory -> memory
3778 /// Ideally, we would pick the most specific constraint possible: if we have
3779 /// something that fits into a register, we would pick it. The problem here
3780 /// is that if we have something that could either be in a register or in
3781 /// memory that use of the register could cause selection of *other*
3782 /// operands to fail: they might only succeed if we pick memory. Because of
3783 /// this the heuristic we use is:
3784 ///
3785 /// 1) If there is an 'other' constraint, and if the operand is valid for
3786 /// that constraint, use it. This makes us take advantage of 'i'
3787 /// constraints when available.
3788 /// 2) Otherwise, pick the most general constraint present. This prefers
3789 /// 'm' over 'r', for example.
3790 ///
3792  const TargetLowering &TLI,
3793  SDValue Op, SelectionDAG *DAG) {
3794  assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
3795  unsigned BestIdx = 0;
3797  int BestGenerality = -1;
3798 
3799  // Loop over the options, keeping track of the most general one.
3800  for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
3802  TLI.getConstraintType(OpInfo.Codes[i]);
3803 
3804  // If this is an 'other' constraint, see if the operand is valid for it.
3805  // For example, on X86 we might have an 'rI' constraint. If the operand
3806  // is an integer in the range [0..31] we want to use I (saving a load
3807  // of a register), otherwise we must use 'r'.
3808  if (CType == TargetLowering::C_Other && Op.getNode()) {
3809  assert(OpInfo.Codes[i].size() == 1 &&
3810  "Unhandled multi-letter 'other' constraint");
3811  std::vector<SDValue> ResultOps;
3812  TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
3813  ResultOps, *DAG);
3814  if (!ResultOps.empty()) {
3815  BestType = CType;
3816  BestIdx = i;
3817  break;
3818  }
3819  }
3820 
3821  // Things with matching constraints can only be registers, per gcc
3822  // documentation. This mainly affects "g" constraints.
3823  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
3824  continue;
3825 
3826  // This constraint letter is more general than the previous one, use it.
3827  int Generality = getConstraintGenerality(CType);
3828  if (Generality > BestGenerality) {
3829  BestType = CType;
3830  BestIdx = i;
3831  BestGenerality = Generality;
3832  }
3833  }
3834 
3835  OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
3836  OpInfo.ConstraintType = BestType;
3837 }
3838 
3839 /// Determines the constraint code and constraint type to use for the specific
3840 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
3842  SDValue Op,
3843  SelectionDAG *DAG) const {
3844  assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
3845 
3846  // Single-letter constraints ('r') are very common.
3847  if (OpInfo.Codes.size() == 1) {
3848  OpInfo.ConstraintCode = OpInfo.Codes[0];
3850  } else {
3851  ChooseConstraint(OpInfo, *this, Op, DAG);
3852  }
3853 
3854  // 'X' matches anything.
3855  if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
3856  // Labels and constants are handled elsewhere ('X' is the only thing
3857  // that matches labels). For Functions, the type here is the type of
3858  // the result, which is not what we want to look at; leave them alone.
3859  Value *v = OpInfo.CallOperandVal;
3860  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
3861  OpInfo.CallOperandVal = v;
3862  return;
3863  }
3864 
3865  if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
3866  return;
3867 
3868  // Otherwise, try to resolve it to something we know about by looking at
3869  // the actual operand type.
3870  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
3871  OpInfo.ConstraintCode = Repl;
3873  }
3874  }
3875 }
3876 
3877 /// Given an exact SDIV by a constant, create a multiplication
3878 /// with the multiplicative inverse of the constant.
3880  const SDLoc &dl, SelectionDAG &DAG,
3881  SmallVectorImpl<SDNode *> &Created) {
3882  SDValue Op0 = N->getOperand(0);
3883  SDValue Op1 = N->getOperand(1);
3884  EVT VT = N->getValueType(0);
3885  EVT SVT = VT.getScalarType();
3886  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3887  EVT ShSVT = ShVT.getScalarType();
3888 
3889  bool UseSRA = false;
3890  SmallVector<SDValue, 16> Shifts, Factors;
3891 
3892  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
3893  if (C->isNullValue())
3894  return false;
3895  APInt Divisor = C->getAPIntValue();
3896  unsigned Shift = Divisor.countTrailingZeros();
3897  if (Shift) {
3898  Divisor.ashrInPlace(Shift);
3899  UseSRA = true;
3900  }
3901  // Calculate the multiplicative inverse, using Newton's method.
3902  APInt t;
3903  APInt Factor = Divisor;
3904  while ((t = Divisor * Factor) != 1)
3905  Factor *= APInt(Divisor.getBitWidth(), 2) - t;
3906  Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
3907  Factors.push_back(DAG.getConstant(Factor, dl, SVT));
3908  return true;
3909  };
3910 
3911  // Collect all magic values from the build vector.
3912  if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
3913  return SDValue();
3914 
3915  SDValue Shift, Factor;
3916  if (VT.isVector()) {
3917  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
3918  Factor = DAG.getBuildVector(VT, dl, Factors);
3919  } else {
3920  Shift = Shifts[0];
3921  Factor = Factors[0];
3922  }
3923 
3924  SDValue Res = Op0;
3925 
3926  // Shift the value upfront if it is even, so the LSB is one.
3927  if (UseSRA) {
3928  // TODO: For UDIV use SRL instead of SRA.
3929  SDNodeFlags Flags;
3930  Flags.setExact(true);
3931  Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
3932  Created.push_back(Res.getNode());
3933  }
3934 
3935  return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
3936 }
3937 
3939  SelectionDAG &DAG,
3940  SmallVectorImpl<SDNode *> &Created) const {
3942  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3943  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
3944  return SDValue(N, 0); // Lower SDIV as SDIV
3945  return SDValue();
3946 }
3947 
3948 /// Given an ISD::SDIV node expressing a divide by constant,
3949 /// return a DAG expression to select that will generate the same value by
3950 /// multiplying by a magic number.
3951 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3953  bool IsAfterLegalization,
3954  SmallVectorImpl<SDNode *> &Created) const {
3955  SDLoc dl(N);
3956  EVT VT = N->getValueType(0);
3957  EVT SVT = VT.getScalarType();
3958  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
3959  EVT ShSVT = ShVT.getScalarType();
3960  unsigned EltBits = VT.getScalarSizeInBits();
3961 
3962  // Check to see if we can do this.
3963  // FIXME: We should be more aggressive here.
3964  if (!isTypeLegal(VT))
3965  return SDValue();
3966 
3967  // If the sdiv has an 'exact' bit we can use a simpler lowering.
3968  if (N->getFlags().hasExact())
3969  return BuildExactSDIV(*this, N, dl, DAG, Created);
3970 
3971  SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
3972 
3973  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
3974  if (C->isNullValue())
3975  return false;
3976 
3977  const APInt &Divisor = C->getAPIntValue();
3978  APInt::ms magics = Divisor.magic();
3979  int NumeratorFactor = 0;
3980  int ShiftMask = -1;
3981 
3982  if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
3983  // If d is +1/-1, we just multiply the numerator by +1/-1.
3984  NumeratorFactor = Divisor.getSExtValue();
3985  magics.m = 0;
3986  magics.s = 0;
3987  ShiftMask = 0;
3988  } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
3989  // If d > 0 and m < 0, add the numerator.
3990  NumeratorFactor = 1;
3991  } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
3992  // If d < 0 and m > 0, subtract the numerator.
3993  NumeratorFactor = -1;
3994  }
3995 
3996  MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
3997  Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
3998  Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
3999  ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
4000  return true;
4001  };
4002 
4003  SDValue N0 = N->getOperand(0);
4004  SDValue N1 = N->getOperand(1);
4005 
4006  // Collect the shifts / magic values from each element.
4007  if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
4008  return SDValue();
4009 
4010  SDValue MagicFactor, Factor, Shift, ShiftMask;
4011  if (VT.isVector()) {
4012  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
4013  Factor = DAG.getBuildVector(VT, dl, Factors);
4014  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
4015  ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
4016  } else {
4017  MagicFactor = MagicFactors[0];
4018  Factor = Factors[0];
4019  Shift = Shifts[0];
4020  ShiftMask = ShiftMasks[0];
4021  }
4022 
4023  // Multiply the numerator (operand 0) by the magic value.
4024  // FIXME: We should support doing a MUL in a wider type.
4025  SDValue Q;
4026  if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
4028  Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
4029  else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
4031  SDValue LoHi =
4032  DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
4033  Q = SDValue(LoHi.getNode(), 1);
4034  } else
4035  return SDValue(); // No mulhs or equivalent.
4036  Created.push_back(Q.getNode());
4037 
4038  // (Optionally) Add/subtract the numerator using Factor.
4039  Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
4040  Created.push_back(Factor.getNode());
4041  Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
4042  Created.push_back(Q.getNode());
4043 
4044  // Shift right algebraic by shift value.
4045  Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
4046  Created.push_back(Q.getNode());
4047 
4048  // Extract the sign bit, mask it and add it to the quotient.
4049  SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
4050  SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
4051  Created.push_back(T.getNode());
4052  T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
4053  Created.push_back(T.getNode());
4054  return DAG.getNode(ISD::ADD, dl, VT, Q, T);
4055 }
4056 
4057 /// Given an ISD::UDIV node expressing a divide by constant,
4058 /// return a DAG expression to select that will generate the same value by
4059 /// multiplying by a magic number.
4060 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4062  bool IsAfterLegalization,
4063  SmallVectorImpl<SDNode *> &Created) const {
4064  SDLoc dl(N);
4065  EVT VT = N->getValueType(0);
4066  EVT SVT = VT.getScalarType();
4067  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4068  EVT ShSVT = ShVT.getScalarType();
4069  unsigned EltBits = VT.getScalarSizeInBits();
4070 
4071  // Check to see if we can do this.
4072  // FIXME: We should be more aggressive here.
4073  if (!isTypeLegal(VT))
4074  return SDValue();
4075 
4076  bool UseNPQ = false;
4077  SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
4078 
4079  auto BuildUDIVPattern = [&](ConstantSDNode *C) {
4080  if (C->isNullValue())
4081  return false;
4082  // FIXME: We should use a narrower constant when the upper
4083  // bits are known to be zero.
4084  APInt Divisor = C->getAPIntValue();
4085  APInt::mu magics = Divisor.magicu();
4086  unsigned PreShift = 0, PostShift = 0;
4087 
4088  // If the divisor is even, we can avoid using the expensive fixup by
4089  // shifting the divided value upfront.
4090  if (magics.a != 0 && !Divisor[0]) {
4091  PreShift = Divisor.countTrailingZeros();
4092  // Get magic number for the shifted divisor.
4093  magics = Divisor.lshr(PreShift).magicu(PreShift);
4094  assert(magics.a == 0 && "Should use cheap fixup now");
4095  }
4096 
4097  APInt Magic = magics.m;
4098 
4099  unsigned SelNPQ;
4100  if (magics.a == 0 || Divisor.isOneValue()) {
4101  assert(magics.s < Divisor.getBitWidth() &&
4102  "We shouldn't generate an undefined shift!");
4103  PostShift = magics.s;
4104  SelNPQ = false;
4105  } else {
4106  PostShift = magics.s - 1;
4107  SelNPQ = true;
4108  }
4109 
4110  PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
4111  MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
4112  NPQFactors.push_back(
4113  DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
4114  : APInt::getNullValue(EltBits),
4115  dl, SVT));
4116  PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
4117  UseNPQ |= SelNPQ;
4118  return true;
4119  };
4120 
4121  SDValue N0 = N->getOperand(0);
4122  SDValue N1 = N->getOperand(1);
4123 
4124  // Collect the shifts/magic values from each element.
4125  if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
4126  return SDValue();
4127 
4128  SDValue PreShift, PostShift, MagicFactor, NPQFactor;
4129  if (VT.isVector()) {
4130  PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
4131  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
4132  NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
4133  PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
4134  } else {
4135  PreShift = PreShifts[0];
4136  MagicFactor = MagicFactors[0];
4137  PostShift = PostShifts[0];
4138  }
4139 
4140  SDValue Q = N0;
4141  Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
4142  Created.push_back(Q.getNode());
4143 
4144  // FIXME: We should support doing a MUL in a wider type.
4145  auto GetMULHU = [&](SDValue X, SDValue Y) {
4146  if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
4148  return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
4149  if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
4151  SDValue LoHi =
4152  DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
4153  return SDValue(LoHi.getNode(), 1);
4154  }
4155  return SDValue(); // No mulhu or equivalent
4156  };
4157 
4158  // Multiply the numerator (operand 0) by the magic value.
4159  Q = GetMULHU(Q, MagicFactor);
4160  if (!Q)
4161  return SDValue();
4162 
4163  Created.push_back(Q.getNode());
4164 
4165  if (UseNPQ) {
4166  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
4167  Created.push_back(NPQ.getNode());
4168 
4169  // For vectors we might have a mix of non-NPQ/NPQ paths, so use
4170  // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
4171  if (VT.isVector())
4172  NPQ = GetMULHU(NPQ, NPQFactor);
4173  else
4174  NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
4175 
4176  Created.push_back(NPQ.getNode());
4177 
4178  Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
4179  Created.push_back(Q.getNode());
4180  }
4181 
4182  Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
4183  Created.push_back(Q.getNode());
4184 
4185  SDValue One = DAG.getConstant(1, dl, VT);
4186  SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
4187  return DAG.getSelect(dl, VT, IsOne, N0, Q);
4188 }
4189 
4190 bool TargetLowering::
4192  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
4193  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
4194  "be a constant integer");
4195  return true;
4196  }
4197 
4198  return false;
4199 }
4200 
4201 //===----------------------------------------------------------------------===//
4202 // Legalization Utilities
4203 //===----------------------------------------------------------------------===//
4204 
4205 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
4206  SDValue LHS, SDValue RHS,
4207  SmallVectorImpl<SDValue> &Result,
4208  EVT HiLoVT, SelectionDAG &DAG,
4210  SDValue LH, SDValue RL, SDValue RH) const {
4211  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
4212  Opcode == ISD::SMUL_LOHI);
4213 
4214  bool HasMULHS = (Kind == MulExpansionKind::Always) ||
4216  bool HasMULHU = (Kind == MulExpansionKind::Always) ||
4218  bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
4220  bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
4222 
4223  if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
4224  return false;
4225 
4226  unsigned OuterBitSize = VT.getScalarSizeInBits();
4227  unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
4228  unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
4229  unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
4230 
4231  // LL, LH, RL, and RH must be either all NULL or all set to a value.
4232  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
4233  (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
4234 
4235  SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
4236  auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
4237  bool Signed) -> bool {
4238  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
4239  Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
4240  Hi = SDValue(Lo.getNode(), 1);
4241  return true;
4242  }
4243  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
4244  Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
4245  Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
4246  return true;
4247  }
4248  return false;
4249  };
4250 
4251  SDValue Lo, Hi;
4252 
4253  if (!LL.getNode() && !RL.getNode() &&
4255  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
4256  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
4257  }
4258 
4259  if (!LL.getNode())
4260  return false;
4261 
4262  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
4263  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
4264  DAG.MaskedValueIsZero(RHS, HighMask)) {
4265  // The inputs are both zero-extended.
4266  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
4267  Result.push_back(Lo);
4268  Result.push_back(Hi);
4269  if (Opcode != ISD::MUL) {
4270  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
4271  Result.push_back(Zero);
4272  Result.push_back(Zero);
4273  }
4274  return true;
4275  }
4276  }
4277 
4278  if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
4279  RHSSB > InnerBitSize) {
4280  // The input values are both sign-extended.
4281  // TODO non-MUL case?
4282  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
4283  Result.push_back(Lo);
4284  Result.push_back(Hi);
4285  return true;
4286  }
4287  }
4288 
4289  unsigned ShiftAmount = OuterBitSize - InnerBitSize;
4290  EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
4291  if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
4292  // FIXME getShiftAmountTy does not always return a sensible result when VT
4293  // is an illegal type, and so the type may be too small to fit the shift
4294  // amount. Override it with i32. The shift will have to be legalized.
4295  ShiftAmountTy = MVT::i32;
4296  }
4297  SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
4298 
4299  if (!LH.getNode() && !RH.getNode() &&
4302  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
4303  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
4304  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
4305  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
4306  }
4307 
4308  if (!LH.getNode())
4309  return false;
4310 
4311  if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
4312  return false;
4313 
4314  Result.push_back(Lo);
4315 
4316  if (Opcode == ISD::MUL) {
4317  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
4318  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
4319  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
4320  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
4321  Result.push_back(Hi);
4322  return true;
4323  }
4324 
4325  // Compute the full width result.
4326  auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
4327  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
4328  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
4329  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
4330  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
4331  };
4332 
4333  SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
4334  if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
4335  return false;
4336 
4337  // This is effectively the add part of a multiply-add of half-sized operands,
4338  // so it cannot overflow.
4339  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
4340 
4341  if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
4342  return false;
4343 
4344  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
4345  EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
4346 
4347  bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
4349  if (UseGlue)
4350  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
4351  Merge(Lo, Hi));
4352  else
4353  Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
4354  Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
4355 
4356  SDValue Carry = Next.getValue(1);
4357  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
4358  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
4359 
4360  if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
4361  return false;
4362 
4363  if (UseGlue)
4364  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
4365  Carry);
4366  else
4367  Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
4368  Zero, Carry);
4369 
4370  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
4371 
4372  if (Opcode == ISD::SMUL_LOHI) {
4373  SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
4374  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
4375  Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
4376 
4377  NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
4378  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
4379  Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
4380  }
4381 
4382  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
4383  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
4384  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
4385  return true;
4386 }
4387 
4390  SDValue LL, SDValue LH, SDValue RL,
4391  SDValue RH) const {
4392  SmallVector<SDValue, 2> Result;
4393  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
4394  N->getOperand(0), N->getOperand(1), Result, HiLoVT,
4395  DAG, Kind, LL, LH, RL, RH);
4396  if (Ok) {
4397  assert(Result.size() == 2);
4398  Lo = Result[0];
4399  Hi = Result[1];
4400  }
4401  return Ok;
4402 }
4403 
4405  SelectionDAG &DAG) const {
4406  EVT VT = Node->getValueType(0);
4407 
4408  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
4412  return false;
4413 
4414  // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
4415  // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
4416  SDValue X = Node->getOperand(0);
4417  SDValue Y = Node->getOperand(1);
4418  SDValue Z = Node->getOperand(2);
4419 
4420  unsigned EltSizeInBits = VT.getScalarSizeInBits();
4421  bool IsFSHL = Node->getOpcode() == ISD::FSHL;
4422  SDLoc DL(SDValue(Node, 0));
4423 
4424  EVT ShVT = Z.getValueType();
4425  SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
4426  SDValue Zero = DAG.getConstant(0, DL, ShVT);
4427 
4428  SDValue ShAmt;
4429  if (isPowerOf2_32(EltSizeInBits)) {
4430  SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
4431  ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
4432  } else {
4433  ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
4434  }
4435 
4436  SDValue InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
4437  SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
4438  SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
4439  SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
4440 
4441  // If (Z % BW == 0), then the opposite direction shift is shift-by-bitwidth,
4442  // and that is undefined. We must compare and select to avoid UB.
4443  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShVT);
4444 
4445  // For fshl, 0-shift returns the 1st arg (X).
4446  // For fshr, 0-shift returns the 2nd arg (Y).
4447  SDValue IsZeroShift = DAG.getSetCC(DL, CCVT, ShAmt, Zero, ISD::SETEQ);
4448  Result = DAG.getSelect(DL, VT, IsZeroShift, IsFSHL ? X : Y, Or);
4449  return true;
4450 }
4451 
4452 // TODO: Merge with expandFunnelShift.
4454  SelectionDAG &DAG) const {
4455  EVT VT = Node->getValueType(0);
4456  unsigned EltSizeInBits = VT.getScalarSizeInBits();
4457  bool IsLeft = Node->getOpcode() == ISD::ROTL;
4458  SDValue Op0 = Node->getOperand(0);
4459  SDValue Op1 = Node->getOperand(1);
4460  SDLoc DL(SDValue(Node, 0));
4461 
4462  EVT ShVT = Op1.getValueType();
4463  SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
4464 
4465  // If a rotate in the other direction is legal, use it.
4466  unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
4467  if (isOperationLegal(RevRot, VT)) {
4468  SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
4469  Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
4470  return true;
4471  }
4472 
4473  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
4478  return false;
4479 
4480  // Otherwise,
4481  // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
4482  // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
4483  //
4484  assert(isPowerOf2_32(EltSizeInBits) && EltSizeInBits > 1 &&
4485  "Expecting the type bitwidth to be a power of 2");
4486  unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
4487  unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
4488  SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
4489  SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
4490  SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
4491  SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
4492  Result = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ShOpc, DL, VT, Op0, And0),
4493  DAG.getNode(HsOpc, DL, VT, Op0, And1));
4494  return true;
4495 }
4496 
4498  SelectionDAG &DAG) const {
4499  SDValue Src = Node->getOperand(0);
4500  EVT SrcVT = Src.getValueType();
4501  EVT DstVT = Node->getValueType(0);
4502  SDLoc dl(SDValue(Node, 0));
4503 
4504  // FIXME: Only f32 to i64 conversions are supported.
4505  if (SrcVT != MVT::f32 || DstVT != MVT::i64)
4506  return false;
4507 
4508  // Expand f32 -> i64 conversion
4509  // This algorithm comes from compiler-rt's implementation of fixsfdi:
4510  // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c
4511  unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
4512  EVT IntVT = SrcVT.changeTypeToInteger();
4513  EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
4514 
4515  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
4516  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
4517  SDValue Bias = DAG.getConstant(127, dl, IntVT);
4518  SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
4519  SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
4520  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
4521 
4522  SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
4523 
4524  SDValue ExponentBits = DAG.getNode(
4525  ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
4526  DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
4527  SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
4528 
4529  SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
4530  DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
4531  DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
4532  Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
4533 
4534  SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
4535  DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
4536  DAG.getConstant(0x00800000, dl, IntVT));
4537 
4538  R = DAG.getZExtOrTrunc(R, dl, DstVT);
4539 
4540  R = DAG.getSelectCC(
4541  dl, Exponent, ExponentLoBit,
4542  DAG.getNode(ISD::SHL, dl, DstVT, R,
4543  DAG.getZExtOrTrunc(
4544  DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
4545  dl, IntShVT)),
4546  DAG.getNode(ISD::SRL, dl, DstVT, R,
4547  DAG.getZExtOrTrunc(
4548  DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
4549  dl, IntShVT)),
4550  ISD::SETGT);
4551 
4552  SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
4553  DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
4554 
4555  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
4556  DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
4557  return true;
4558 }
4559 
4561  SelectionDAG &DAG) const {
4562  SDLoc dl(SDValue(Node, 0));
4563  SDValue Src = Node->getOperand(0);
4564 
4565  EVT SrcVT = Src.getValueType();
4566  EVT DstVT = Node->getValueType(0);
4567  EVT SetCCVT =
4568  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
4569 
4570  // Only expand vector types if we have the appropriate vector bit operations.
4571  if (DstVT.isVector() && (!isOperationLegalOrCustom(ISD::FP_TO_SINT, DstVT) ||
4573  return false;
4574 
4575  // If the maximum float value is smaller then the signed integer range,
4576  // the destination signmask can't be represented by the float, so we can
4577  // just use FP_TO_SINT directly.
4578  const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
4579  APFloat APF(APFSem, APInt::getNullValue(SrcVT.getScalarSizeInBits()));
4580  APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
4581  if (APFloat::opOverflow &
4582  APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
4583  Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
4584  return true;
4585  }
4586 
4587  SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
4588  SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
4589 
4590  bool Strict = shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
4591  if (Strict) {
4592  // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
4593  // signmask then offset (the result of which should be fully representable).
4594  // Sel = Src < 0x8000000000000000
4595  // Val = select Sel, Src, Src - 0x8000000000000000
4596  // Ofs = select Sel, 0, 0x8000000000000000
4597  // Result = fp_to_sint(Val) ^ Ofs
4598 
4599  // TODO: Should any fast-math-flags be set for the FSUB?
4600  SDValue Val = DAG.getSelect(dl, SrcVT, Sel, Src,
4601  DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
4602  SDValue Ofs = DAG.getSelect(dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT),
4603  DAG.getConstant(SignMask, dl, DstVT));
4604  Result = DAG.getNode(ISD::XOR, dl, DstVT,
4605  DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val), Ofs);
4606  } else {
4607  // Expand based on maximum range of FP_TO_SINT:
4608  // True = fp_to_sint(Src)
4609  // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
4610  // Result = select (Src < 0x8000000000000000), True, False
4611 
4612  SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
4613  // TODO: Should any fast-math-flags be set for the FSUB?
4614  SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
4615  DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
4616  False = DAG.getNode(ISD::XOR, dl, DstVT, False,
4617  DAG.getConstant(SignMask, dl, DstVT));
4618  Result = DAG.getSelect(dl, DstVT, Sel, True, False);
4619  }
4620  return true;
4621 }
4622 
4624  SelectionDAG &DAG) const {
4625  SDValue Src = Node->getOperand(0);
4626  EVT SrcVT = Src.getValueType();
4627  EVT DstVT = Node->getValueType(0);
4628 
4629  if (SrcVT.getScalarType() != MVT::i64)
4630  return false;
4631 
4632  SDLoc dl(SDValue(Node, 0));
4633  EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());
4634 
4635  if (DstVT.getScalarType() == MVT::f32) {
4636  // Only expand vector types if we have the appropriate vector bit
4637  // operations.
4638  if (SrcVT.isVector() &&
4639  (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
4644  return false;
4645 
4646  // For unsigned conversions, convert them to signed conversions using the
4647  // algorithm from the x86_64 __floatundidf in compiler_rt.
4648  SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
4649 
4650  SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
4651  SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Src, ShiftConst);
4652  SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
4653  SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Src, AndConst);
4654  SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
4655 
4656  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Or);
4657  SDValue Slow = DAG.getNode(ISD::FADD, dl, DstVT, SignCvt, SignCvt);
4658 
4659  // TODO: This really should be implemented using a branch rather than a
4660  // select. We happen to get lucky and machinesink does the right
4661  // thing most of the time. This would be a good candidate for a
4662  // pseudo-op, or, even better, for whole-function isel.
4663  EVT SetCCVT =
4664  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
4665 
4666  SDValue SignBitTest = DAG.getSetCC(
4667  dl, SetCCVT, Src, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
4668  Result = DAG.getSelect(dl, DstVT, SignBitTest, Slow, Fast);
4669  return true;
4670  }
4671 
4672  if (DstVT.getScalarType() == MVT::f64) {
4673  // Only expand vector types if we have the appropriate vector bit
4674  // operations.
4675  if (SrcVT.isVector() &&
4676  (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
4681  return false;
4682 
4683  // Implementation of unsigned i64 to f64 following the algorithm in
4684  // __floatundidf in compiler_rt. This implementation has the advantage
4685  // of performing rounding correctly, both in the default rounding mode
4686  // and in all alternate rounding modes.
4687  SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
4688  SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
4689  BitsToDouble(UINT64_C(0x4530000000100000)), dl, DstVT);
4690  SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
4691  SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
4692  SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
4693 
4694  SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
4695  SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
4696  SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
4697  SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
4698  SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
4699  SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
4700  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
4701  Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
4702  return true;
4703  }
4704 
4705  return false;
4706 }
4707 
4709  SelectionDAG &DAG) const {
4710  SDLoc dl(Node);
4711  unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
4713  EVT VT = Node->getValueType(0);
4714  if (isOperationLegalOrCustom(NewOp, VT)) {
4715  SDValue Quiet0 = Node->getOperand(0);
4716  SDValue Quiet1 = Node->getOperand(1);
4717 
4718  if (!Node->getFlags().hasNoNaNs()) {
4719  // Insert canonicalizes if it's possible we need to quiet to get correct
4720  // sNaN behavior.
4721  if (!DAG.isKnownNeverSNaN(Quiet0)) {
4722  Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
4723  Node->getFlags());
4724  }
4725  if (!DAG.isKnownNeverSNaN(Quiet1)) {
4726  Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
4727  Node->getFlags());
4728  }
4729  }
4730 
4731  return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
4732  }
4733 
4734  return SDValue();
4735 }
4736 
4738  SelectionDAG &DAG) const {
4739  SDLoc dl(Node);
4740  EVT VT = Node->getValueType(0);
4741  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4742  SDValue Op = Node->getOperand(0);
4743  unsigned Len = VT.getScalarSizeInBits();
4744  assert(VT.isInteger() && "CTPOP not implemented for this type.");
4745 
4746  // TODO: Add support for irregular type lengths.
4747  if (!(Len <= 128 && Len % 8 == 0))
4748  return false;
4749 
4750  // Only expand vector types if we have the appropriate vector bit operations.
4751  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||
4754  (Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||
4756  return false;
4757 
4758  // This is the "best" algorithm from
4759  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
4760  SDValue Mask55 =
4761  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
4762  SDValue Mask33 =
4763  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
4764  SDValue Mask0F =
4765  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
4766  SDValue Mask01 =
4767  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
4768 
4769  // v = v - ((v >> 1) & 0x55555555...)
4770  Op = DAG.getNode(ISD::SUB, dl, VT, Op,
4771  DAG.getNode(ISD::AND, dl, VT,
4772  DAG.getNode(ISD::SRL, dl, VT, Op,
4773  DAG.getConstant(1, dl, ShVT)),
4774  Mask55));
4775  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
4776  Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
4777  DAG.getNode(ISD::AND, dl, VT,
4778  DAG.getNode(ISD::SRL, dl, VT, Op,
4779  DAG.getConstant(2, dl, ShVT)),
4780  Mask33));
4781  // v = (v + (v >> 4)) & 0x0F0F0F0F...
4782  Op = DAG.getNode(ISD::AND, dl, VT,
4783  DAG.getNode(ISD::ADD, dl, VT, Op,
4784  DAG.getNode(ISD::SRL, dl, VT, Op,
4785  DAG.getConstant(4, dl, ShVT))),
4786  Mask0F);
4787  // v = (v * 0x01010101...) >> (Len - 8)
4788  if (Len > 8)
4789  Op =
4790  DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
4791  DAG.getConstant(Len - 8, dl, ShVT));
4792 
4793  Result = Op;
4794  return true;
4795 }
4796 
4798  SelectionDAG &DAG) const {
4799  SDLoc dl(Node);
4800  EVT VT = Node->getValueType(0);
4801  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4802  SDValue Op = Node->getOperand(0);
4803  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
4804 
4805  // If the non-ZERO_UNDEF version is supported we can use that instead.
4806  if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
4808  Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);
4809  return true;
4810  }
4811 
4812  // If the ZERO_UNDEF version is supported use that and handle the zero case.
4814  EVT SetCCVT =
4815  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
4816  SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
4817  SDValue Zero = DAG.getConstant(0, dl, VT);
4818  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
4819  Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
4820  DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
4821  return true;
4822  }
4823 
4824  // Only expand vector types if we have the appropriate vector bit operations.
4825  if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
4829  return false;
4830 
4831  // for now, we do this:
4832  // x = x | (x >> 1);
4833  // x = x | (x >> 2);
4834  // ...
4835  // x = x | (x >>16);
4836  // x = x | (x >>32); // for 64-bit input
4837  // return popcount(~x);
4838  //
4839  // Ref: "Hacker's Delight" by Henry Warren
4840  for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {
4841  SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
4842  Op = DAG.getNode(ISD::OR, dl, VT, Op,
4843  DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
4844  }
4845  Op = DAG.getNOT(dl, Op, VT);
4846  Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
4847  return true;
4848 }
4849 
4851  SelectionDAG &DAG) const {
4852  SDLoc dl(Node);
4853  EVT VT = Node->getValueType(0);
4854  SDValue Op = Node->getOperand(0);
4855  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
4856 
4857  // If the non-ZERO_UNDEF version is supported we can use that instead.
4858  if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
4860  Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
4861  return true;
4862  }
4863 
4864  // If the ZERO_UNDEF version is supported use that and handle the zero case.
4865  if (isOperationLegalOr