LLVM  8.0.0svn
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the TargetLowering class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/BitVector.h"
16 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/GlobalVariable.h"
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCExpr.h"
32 #include "llvm/Support/KnownBits.h"
36 #include <cctype>
37 using namespace llvm;
38 
39 /// NOTE: The TargetMachine owns TLOF.
41  : TargetLoweringBase(tm) {}
42 
43 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
44  return nullptr;
45 }
46 
49 }
50 
51 /// Check whether a given call node is in tail position within its function. If
52 /// so, it sets Chain to the input chain of the tail call.
54  SDValue &Chain) const {
55  const Function &F = DAG.getMachineFunction().getFunction();
56 
57  // Conservatively require the attributes of the call to match those of
58  // the return. Ignore NoAlias and NonNull because they don't affect the
59  // call sequence.
60  AttributeList CallerAttrs = F.getAttributes();
61  if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
62  .removeAttribute(Attribute::NoAlias)
63  .removeAttribute(Attribute::NonNull)
64  .hasAttributes())
65  return false;
66 
67  // It's not safe to eliminate the sign / zero extension of the return value.
68  if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
69  CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
70  return false;
71 
72  // Check if the only use is a function return node.
73  return isUsedByReturnOnly(Node, Chain);
74 }
75 
77  const uint32_t *CallerPreservedMask,
78  const SmallVectorImpl<CCValAssign> &ArgLocs,
79  const SmallVectorImpl<SDValue> &OutVals) const {
80  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
81  const CCValAssign &ArgLoc = ArgLocs[I];
82  if (!ArgLoc.isRegLoc())
83  continue;
84  unsigned Reg = ArgLoc.getLocReg();
85  // Only look at callee saved registers.
86  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
87  continue;
88  // Check that we pass the value used for the caller.
89  // (We look for a CopyFromReg reading a virtual register that is used
90  // for the function live-in value of register Reg)
91  SDValue Value = OutVals[I];
92  if (Value->getOpcode() != ISD::CopyFromReg)
93  return false;
94  unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
95  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
96  return false;
97  }
98  return true;
99 }
100 
101 /// Set CallLoweringInfo attribute flags based on a call instruction
102 /// and called function attributes.
104  unsigned ArgIdx) {
105  IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt);
106  IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt);
107  IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg);
108  IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet);
109  IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest);
110  IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal);
111  IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca);
112  IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned);
113  IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
114  IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError);
115  Alignment = CS->getParamAlignment(ArgIdx);
116 }
117 
118 /// Generate a libcall taking the given operands as arguments and returning a
119 /// result of type RetVT.
120 std::pair<SDValue, SDValue>
122  ArrayRef<SDValue> Ops, bool isSigned,
123  const SDLoc &dl, bool doesNotReturn,
124  bool isReturnValueUsed) const {
126  Args.reserve(Ops.size());
127 
129  for (SDValue Op : Ops) {
130  Entry.Node = Op;
131  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
132  Entry.IsSExt = shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
133  Entry.IsZExt = !shouldSignExtendTypeInLibCall(Op.getValueType(), isSigned);
134  Args.push_back(Entry);
135  }
136 
137  if (LC == RTLIB::UNKNOWN_LIBCALL)
138  report_fatal_error("Unsupported library call operation!");
140  getPointerTy(DAG.getDataLayout()));
141 
142  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
144  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, isSigned);
145  CLI.setDebugLoc(dl)
146  .setChain(DAG.getEntryNode())
147  .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
148  .setNoReturn(doesNotReturn)
149  .setDiscardResult(!isReturnValueUsed)
150  .setSExtResult(signExtend)
151  .setZExtResult(!signExtend);
152  return LowerCallTo(CLI);
153 }
154 
155 /// Soften the operands of a comparison. This code is shared among BR_CC,
156 /// SELECT_CC, and SETCC handlers.
158  SDValue &NewLHS, SDValue &NewRHS,
159  ISD::CondCode &CCCode,
160  const SDLoc &dl) const {
161  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
162  && "Unsupported setcc type!");
163 
164  // Expand into one or more soft-fp libcall(s).
165  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
166  bool ShouldInvertCC = false;
167  switch (CCCode) {
168  case ISD::SETEQ:
169  case ISD::SETOEQ:
170  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
171  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
172  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
173  break;
174  case ISD::SETNE:
175  case ISD::SETUNE:
176  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
177  (VT == MVT::f64) ? RTLIB::UNE_F64 :
178  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
179  break;
180  case ISD::SETGE:
181  case ISD::SETOGE:
182  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
183  (VT == MVT::f64) ? RTLIB::OGE_F64 :
184  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
185  break;
186  case ISD::SETLT:
187  case ISD::SETOLT:
188  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
189  (VT == MVT::f64) ? RTLIB::OLT_F64 :
190  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
191  break;
192  case ISD::SETLE:
193  case ISD::SETOLE:
194  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
195  (VT == MVT::f64) ? RTLIB::OLE_F64 :
196  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
197  break;
198  case ISD::SETGT:
199  case ISD::SETOGT:
200  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
201  (VT == MVT::f64) ? RTLIB::OGT_F64 :
202  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
203  break;
204  case ISD::SETUO:
205  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
206  (VT == MVT::f64) ? RTLIB::UO_F64 :
207  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
208  break;
209  case ISD::SETO:
210  LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
211  (VT == MVT::f64) ? RTLIB::O_F64 :
212  (VT == MVT::f128) ? RTLIB::O_F128 : RTLIB::O_PPCF128;
213  break;
214  case ISD::SETONE:
215  // SETONE = SETOLT | SETOGT
216  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
217  (VT == MVT::f64) ? RTLIB::OLT_F64 :
218  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
219  LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
220  (VT == MVT::f64) ? RTLIB::OGT_F64 :
221  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
222  break;
223  case ISD::SETUEQ:
224  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
225  (VT == MVT::f64) ? RTLIB::UO_F64 :
226  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
227  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
228  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
229  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
230  break;
231  default:
232  // Invert CC for unordered comparisons
233  ShouldInvertCC = true;
234  switch (CCCode) {
235  case ISD::SETULT:
236  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
237  (VT == MVT::f64) ? RTLIB::OGE_F64 :
238  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
239  break;
240  case ISD::SETULE:
241  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
242  (VT == MVT::f64) ? RTLIB::OGT_F64 :
243  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
244  break;
245  case ISD::SETUGT:
246  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
247  (VT == MVT::f64) ? RTLIB::OLE_F64 :
248  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
249  break;
250  case ISD::SETUGE:
251  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
252  (VT == MVT::f64) ? RTLIB::OLT_F64 :
253  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
254  break;
255  default: llvm_unreachable("Do not know how to soften this setcc!");
256  }
257  }
258 
259  // Use the target specific return value for comparions lib calls.
260  EVT RetVT = getCmpLibcallReturnType();
261  SDValue Ops[2] = {NewLHS, NewRHS};
262  NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, false /*sign irrelevant*/,
263  dl).first;
264  NewRHS = DAG.getConstant(0, dl, RetVT);
265 
266  CCCode = getCmpLibcallCC(LC1);
267  if (ShouldInvertCC)
268  CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);
269 
270  if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
271  SDValue Tmp = DAG.getNode(
272  ISD::SETCC, dl,
273  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
274  NewLHS, NewRHS, DAG.getCondCode(CCCode));
275  NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, false/*sign irrelevant*/,
276  dl).first;
277  NewLHS = DAG.getNode(
278  ISD::SETCC, dl,
279  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
280  NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
281  NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
282  NewRHS = SDValue();
283  }
284 }
285 
286 /// Return the entry encoding for a jump table in the current function. The
287 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
289  // In non-pic modes, just use the address of a block.
290  if (!isPositionIndependent())
292 
293  // In PIC mode, if the target supports a GPRel32 directive, use it.
294  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
296 
297  // Otherwise, use a label difference.
299 }
300 
302  SelectionDAG &DAG) const {
303  // If our PIC model is GP relative, use the global offset table as the base.
304  unsigned JTEncoding = getJumpTableEncoding();
305 
306  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
309 
310  return Table;
311 }
312 
313 /// This returns the relocation base for the given PIC jumptable, the same as
314 /// getPICJumpTableRelocBase, but as an MCExpr.
315 const MCExpr *
317  unsigned JTI,MCContext &Ctx) const{
318  // The normal PIC reloc base is the label at the start of the jump table.
319  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
320 }
321 
322 bool
324  const TargetMachine &TM = getTargetMachine();
325  const GlobalValue *GV = GA->getGlobal();
326 
327  // If the address is not even local to this DSO we will have to load it from
328  // a got and then add the offset.
329  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
330  return false;
331 
332  // If the code is position independent we will have to add a base register.
333  if (isPositionIndependent())
334  return false;
335 
336  // Otherwise we can do it.
337  return true;
338 }
339 
340 //===----------------------------------------------------------------------===//
341 // Optimization Methods
342 //===----------------------------------------------------------------------===//
343 
344 /// If the specified instruction has a constant integer operand and there are
345 /// bits set in that constant that are not demanded, then clear those bits and
346 /// return true.
348  TargetLoweringOpt &TLO) const {
349  SelectionDAG &DAG = TLO.DAG;
350  SDLoc DL(Op);
351  unsigned Opcode = Op.getOpcode();
352 
353  // Do target-specific constant optimization.
354  if (targetShrinkDemandedConstant(Op, Demanded, TLO))
355  return TLO.New.getNode();
356 
357  // FIXME: ISD::SELECT, ISD::SELECT_CC
358  switch (Opcode) {
359  default:
360  break;
361  case ISD::XOR:
362  case ISD::AND:
363  case ISD::OR: {
364  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
365  if (!Op1C)
366  return false;
367 
368  // If this is a 'not' op, don't touch it because that's a canonical form.
369  const APInt &C = Op1C->getAPIntValue();
370  if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
371  return false;
372 
373  if (!C.isSubsetOf(Demanded)) {
374  EVT VT = Op.getValueType();
375  SDValue NewC = DAG.getConstant(Demanded & C, DL, VT);
376  SDValue NewOp = DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
377  return TLO.CombineTo(Op, NewOp);
378  }
379 
380  break;
381  }
382  }
383 
384  return false;
385 }
386 
387 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
388 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
389 /// generalized for targets with other types of implicit widening casts.
391  const APInt &Demanded,
392  TargetLoweringOpt &TLO) const {
393  assert(Op.getNumOperands() == 2 &&
394  "ShrinkDemandedOp only supports binary operators!");
395  assert(Op.getNode()->getNumValues() == 1 &&
396  "ShrinkDemandedOp only supports nodes with one result!");
397 
398  SelectionDAG &DAG = TLO.DAG;
399  SDLoc dl(Op);
400 
401  // Early return, as this function cannot handle vector types.
402  if (Op.getValueType().isVector())
403  return false;
404 
405  // Don't do this if the node has another user, which may require the
406  // full value.
407  if (!Op.getNode()->hasOneUse())
408  return false;
409 
410  // Search for the smallest integer type with free casts to and from
411  // Op's type. For expedience, just check power-of-2 integer types.
412  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
413  unsigned DemandedSize = Demanded.getActiveBits();
414  unsigned SmallVTBits = DemandedSize;
415  if (!isPowerOf2_32(SmallVTBits))
416  SmallVTBits = NextPowerOf2(SmallVTBits);
417  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
418  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
419  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
420  TLI.isZExtFree(SmallVT, Op.getValueType())) {
421  // We found a type with free casts.
422  SDValue X = DAG.getNode(
423  Op.getOpcode(), dl, SmallVT,
424  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
425  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
426  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
427  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
428  return TLO.CombineTo(Op, Z);
429  }
430  }
431  return false;
432 }
433 
434 bool
436  const APInt &DemandedBits,
437  DAGCombinerInfo &DCI,
438  TargetLoweringOpt &TLO) const {
439  SDValue Op = User->getOperand(OpIdx);
440  KnownBits Known;
441 
442  if (!SimplifyDemandedBits(Op, DemandedBits, Known, TLO, 0, true))
443  return false;
444 
445 
446  // Old will not always be the same as Op. For example:
447  //
448  // Demanded = 0xffffff
449  // Op = i64 truncate (i32 and x, 0xffffff)
450  // In this case simplify demand bits will want to replace the 'and' node
451  // with the value 'x', which will give us:
452  // Old = i32 and x, 0xffffff
453  // New = x
454  if (TLO.Old.hasOneUse()) {
455  // For the one use case, we just commit the change.
456  DCI.CommitTargetLoweringOpt(TLO);
457  return true;
458  }
459 
460  // If Old has more than one use then it must be Op, because the
461  // AssumeSingleUse flag is not propogated to recursive calls of
462  // SimplifyDemanded bits, so the only node with multiple use that
463  // it will attempt to combine will be Op.
464  assert(TLO.Old == Op);
465 
467  for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
468  if (i == OpIdx) {
469  NewOps.push_back(TLO.New);
470  continue;
471  }
472  NewOps.push_back(User->getOperand(i));
473  }
474  User = TLO.DAG.UpdateNodeOperands(User, NewOps);
475  // Op has less users now, so we may be able to perform additional combines
476  // with it.
477  DCI.AddToWorklist(Op.getNode());
478  // User's operands have been updated, so we may be able to do new combines
479  // with it.
480  DCI.AddToWorklist(User);
481  return true;
482 }
483 
485  DAGCombinerInfo &DCI) const {
486  SelectionDAG &DAG = DCI.DAG;
487  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
488  !DCI.isBeforeLegalizeOps());
489  KnownBits Known;
490 
491  bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
492  if (Simplified) {
493  DCI.AddToWorklist(Op.getNode());
494  DCI.CommitTargetLoweringOpt(TLO);
495  }
496  return Simplified;
497 }
498 
500  KnownBits &Known,
501  TargetLoweringOpt &TLO,
502  unsigned Depth,
503  bool AssumeSingleUse) const {
504  EVT VT = Op.getValueType();
505  APInt DemandedElts = VT.isVector()
507  : APInt(1, 1);
508  return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
509  AssumeSingleUse);
510 }
511 
512 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
513 /// result of Op are ever used downstream. If we can use this information to
514 /// simplify Op, create a new simplified DAG node and return true, returning the
515 /// original and new nodes in Old and New. Otherwise, analyze the expression and
516 /// return a mask of Known bits for the expression (used to simplify the
517 /// caller). The Known bits may only be accurate for those bits in the
518 /// OriginalDemandedBits and OriginalDemandedElts.
520  SDValue Op, const APInt &OriginalDemandedBits,
521  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
522  unsigned Depth, bool AssumeSingleUse) const {
523  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
524  assert(Op.getScalarValueSizeInBits() == BitWidth &&
525  "Mask size mismatches value type size!");
526 
527  unsigned NumElts = OriginalDemandedElts.getBitWidth();
528  assert((!Op.getValueType().isVector() ||
529  NumElts == Op.getValueType().getVectorNumElements()) &&
530  "Unexpected vector size");
531 
532  APInt DemandedBits = OriginalDemandedBits;
533  APInt DemandedElts = OriginalDemandedElts;
534  SDLoc dl(Op);
535  auto &DL = TLO.DAG.getDataLayout();
536 
537  // Don't know anything.
538  Known = KnownBits(BitWidth);
539 
540  if (Op.getOpcode() == ISD::Constant) {
541  // We know all of the bits for a constant!
542  Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
543  Known.Zero = ~Known.One;
544  return false;
545  }
546 
547  // Other users may use these bits.
548  EVT VT = Op.getValueType();
549  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
550  if (Depth != 0) {
551  // If not at the root, Just compute the Known bits to
552  // simplify things downstream.
553  TLO.DAG.computeKnownBits(Op, Known, DemandedElts, Depth);
554  return false;
555  }
556  // If this is the root being simplified, allow it to have multiple uses,
557  // just set the DemandedBits/Elts to all bits.
558  DemandedBits = APInt::getAllOnesValue(BitWidth);
559  DemandedElts = APInt::getAllOnesValue(NumElts);
560  } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
561  // Not demanding any bits/elts from Op.
562  if (!Op.isUndef())
563  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
564  return false;
565  } else if (Depth == 6) { // Limit search depth.
566  return false;
567  }
568 
569  KnownBits Known2, KnownOut;
570  switch (Op.getOpcode()) {
571  case ISD::BUILD_VECTOR:
572  // Collect the known bits that are shared by every constant vector element.
573  Known.Zero.setAllBits(); Known.One.setAllBits();
574  for (SDValue SrcOp : Op->ops()) {
575  if (!isa<ConstantSDNode>(SrcOp)) {
576  // We can only handle all constant values - bail out with no known bits.
577  Known = KnownBits(BitWidth);
578  return false;
579  }
580  Known2.One = cast<ConstantSDNode>(SrcOp)->getAPIntValue();
581  Known2.Zero = ~Known2.One;
582 
583  // BUILD_VECTOR can implicitly truncate sources, we must handle this.
584  if (Known2.One.getBitWidth() != BitWidth) {
585  assert(Known2.getBitWidth() > BitWidth &&
586  "Expected BUILD_VECTOR implicit truncation");
587  Known2 = Known2.trunc(BitWidth);
588  }
589 
590  // Known bits are the values that are shared by every element.
591  // TODO: support per-element known bits.
592  Known.One &= Known2.One;
593  Known.Zero &= Known2.Zero;
594  }
595  return false; // Don't fall through, will infinitely loop.
596  case ISD::CONCAT_VECTORS: {
597  Known.Zero.setAllBits();
598  Known.One.setAllBits();
599  EVT SubVT = Op.getOperand(0).getValueType();
600  unsigned NumSubVecs = Op.getNumOperands();
601  unsigned NumSubElts = SubVT.getVectorNumElements();
602  for (unsigned i = 0; i != NumSubVecs; ++i) {
603  APInt DemandedSubElts =
604  DemandedElts.extractBits(NumSubElts, i * NumSubElts);
605  if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
606  Known2, TLO, Depth + 1))
607  return true;
608  // Known bits are shared by every demanded subvector element.
609  if (!!DemandedSubElts) {
610  Known.One &= Known2.One;
611  Known.Zero &= Known2.Zero;
612  }
613  }
614  break;
615  }
616  case ISD::VECTOR_SHUFFLE: {
617  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
618 
619  // Collect demanded elements from shuffle operands..
620  APInt DemandedLHS(NumElts, 0);
621  APInt DemandedRHS(NumElts, 0);
622  for (unsigned i = 0; i != NumElts; ++i) {
623  if (!DemandedElts[i])
624  continue;
625  int M = ShuffleMask[i];
626  if (M < 0) {
627  // For UNDEF elements, we don't know anything about the common state of
628  // the shuffle result.
629  DemandedLHS.clearAllBits();
630  DemandedRHS.clearAllBits();
631  break;
632  }
633  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
634  if (M < (int)NumElts)
635  DemandedLHS.setBit(M);
636  else
637  DemandedRHS.setBit(M - NumElts);
638  }
639 
640  if (!!DemandedLHS || !!DemandedRHS) {
641  Known.Zero.setAllBits();
642  Known.One.setAllBits();
643  if (!!DemandedLHS) {
644  if (SimplifyDemandedBits(Op.getOperand(0), DemandedBits, DemandedLHS,
645  Known2, TLO, Depth + 1))
646  return true;
647  Known.One &= Known2.One;
648  Known.Zero &= Known2.Zero;
649  }
650  if (!!DemandedRHS) {
651  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedRHS,
652  Known2, TLO, Depth + 1))
653  return true;
654  Known.One &= Known2.One;
655  Known.Zero &= Known2.Zero;
656  }
657  }
658  break;
659  }
660  case ISD::AND: {
661  SDValue Op0 = Op.getOperand(0);
662  SDValue Op1 = Op.getOperand(1);
663 
664  // If the RHS is a constant, check to see if the LHS would be zero without
665  // using the bits from the RHS. Below, we use knowledge about the RHS to
666  // simplify the LHS, here we're using information from the LHS to simplify
667  // the RHS.
668  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
669  KnownBits LHSKnown;
670  // Do not increment Depth here; that can cause an infinite loop.
671  TLO.DAG.computeKnownBits(Op0, LHSKnown, DemandedElts, Depth);
672  // If the LHS already has zeros where RHSC does, this 'and' is dead.
673  if ((LHSKnown.Zero & DemandedBits) ==
674  (~RHSC->getAPIntValue() & DemandedBits))
675  return TLO.CombineTo(Op, Op0);
676 
677  // If any of the set bits in the RHS are known zero on the LHS, shrink
678  // the constant.
679  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits, TLO))
680  return true;
681 
682  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
683  // constant, but if this 'and' is only clearing bits that were just set by
684  // the xor, then this 'and' can be eliminated by shrinking the mask of
685  // the xor. For example, for a 32-bit X:
686  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
687  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
688  LHSKnown.One == ~RHSC->getAPIntValue()) {
689  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
690  return TLO.CombineTo(Op, Xor);
691  }
692  }
693 
694  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1))
695  return true;
696  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
697  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts, Known2, TLO,
698  Depth + 1))
699  return true;
700  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
701 
702  // If all of the demanded bits are known one on one side, return the other.
703  // These bits cannot contribute to the result of the 'and'.
704  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
705  return TLO.CombineTo(Op, Op0);
706  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
707  return TLO.CombineTo(Op, Op1);
708  // If all of the demanded bits in the inputs are known zeros, return zero.
709  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
710  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
711  // If the RHS is a constant, see if we can simplify it.
712  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, TLO))
713  return true;
714  // If the operation can be done in a smaller type, do so.
715  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
716  return true;
717 
718  // Output known-1 bits are only known if set in both the LHS & RHS.
719  Known.One &= Known2.One;
720  // Output known-0 are known to be clear if zero in either the LHS | RHS.
721  Known.Zero |= Known2.Zero;
722  break;
723  }
724  case ISD::OR: {
725  SDValue Op0 = Op.getOperand(0);
726  SDValue Op1 = Op.getOperand(1);
727 
728  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1))
729  return true;
730  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
731  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts, Known2, TLO,
732  Depth + 1))
733  return true;
734  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
735 
736  // If all of the demanded bits are known zero on one side, return the other.
737  // These bits cannot contribute to the result of the 'or'.
738  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
739  return TLO.CombineTo(Op, Op0);
740  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
741  return TLO.CombineTo(Op, Op1);
742  // If the RHS is a constant, see if we can simplify it.
743  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
744  return true;
745  // If the operation can be done in a smaller type, do so.
746  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
747  return true;
748 
749  // Output known-0 bits are only known if clear in both the LHS & RHS.
750  Known.Zero &= Known2.Zero;
751  // Output known-1 are known to be set if set in either the LHS | RHS.
752  Known.One |= Known2.One;
753  break;
754  }
755  case ISD::XOR: {
756  SDValue Op0 = Op.getOperand(0);
757  SDValue Op1 = Op.getOperand(1);
758 
759  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1))
760  return true;
761  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
762  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO, Depth + 1))
763  return true;
764  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
765 
766  // If all of the demanded bits are known zero on one side, return the other.
767  // These bits cannot contribute to the result of the 'xor'.
768  if (DemandedBits.isSubsetOf(Known.Zero))
769  return TLO.CombineTo(Op, Op0);
770  if (DemandedBits.isSubsetOf(Known2.Zero))
771  return TLO.CombineTo(Op, Op1);
772  // If the operation can be done in a smaller type, do so.
773  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
774  return true;
775 
776  // If all of the unknown bits are known to be zero on one side or the other
777  // (but not both) turn this into an *inclusive* or.
778  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
779  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
780  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
781 
782  // Output known-0 bits are known if clear or set in both the LHS & RHS.
783  KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
784  // Output known-1 are known to be set if set in only one of the LHS, RHS.
785  KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
786 
787  if (ConstantSDNode *C = isConstOrConstSplat(Op1)) {
788  // If one side is a constant, and all of the known set bits on the other
789  // side are also set in the constant, turn this into an AND, as we know
790  // the bits will be cleared.
791  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
792  // NB: it is okay if more bits are known than are requested
793  if (C->getAPIntValue() == Known2.One) {
794  SDValue ANDC =
795  TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
796  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
797  }
798 
799  // If the RHS is a constant, see if we can change it. Don't alter a -1
800  // constant because that's a 'not' op, and that is better for combining
801  // and codegen.
802  if (!C->isAllOnesValue()) {
803  if (DemandedBits.isSubsetOf(C->getAPIntValue())) {
804  // We're flipping all demanded bits. Flip the undemanded bits too.
805  SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
806  return TLO.CombineTo(Op, New);
807  }
808  // If we can't turn this into a 'not', try to shrink the constant.
809  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
810  return true;
811  }
812  }
813 
814  Known = std::move(KnownOut);
815  break;
816  }
817  case ISD::SELECT:
818  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
819  Depth + 1))
820  return true;
821  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
822  Depth + 1))
823  return true;
824  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
825  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
826 
827  // If the operands are constants, see if we can simplify them.
828  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
829  return true;
830 
831  // Only known if known in both the LHS and RHS.
832  Known.One &= Known2.One;
833  Known.Zero &= Known2.Zero;
834  break;
835  case ISD::SELECT_CC:
836  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
837  Depth + 1))
838  return true;
839  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
840  Depth + 1))
841  return true;
842  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
843  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
844 
845  // If the operands are constants, see if we can simplify them.
846  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
847  return true;
848 
849  // Only known if known in both the LHS and RHS.
850  Known.One &= Known2.One;
851  Known.Zero &= Known2.Zero;
852  break;
853  case ISD::SETCC: {
854  SDValue Op0 = Op.getOperand(0);
855  SDValue Op1 = Op.getOperand(1);
856  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
857  // If (1) we only need the sign-bit, (2) the setcc operands are the same
858  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
859  // -1, we may be able to bypass the setcc.
860  if (DemandedBits.isSignMask() &&
861  Op0.getScalarValueSizeInBits() == BitWidth &&
862  getBooleanContents(VT) ==
863  BooleanContent::ZeroOrNegativeOneBooleanContent) {
864  // If we're testing X < 0, then this compare isn't needed - just use X!
865  // FIXME: We're limiting to integer types here, but this should also work
866  // if we don't care about FP signed-zero. The use of SETLT with FP means
867  // that we don't care about NaNs.
868  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
870  return TLO.CombineTo(Op, Op0);
871 
872  // TODO: Should we check for other forms of sign-bit comparisons?
873  // Examples: X <= -1, X >= 0
874  }
875  if (getBooleanContents(Op0.getValueType()) ==
877  BitWidth > 1)
878  Known.Zero.setBitsFrom(1);
879  break;
880  }
881  case ISD::SHL: {
882  SDValue Op0 = Op.getOperand(0);
883  SDValue Op1 = Op.getOperand(1);
884 
885  if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) {
886  // If the shift count is an invalid immediate, don't do anything.
887  if (SA->getAPIntValue().uge(BitWidth))
888  break;
889 
890  unsigned ShAmt = SA->getZExtValue();
891 
892  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
893  // single shift. We can do this if the bottom bits (which are shifted
894  // out) are never demanded.
895  if (Op0.getOpcode() == ISD::SRL) {
896  if (ShAmt &&
897  (DemandedBits & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) {
898  if (ConstantSDNode *SA2 = isConstOrConstSplat(Op0.getOperand(1))) {
899  if (SA2->getAPIntValue().ult(BitWidth)) {
900  unsigned C1 = SA2->getZExtValue();
901  unsigned Opc = ISD::SHL;
902  int Diff = ShAmt - C1;
903  if (Diff < 0) {
904  Diff = -Diff;
905  Opc = ISD::SRL;
906  }
907 
908  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
909  return TLO.CombineTo(
910  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
911  }
912  }
913  }
914  }
915 
916  if (SimplifyDemandedBits(Op0, DemandedBits.lshr(ShAmt), DemandedElts, Known, TLO,
917  Depth + 1))
918  return true;
919 
920  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
921  // are not demanded. This will likely allow the anyext to be folded away.
922  if (Op0.getOpcode() == ISD::ANY_EXTEND) {
923  SDValue InnerOp = Op0.getOperand(0);
924  EVT InnerVT = InnerOp.getValueType();
925  unsigned InnerBits = InnerVT.getScalarSizeInBits();
926  if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
927  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
928  EVT ShTy = getShiftAmountTy(InnerVT, DL);
929  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
930  ShTy = InnerVT;
931  SDValue NarrowShl =
932  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
933  TLO.DAG.getConstant(ShAmt, dl, ShTy));
934  return TLO.CombineTo(
935  Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
936  }
937  // Repeat the SHL optimization above in cases where an extension
938  // intervenes: (shl (anyext (shr x, c1)), c2) to
939  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
940  // aren't demanded (as above) and that the shifted upper c1 bits of
941  // x aren't demanded.
942  if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
943  InnerOp.hasOneUse()) {
944  if (ConstantSDNode *SA2 =
945  isConstOrConstSplat(InnerOp.getOperand(1))) {
946  unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
947  if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
948  DemandedBits.getActiveBits() <=
949  (InnerBits - InnerShAmt + ShAmt) &&
950  DemandedBits.countTrailingZeros() >= ShAmt) {
951  SDValue NewSA = TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
952  Op1.getValueType());
953  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
954  InnerOp.getOperand(0));
955  return TLO.CombineTo(
956  Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
957  }
958  }
959  }
960  }
961 
962  Known.Zero <<= ShAmt;
963  Known.One <<= ShAmt;
964  // low bits known zero.
965  Known.Zero.setLowBits(ShAmt);
966  }
967  break;
968  }
969  case ISD::SRL: {
970  SDValue Op0 = Op.getOperand(0);
971  SDValue Op1 = Op.getOperand(1);
972 
973  if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) {
974  // If the shift count is an invalid immediate, don't do anything.
975  if (SA->getAPIntValue().uge(BitWidth))
976  break;
977 
978  unsigned ShAmt = SA->getZExtValue();
979  APInt InDemandedMask = (DemandedBits << ShAmt);
980 
981  // If the shift is exact, then it does demand the low bits (and knows that
982  // they are zero).
983  if (Op->getFlags().hasExact())
984  InDemandedMask.setLowBits(ShAmt);
985 
986  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
987  // single shift. We can do this if the top bits (which are shifted out)
988  // are never demanded.
989  if (Op0.getOpcode() == ISD::SHL) {
990  if (ConstantSDNode *SA2 = isConstOrConstSplat(Op0.getOperand(1))) {
991  if (ShAmt &&
992  (DemandedBits & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) {
993  if (SA2->getAPIntValue().ult(BitWidth)) {
994  unsigned C1 = SA2->getZExtValue();
995  unsigned Opc = ISD::SRL;
996  int Diff = ShAmt - C1;
997  if (Diff < 0) {
998  Diff = -Diff;
999  Opc = ISD::SHL;
1000  }
1001 
1002  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
1003  return TLO.CombineTo(
1004  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1005  }
1006  }
1007  }
1008  }
1009 
1010  // Compute the new bits that are at the top now.
1011  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, Depth + 1))
1012  return true;
1013  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1014  Known.Zero.lshrInPlace(ShAmt);
1015  Known.One.lshrInPlace(ShAmt);
1016 
1017  Known.Zero.setHighBits(ShAmt); // High bits known zero.
1018  }
1019  break;
1020  }
1021  case ISD::SRA: {
1022  SDValue Op0 = Op.getOperand(0);
1023  SDValue Op1 = Op.getOperand(1);
1024 
1025  // If this is an arithmetic shift right and only the low-bit is set, we can
1026  // always convert this into a logical shr, even if the shift amount is
1027  // variable. The low bit of the shift cannot be an input sign bit unless
1028  // the shift amount is >= the size of the datatype, which is undefined.
1029  if (DemandedBits.isOneValue())
1030  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1031 
1032  if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) {
1033  // If the shift count is an invalid immediate, don't do anything.
1034  if (SA->getAPIntValue().uge(BitWidth))
1035  break;
1036 
1037  unsigned ShAmt = SA->getZExtValue();
1038  APInt InDemandedMask = (DemandedBits << ShAmt);
1039 
1040  // If the shift is exact, then it does demand the low bits (and knows that
1041  // they are zero).
1042  if (Op->getFlags().hasExact())
1043  InDemandedMask.setLowBits(ShAmt);
1044 
1045  // If any of the demanded bits are produced by the sign extension, we also
1046  // demand the input sign bit.
1047  if (DemandedBits.countLeadingZeros() < ShAmt)
1048  InDemandedMask.setSignBit();
1049 
1050  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, Depth + 1))
1051  return true;
1052  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1053  Known.Zero.lshrInPlace(ShAmt);
1054  Known.One.lshrInPlace(ShAmt);
1055 
1056  // If the input sign bit is known to be zero, or if none of the top bits
1057  // are demanded, turn this into an unsigned shift right.
1058  if (Known.Zero[BitWidth - ShAmt - 1] ||
1059  DemandedBits.countLeadingZeros() >= ShAmt) {
1060  SDNodeFlags Flags;
1061  Flags.setExact(Op->getFlags().hasExact());
1062  return TLO.CombineTo(
1063  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1064  }
1065 
1066  int Log2 = DemandedBits.exactLogBase2();
1067  if (Log2 >= 0) {
1068  // The bit must come from the sign.
1069  SDValue NewSA =
1070  TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, Op1.getValueType());
1071  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1072  }
1073 
1074  if (Known.One[BitWidth - ShAmt - 1])
1075  // New bits are known one.
1076  Known.One.setHighBits(ShAmt);
1077  }
1078  break;
1079  }
1080  case ISD::SIGN_EXTEND_INREG: {
1081  SDValue Op0 = Op.getOperand(0);
1082  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1083  unsigned ExVTBits = ExVT.getScalarSizeInBits();
1084 
1085  // If we only care about the highest bit, don't bother shifting right.
1086  if (DemandedBits.isSignMask()) {
1087  bool AlreadySignExtended =
1088  TLO.DAG.ComputeNumSignBits(Op0) >= BitWidth - ExVTBits + 1;
1089  // However if the input is already sign extended we expect the sign
1090  // extension to be dropped altogether later and do not simplify.
1091  if (!AlreadySignExtended) {
1092  // Compute the correct shift amount type, which must be getShiftAmountTy
1093  // for scalar types after legalization.
1094  EVT ShiftAmtTy = VT;
1095  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1096  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1097 
1098  SDValue ShiftAmt =
1099  TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1100  return TLO.CombineTo(Op,
1101  TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1102  }
1103  }
1104 
1105  // If none of the extended bits are demanded, eliminate the sextinreg.
1106  if (DemandedBits.getActiveBits() <= ExVTBits)
1107  return TLO.CombineTo(Op, Op0);
1108 
1109  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1110 
1111  // Since the sign extended bits are demanded, we know that the sign
1112  // bit is demanded.
1113  InputDemandedBits.setBit(ExVTBits - 1);
1114 
1115  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1116  return true;
1117  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1118 
1119  // If the sign bit of the input is known set or clear, then we know the
1120  // top bits of the result.
1121 
1122  // If the input sign bit is known zero, convert this into a zero extension.
1123  if (Known.Zero[ExVTBits - 1])
1124  return TLO.CombineTo(
1125  Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT.getScalarType()));
1126 
1127  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1128  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1129  Known.One.setBitsFrom(ExVTBits);
1130  Known.Zero &= Mask;
1131  } else { // Input sign bit unknown
1132  Known.Zero &= Mask;
1133  Known.One &= Mask;
1134  }
1135  break;
1136  }
1137  case ISD::BUILD_PAIR: {
1138  EVT HalfVT = Op.getOperand(0).getValueType();
1139  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1140 
1141  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1142  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1143 
1144  KnownBits KnownLo, KnownHi;
1145 
1146  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1147  return true;
1148 
1149  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1150  return true;
1151 
1152  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1153  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1154 
1155  Known.One = KnownLo.One.zext(BitWidth) |
1156  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1157  break;
1158  }
1159  case ISD::ZERO_EXTEND: {
1160  SDValue Src = Op.getOperand(0);
1161  unsigned InBits = Src.getScalarValueSizeInBits();
1162 
1163  // If none of the top bits are demanded, convert this into an any_extend.
1164  if (DemandedBits.getActiveBits() <= InBits)
1165  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, Src));
1166 
1167  APInt InDemandedBits = DemandedBits.trunc(InBits);
1168  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth+1))
1169  return true;
1170  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1171  Known = Known.zext(BitWidth);
1172  Known.Zero.setBitsFrom(InBits);
1173  break;
1174  }
1175  case ISD::SIGN_EXTEND: {
1176  SDValue Src = Op.getOperand(0);
1177  unsigned InBits = Src.getScalarValueSizeInBits();
1178 
1179  // If none of the top bits are demanded, convert this into an any_extend.
1180  if (DemandedBits.getActiveBits() <= InBits)
1181  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, Src));
1182 
1183  // Since some of the sign extended bits are demanded, we know that the sign
1184  // bit is demanded.
1185  APInt InDemandedBits = DemandedBits.trunc(InBits);
1186  InDemandedBits.setBit(InBits - 1);
1187 
1188  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1))
1189  return true;
1190  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1191  // If the sign bit is known one, the top bits match.
1192  Known = Known.sext(BitWidth);
1193 
1194  // If the sign bit is known zero, convert this to a zero extend.
1195  if (Known.isNonNegative())
1196  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Src));
1197  break;
1198  }
1200  // TODO - merge this with SIGN_EXTEND above?
1201  SDValue Src = Op.getOperand(0);
1202  unsigned InBits = Src.getScalarValueSizeInBits();
1203 
1204  APInt InDemandedBits = DemandedBits.trunc(InBits);
1205 
1206  // If some of the sign extended bits are demanded, we know that the sign
1207  // bit is demanded.
1208  if (InBits < DemandedBits.getActiveBits())
1209  InDemandedBits.setBit(InBits - 1);
1210 
1211  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1))
1212  return true;
1213  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1214  // If the sign bit is known one, the top bits match.
1215  Known = Known.sext(BitWidth);
1216  break;
1217  }
1218  case ISD::ANY_EXTEND: {
1219  SDValue Src = Op.getOperand(0);
1220  unsigned InBits = Src.getScalarValueSizeInBits();
1221  APInt InDemandedBits = DemandedBits.trunc(InBits);
1222  if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth+1))
1223  return true;
1224  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1225  Known = Known.zext(BitWidth);
1226  break;
1227  }
1228  case ISD::TRUNCATE: {
1229  SDValue Src = Op.getOperand(0);
1230 
1231  // Simplify the input, using demanded bit information, and compute the known
1232  // zero/one bits live out.
1233  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1234  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1235  if (SimplifyDemandedBits(Src, TruncMask, Known, TLO, Depth + 1))
1236  return true;
1237  Known = Known.trunc(BitWidth);
1238 
1239  // If the input is only used by this truncate, see if we can shrink it based
1240  // on the known demanded bits.
1241  if (Src.getNode()->hasOneUse()) {
1242  switch (Src.getOpcode()) {
1243  default:
1244  break;
1245  case ISD::SRL:
1246  // Shrink SRL by a constant if none of the high bits shifted in are
1247  // demanded.
1248  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
1249  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
1250  // undesirable.
1251  break;
1253  if (!ShAmt)
1254  break;
1255  SDValue Shift = Src.getOperand(1);
1256  if (TLO.LegalTypes()) {
1257  uint64_t ShVal = ShAmt->getZExtValue();
1258  Shift = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));
1259  }
1260 
1261  if (ShAmt->getZExtValue() < BitWidth) {
1262  APInt HighBits = APInt::getHighBitsSet(OperandBitWidth,
1263  OperandBitWidth - BitWidth);
1264  HighBits.lshrInPlace(ShAmt->getZExtValue());
1265  HighBits = HighBits.trunc(BitWidth);
1266 
1267  if (!(HighBits & DemandedBits)) {
1268  // None of the shifted in bits are needed. Add a truncate of the
1269  // shift input, then shift it.
1270  SDValue NewTrunc =
1271  TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
1272  return TLO.CombineTo(
1273  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, Shift));
1274  }
1275  }
1276  break;
1277  }
1278  }
1279 
1280  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1281  break;
1282  }
1283  case ISD::AssertZext: {
1284  // AssertZext demands all of the high bits, plus any of the low bits
1285  // demanded by its users.
1286  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1287  APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
1288  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits,
1289  Known, TLO, Depth+1))
1290  return true;
1291  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1292 
1293  Known.Zero |= ~InMask;
1294  break;
1295  }
1296  case ISD::EXTRACT_VECTOR_ELT: {
1297  SDValue Src = Op.getOperand(0);
1298  SDValue Idx = Op.getOperand(1);
1299  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1300  unsigned EltBitWidth = Src.getScalarValueSizeInBits();
1301 
1302  // Demand the bits from every vector element without a constant index.
1303  APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
1304  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
1305  if (CIdx->getAPIntValue().ult(NumSrcElts))
1306  DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
1307 
1308  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
1309  // anything about the extended bits.
1310  APInt DemandedSrcBits = DemandedBits;
1311  if (BitWidth > EltBitWidth)
1312  DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
1313 
1314  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
1315  Depth + 1))
1316  return true;
1317 
1318  Known = Known2;
1319  if (BitWidth > EltBitWidth)
1320  Known = Known.zext(BitWidth);
1321  break;
1322  }
1323  case ISD::BITCAST: {
1324  SDValue Src = Op.getOperand(0);
1325  EVT SrcVT = Src.getValueType();
1326  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
1327 
1328  // If this is an FP->Int bitcast and if the sign bit is the only
1329  // thing demanded, turn this into a FGETSIGN.
1330  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
1331  DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
1332  SrcVT.isFloatingPoint()) {
1333  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
1335  if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
1336  SrcVT != MVT::f128) {
1337  // Cannot eliminate/lower SHL for f128 yet.
1338  EVT Ty = OpVTLegal ? VT : MVT::i32;
1339  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
1340  // place. We expect the SHL to be eliminated by other optimizations.
1341  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
1342  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
1343  if (!OpVTLegal && OpVTSizeInBits > 32)
1344  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
1345  unsigned ShVal = Op.getValueSizeInBits() - 1;
1346  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1347  return TLO.CombineTo(Op,
1348  TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
1349  }
1350  }
1351  // If bitcast from a vector, see if we can use SimplifyDemandedVectorElts by
1352  // demanding the element if any bits from it are demanded.
1353  // TODO - bigendian once we have test coverage.
1354  // TODO - bool vectors once SimplifyDemandedVectorElts has SETCC support.
1355  if (SrcVT.isVector() && NumSrcEltBits > 1 &&
1356  (BitWidth % NumSrcEltBits) == 0 &&
1357  TLO.DAG.getDataLayout().isLittleEndian()) {
1358  unsigned Scale = BitWidth / NumSrcEltBits;
1359  auto GetDemandedSubMask = [&](APInt &DemandedSubElts) -> bool {
1360  DemandedSubElts = APInt::getNullValue(Scale);
1361  for (unsigned i = 0; i != Scale; ++i) {
1362  unsigned Offset = i * NumSrcEltBits;
1363  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
1364  if (!Sub.isNullValue())
1365  DemandedSubElts.setBit(i);
1366  }
1367  return true;
1368  };
1369 
1370  APInt DemandedSubElts;
1371  if (GetDemandedSubMask(DemandedSubElts)) {
1372  unsigned NumSrcElts = SrcVT.getVectorNumElements();
1373  APInt DemandedElts = APInt::getSplat(NumSrcElts, DemandedSubElts);
1374 
1375  APInt KnownUndef, KnownZero;
1376  if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
1377  TLO, Depth + 1))
1378  return true;
1379  }
1380  }
1381  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
1382  // recursive call where Known may be useful to the caller.
1383  if (Depth > 0) {
1384  TLO.DAG.computeKnownBits(Op, Known, Depth);
1385  return false;
1386  }
1387  break;
1388  }
1389  case ISD::ADD:
1390  case ISD::MUL:
1391  case ISD::SUB: {
1392  // Add, Sub, and Mul don't demand any bits in positions beyond that
1393  // of the highest bit demanded of them.
1394  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
1395  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
1396  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
1397  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO, Depth + 1) ||
1398  SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO, Depth + 1) ||
1399  // See if the operation should be performed at a smaller bit width.
1400  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
1401  SDNodeFlags Flags = Op.getNode()->getFlags();
1402  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1403  // Disable the nsw and nuw flags. We can no longer guarantee that we
1404  // won't wrap after simplification.
1405  Flags.setNoSignedWrap(false);
1406  Flags.setNoUnsignedWrap(false);
1407  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1,
1408  Flags);
1409  return TLO.CombineTo(Op, NewOp);
1410  }
1411  return true;
1412  }
1413 
1414  // If we have a constant operand, we may be able to turn it into -1 if we
1415  // do not demand the high bits. This can make the constant smaller to
1416  // encode, allow more general folding, or match specialized instruction
1417  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
1418  // is probably not useful (and could be detrimental).
1420  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
1421  if (C && !C->isAllOnesValue() && !C->isOne() &&
1422  (C->getAPIntValue() | HighMask).isAllOnesValue()) {
1423  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
1424  // We can't guarantee that the new math op doesn't wrap, so explicitly
1425  // clear those flags to prevent folding with a potential existing node
1426  // that has those flags set.
1427  SDNodeFlags Flags;
1428  Flags.setNoSignedWrap(false);
1429  Flags.setNoUnsignedWrap(false);
1430  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
1431  return TLO.CombineTo(Op, NewOp);
1432  }
1433 
1435  }
1436  default:
1437  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
1438  if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
1439  Known, TLO, Depth))
1440  return true;
1441  break;
1442  }
1443 
1444  // Just use computeKnownBits to compute output bits.
1445  TLO.DAG.computeKnownBits(Op, Known, DemandedElts, Depth);
1446  break;
1447  }
1448 
1449  // If we know the value of all of the demanded bits, return this as a
1450  // constant.
1451  if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
1452  // Avoid folding to a constant if any OpaqueConstant is involved.
1453  const SDNode *N = Op.getNode();
1455  E = SDNodeIterator::end(N);
1456  I != E; ++I) {
1457  SDNode *Op = *I;
1458  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
1459  if (C->isOpaque())
1460  return false;
1461  }
1462  // TODO: Handle float bits as well.
1463  if (VT.isInteger())
1464  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
1465  }
1466 
1467  return false;
1468 }
1469 
1471  const APInt &DemandedElts,
1472  APInt &KnownUndef,
1473  APInt &KnownZero,
1474  DAGCombinerInfo &DCI) const {
1475  SelectionDAG &DAG = DCI.DAG;
1476  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
1477  !DCI.isBeforeLegalizeOps());
1478 
1479  bool Simplified =
1480  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
1481  if (Simplified) {
1482  DCI.AddToWorklist(Op.getNode());
1483  DCI.CommitTargetLoweringOpt(TLO);
1484  }
1485  return Simplified;
1486 }
1487 
1489  SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef,
1490  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
1491  bool AssumeSingleUse) const {
1492  EVT VT = Op.getValueType();
1493  APInt DemandedElts = DemandedEltMask;
1494  unsigned NumElts = DemandedElts.getBitWidth();
1495  assert(VT.isVector() && "Expected vector op");
1496  assert(VT.getVectorNumElements() == NumElts &&
1497  "Mask size mismatches value type element count!");
1498 
1499  KnownUndef = KnownZero = APInt::getNullValue(NumElts);
1500 
1501  // Undef operand.
1502  if (Op.isUndef()) {
1503  KnownUndef.setAllBits();
1504  return false;
1505  }
1506 
1507  // If Op has other users, assume that all elements are needed.
1508  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
1509  DemandedElts.setAllBits();
1510 
1511  // Not demanding any elements from Op.
1512  if (DemandedElts == 0) {
1513  KnownUndef.setAllBits();
1514  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1515  }
1516 
1517  // Limit search depth.
1518  if (Depth >= 6)
1519  return false;
1520 
1521  SDLoc DL(Op);
1522  unsigned EltSizeInBits = VT.getScalarSizeInBits();
1523 
1524  switch (Op.getOpcode()) {
1525  case ISD::SCALAR_TO_VECTOR: {
1526  if (!DemandedElts[0]) {
1527  KnownUndef.setAllBits();
1528  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1529  }
1530  KnownUndef.setHighBits(NumElts - 1);
1531  break;
1532  }
1533  case ISD::BITCAST: {
1534  SDValue Src = Op.getOperand(0);
1535  EVT SrcVT = Src.getValueType();
1536 
1537  // We only handle vectors here.
1538  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
1539  if (!SrcVT.isVector())
1540  break;
1541 
1542  // Fast handling of 'identity' bitcasts.
1543  unsigned NumSrcElts = SrcVT.getVectorNumElements();
1544  if (NumSrcElts == NumElts)
1545  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
1546  KnownZero, TLO, Depth + 1);
1547 
1548  APInt SrcZero, SrcUndef;
1549  APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
1550 
1551  // Bitcast from 'large element' src vector to 'small element' vector, we
1552  // must demand a source element if any DemandedElt maps to it.
1553  if ((NumElts % NumSrcElts) == 0) {
1554  unsigned Scale = NumElts / NumSrcElts;
1555  for (unsigned i = 0; i != NumElts; ++i)
1556  if (DemandedElts[i])
1557  SrcDemandedElts.setBit(i / Scale);
1558 
1559  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1560  TLO, Depth + 1))
1561  return true;
1562 
1563  // Try calling SimplifyDemandedBits, converting demanded elts to the bits
1564  // of the large element.
1565  // TODO - bigendian once we have test coverage.
1566  if (TLO.DAG.getDataLayout().isLittleEndian()) {
1567  unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
1568  APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
1569  for (unsigned i = 0; i != NumElts; ++i)
1570  if (DemandedElts[i]) {
1571  unsigned Ofs = (i % Scale) * EltSizeInBits;
1572  SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
1573  }
1574 
1575  KnownBits Known;
1576  if (SimplifyDemandedBits(Src, SrcDemandedBits, Known, TLO, Depth + 1))
1577  return true;
1578  }
1579 
1580  // If the src element is zero/undef then all the output elements will be -
1581  // only demanded elements are guaranteed to be correct.
1582  for (unsigned i = 0; i != NumSrcElts; ++i) {
1583  if (SrcDemandedElts[i]) {
1584  if (SrcZero[i])
1585  KnownZero.setBits(i * Scale, (i + 1) * Scale);
1586  if (SrcUndef[i])
1587  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
1588  }
1589  }
1590  }
1591 
1592  // Bitcast from 'small element' src vector to 'large element' vector, we
1593  // demand all smaller source elements covered by the larger demanded element
1594  // of this vector.
1595  if ((NumSrcElts % NumElts) == 0) {
1596  unsigned Scale = NumSrcElts / NumElts;
1597  for (unsigned i = 0; i != NumElts; ++i)
1598  if (DemandedElts[i])
1599  SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
1600 
1601  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
1602  TLO, Depth + 1))
1603  return true;
1604 
1605  // If all the src elements covering an output element are zero/undef, then
1606  // the output element will be as well, assuming it was demanded.
1607  for (unsigned i = 0; i != NumElts; ++i) {
1608  if (DemandedElts[i]) {
1609  if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
1610  KnownZero.setBit(i);
1611  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
1612  KnownUndef.setBit(i);
1613  }
1614  }
1615  }
1616  break;
1617  }
1618  case ISD::BUILD_VECTOR: {
1619  // Check all elements and simplify any unused elements with UNDEF.
1620  if (!DemandedElts.isAllOnesValue()) {
1621  // Don't simplify BROADCASTS.
1622  if (llvm::any_of(Op->op_values(),
1623  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
1624  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
1625  bool Updated = false;
1626  for (unsigned i = 0; i != NumElts; ++i) {
1627  if (!DemandedElts[i] && !Ops[i].isUndef()) {
1628  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
1629  KnownUndef.setBit(i);
1630  Updated = true;
1631  }
1632  }
1633  if (Updated)
1634  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
1635  }
1636  }
1637  for (unsigned i = 0; i != NumElts; ++i) {
1638  SDValue SrcOp = Op.getOperand(i);
1639  if (SrcOp.isUndef()) {
1640  KnownUndef.setBit(i);
1641  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
1642  (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
1643  KnownZero.setBit(i);
1644  }
1645  }
1646  break;
1647  }
1648  case ISD::CONCAT_VECTORS: {
1649  EVT SubVT = Op.getOperand(0).getValueType();
1650  unsigned NumSubVecs = Op.getNumOperands();
1651  unsigned NumSubElts = SubVT.getVectorNumElements();
1652  for (unsigned i = 0; i != NumSubVecs; ++i) {
1653  SDValue SubOp = Op.getOperand(i);
1654  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1655  APInt SubUndef, SubZero;
1656  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
1657  Depth + 1))
1658  return true;
1659  KnownUndef.insertBits(SubUndef, i * NumSubElts);
1660  KnownZero.insertBits(SubZero, i * NumSubElts);
1661  }
1662  break;
1663  }
1664  case ISD::INSERT_SUBVECTOR: {
1665  if (!isa<ConstantSDNode>(Op.getOperand(2)))
1666  break;
1667  SDValue Base = Op.getOperand(0);
1668  SDValue Sub = Op.getOperand(1);
1669  EVT SubVT = Sub.getValueType();
1670  unsigned NumSubElts = SubVT.getVectorNumElements();
1671  const APInt& Idx = cast<ConstantSDNode>(Op.getOperand(2))->getAPIntValue();
1672  if (Idx.ugt(NumElts - NumSubElts))
1673  break;
1674  unsigned SubIdx = Idx.getZExtValue();
1675  APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
1676  APInt SubUndef, SubZero;
1677  if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
1678  Depth + 1))
1679  return true;
1680  APInt BaseElts = DemandedElts;
1681  BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
1682  if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
1683  Depth + 1))
1684  return true;
1685  KnownUndef.insertBits(SubUndef, SubIdx);
1686  KnownZero.insertBits(SubZero, SubIdx);
1687  break;
1688  }
1689  case ISD::EXTRACT_SUBVECTOR: {
1690  SDValue Src = Op.getOperand(0);
1692  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1693  if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
1694  // Offset the demanded elts by the subvector index.
1695  uint64_t Idx = SubIdx->getZExtValue();
1696  APInt SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1697  APInt SrcUndef, SrcZero;
1698  if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
1699  Depth + 1))
1700  return true;
1701  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
1702  KnownZero = SrcZero.extractBits(NumElts, Idx);
1703  }
1704  break;
1705  }
1706  case ISD::INSERT_VECTOR_ELT: {
1707  SDValue Vec = Op.getOperand(0);
1708  SDValue Scl = Op.getOperand(1);
1709  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1710 
1711  // For a legal, constant insertion index, if we don't need this insertion
1712  // then strip it, else remove it from the demanded elts.
1713  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
1714  unsigned Idx = CIdx->getZExtValue();
1715  if (!DemandedElts[Idx])
1716  return TLO.CombineTo(Op, Vec);
1717 
1718  APInt DemandedVecElts(DemandedElts);
1719  DemandedVecElts.clearBit(Idx);
1720  if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
1721  KnownZero, TLO, Depth + 1))
1722  return true;
1723 
1724  KnownUndef.clearBit(Idx);
1725  if (Scl.isUndef())
1726  KnownUndef.setBit(Idx);
1727 
1728  KnownZero.clearBit(Idx);
1729  if (isNullConstant(Scl) || isNullFPConstant(Scl))
1730  KnownZero.setBit(Idx);
1731  break;
1732  }
1733 
1734  APInt VecUndef, VecZero;
1735  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
1736  Depth + 1))
1737  return true;
1738  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
1739  break;
1740  }
1741  case ISD::VSELECT: {
1742  // Try to transform the select condition based on the current demanded
1743  // elements.
1744  // TODO: If a condition element is undef, we can choose from one arm of the
1745  // select (and if one arm is undef, then we can propagate that to the
1746  // result).
1747  // TODO - add support for constant vselect masks (see IR version of this).
1748  APInt UnusedUndef, UnusedZero;
1749  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
1750  UnusedZero, TLO, Depth + 1))
1751  return true;
1752 
1753  // See if we can simplify either vselect operand.
1754  APInt DemandedLHS(DemandedElts);
1755  APInt DemandedRHS(DemandedElts);
1756  APInt UndefLHS, ZeroLHS;
1757  APInt UndefRHS, ZeroRHS;
1758  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
1759  ZeroLHS, TLO, Depth + 1))
1760  return true;
1761  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
1762  ZeroRHS, TLO, Depth + 1))
1763  return true;
1764 
1765  KnownUndef = UndefLHS & UndefRHS;
1766  KnownZero = ZeroLHS & ZeroRHS;
1767  break;
1768  }
1769  case ISD::VECTOR_SHUFFLE: {
1770  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1771 
1772  // Collect demanded elements from shuffle operands..
1773  APInt DemandedLHS(NumElts, 0);
1774  APInt DemandedRHS(NumElts, 0);
1775  for (unsigned i = 0; i != NumElts; ++i) {
1776  int M = ShuffleMask[i];
1777  if (M < 0 || !DemandedElts[i])
1778  continue;
1779  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1780  if (M < (int)NumElts)
1781  DemandedLHS.setBit(M);
1782  else
1783  DemandedRHS.setBit(M - NumElts);
1784  }
1785 
1786  // See if we can simplify either shuffle operand.
1787  APInt UndefLHS, ZeroLHS;
1788  APInt UndefRHS, ZeroRHS;
1789  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
1790  ZeroLHS, TLO, Depth + 1))
1791  return true;
1792  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
1793  ZeroRHS, TLO, Depth + 1))
1794  return true;
1795 
1796  // Simplify mask using undef elements from LHS/RHS.
1797  bool Updated = false;
1798  bool IdentityLHS = true, IdentityRHS = true;
1799  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
1800  for (unsigned i = 0; i != NumElts; ++i) {
1801  int &M = NewMask[i];
1802  if (M < 0)
1803  continue;
1804  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
1805  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
1806  Updated = true;
1807  M = -1;
1808  }
1809  IdentityLHS &= (M < 0) || (M == (int)i);
1810  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
1811  }
1812 
1813  // Update legal shuffle masks based on demanded elements if it won't reduce
1814  // to Identity which can cause premature removal of the shuffle mask.
1815  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps &&
1816  isShuffleMaskLegal(NewMask, VT))
1817  return TLO.CombineTo(Op,
1818  TLO.DAG.getVectorShuffle(VT, DL, Op.getOperand(0),
1819  Op.getOperand(1), NewMask));
1820 
1821  // Propagate undef/zero elements from LHS/RHS.
1822  for (unsigned i = 0; i != NumElts; ++i) {
1823  int M = ShuffleMask[i];
1824  if (M < 0) {
1825  KnownUndef.setBit(i);
1826  } else if (M < (int)NumElts) {
1827  if (UndefLHS[M])
1828  KnownUndef.setBit(i);
1829  if (ZeroLHS[M])
1830  KnownZero.setBit(i);
1831  } else {
1832  if (UndefRHS[M - NumElts])
1833  KnownUndef.setBit(i);
1834  if (ZeroRHS[M - NumElts])
1835  KnownZero.setBit(i);
1836  }
1837  }
1838  break;
1839  }
1842  APInt SrcUndef, SrcZero;
1843  SDValue Src = Op.getOperand(0);
1844  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1845  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
1846  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef,
1847  SrcZero, TLO, Depth + 1))
1848  return true;
1849  KnownZero = SrcZero.zextOrTrunc(NumElts);
1850  KnownUndef = SrcUndef.zextOrTrunc(NumElts);
1851  break;
1852  }
1853  case ISD::OR:
1854  case ISD::XOR:
1855  case ISD::ADD:
1856  case ISD::SUB:
1857  case ISD::FADD:
1858  case ISD::FSUB:
1859  case ISD::FMUL:
1860  case ISD::FDIV:
1861  case ISD::FREM: {
1862  APInt SrcUndef, SrcZero;
1863  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
1864  SrcZero, TLO, Depth + 1))
1865  return true;
1866  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1867  KnownZero, TLO, Depth + 1))
1868  return true;
1869  KnownZero &= SrcZero;
1870  KnownUndef &= SrcUndef;
1871  break;
1872  }
1873  case ISD::AND: {
1874  APInt SrcUndef, SrcZero;
1875  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
1876  SrcZero, TLO, Depth + 1))
1877  return true;
1878  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1879  KnownZero, TLO, Depth + 1))
1880  return true;
1881 
1882  // If either side has a zero element, then the result element is zero, even
1883  // if the other is an UNDEF.
1884  KnownZero |= SrcZero;
1885  KnownUndef &= SrcUndef;
1886  KnownUndef &= ~KnownZero;
1887  break;
1888  }
1889  case ISD::TRUNCATE:
1890  case ISD::SIGN_EXTEND:
1891  case ISD::ZERO_EXTEND:
1892  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
1893  KnownZero, TLO, Depth + 1))
1894  return true;
1895  break;
1896  default: {
1897  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
1898  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
1899  KnownZero, TLO, Depth))
1900  return true;
1901  } else {
1902  KnownBits Known;
1903  APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
1904  if (SimplifyDemandedBits(Op, DemandedBits, DemandedEltMask, Known, TLO,
1905  Depth, AssumeSingleUse))
1906  return true;
1907  }
1908  break;
1909  }
1910  }
1911  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
1912 
1913  // Constant fold all undef cases.
1914  // TODO: Handle zero cases as well.
1915  if (DemandedElts.isSubsetOf(KnownUndef))
1916  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
1917 
1918  return false;
1919 }
1920 
1921 /// Determine which of the bits specified in Mask are known to be either zero or
1922 /// one and return them in the Known.
1924  KnownBits &Known,
1925  const APInt &DemandedElts,
1926  const SelectionDAG &DAG,
1927  unsigned Depth) const {
1931  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1932  "Should use MaskedValueIsZero if you don't know whether Op"
1933  " is a target node!");
1934  Known.resetAll();
1935 }
1936 
1938  KnownBits &Known,
1939  const APInt &DemandedElts,
1940  const SelectionDAG &DAG,
1941  unsigned Depth) const {
1942  assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex");
1943 
1944  if (unsigned Align = DAG.InferPtrAlignment(Op)) {
1945  // The low bits are known zero if the pointer is aligned.
1946  Known.Zero.setLowBits(Log2_32(Align));
1947  }
1948 }
1949 
1950 /// This method can be implemented by targets that want to expose additional
1951 /// information about sign bits to the DAG Combiner.
1953  const APInt &,
1954  const SelectionDAG &,
1955  unsigned Depth) const {
1959  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1960  "Should use ComputeNumSignBits if you don't know whether Op"
1961  " is a target node!");
1962  return 1;
1963 }
1964 
1966  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
1967  TargetLoweringOpt &TLO, unsigned Depth) const {
1971  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1972  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
1973  " is a target node!");
1974  return false;
1975 }
1976 
1978  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1979  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
1983  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1984  "Should use SimplifyDemandedBits if you don't know whether Op"
1985  " is a target node!");
1986  computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
1987  return false;
1988 }
1989 
1991  const SelectionDAG &DAG,
1992  bool SNaN,
1993  unsigned Depth) const {
1997  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
1998  "Should use isKnownNeverNaN if you don't know whether Op"
1999  " is a target node!");
2000  return false;
2001 }
2002 
2003 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
2004 // work with truncating build vectors and vectors with elements of less than
2005 // 8 bits.
2007  if (!N)
2008  return false;
2009 
2010  APInt CVal;
2011  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
2012  CVal = CN->getAPIntValue();
2013  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
2014  auto *CN = BV->getConstantSplatNode();
2015  if (!CN)
2016  return false;
2017 
2018  // If this is a truncating build vector, truncate the splat value.
2019  // Otherwise, we may fail to match the expected values below.
2020  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
2021  CVal = CN->getAPIntValue();
2022  if (BVEltWidth < CVal.getBitWidth())
2023  CVal = CVal.trunc(BVEltWidth);
2024  } else {
2025  return false;
2026  }
2027 
2028  switch (getBooleanContents(N->getValueType(0))) {
2030  return CVal[0];
2032  return CVal.isOneValue();
2034  return CVal.isAllOnesValue();
2035  }
2036 
2037  llvm_unreachable("Invalid boolean contents");
2038 }
2039 
2041  if (!N)
2042  return false;
2043 
2044  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
2045  if (!CN) {
2047  if (!BV)
2048  return false;
2049 
2050  // Only interested in constant splats, we don't care about undef
2051  // elements in identifying boolean constants and getConstantSplatNode
2052  // returns NULL if all ops are undef;
2053  CN = BV->getConstantSplatNode();
2054  if (!CN)
2055  return false;
2056  }
2057 
2059  return !CN->getAPIntValue()[0];
2060 
2061  return CN->isNullValue();
2062 }
2063 
2065  bool SExt) const {
2066  if (VT == MVT::i1)
2067  return N->isOne();
2068 
2070  switch (Cnt) {
2072  // An extended value of 1 is always true, unless its original type is i1,
2073  // in which case it will be sign extended to -1.
2074  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
2077  return N->isAllOnesValue() && SExt;
2078  }
2079  llvm_unreachable("Unexpected enumeration.");
2080 }
2081 
2082 /// This helper function of SimplifySetCC tries to optimize the comparison when
2083 /// either operand of the SetCC node is a bitwise-and instruction.
2084 SDValue TargetLowering::simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
2085  ISD::CondCode Cond,
2086  DAGCombinerInfo &DCI,
2087  const SDLoc &DL) const {
2088  // Match these patterns in any of their permutations:
2089  // (X & Y) == Y
2090  // (X & Y) != Y
2091  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
2092  std::swap(N0, N1);
2093 
2094  EVT OpVT = N0.getValueType();
2095  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
2096  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
2097  return SDValue();
2098 
2099  SDValue X, Y;
2100  if (N0.getOperand(0) == N1) {
2101  X = N0.getOperand(1);
2102  Y = N0.getOperand(0);
2103  } else if (N0.getOperand(1) == N1) {
2104  X = N0.getOperand(0);
2105  Y = N0.getOperand(1);
2106  } else {
2107  return SDValue();
2108  }
2109 
2110  SelectionDAG &DAG = DCI.DAG;
2111  SDValue Zero = DAG.getConstant(0, DL, OpVT);
2112  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
2113  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
2114  // Note that where Y is variable and is known to have at most one bit set
2115  // (for example, if it is Z & 1) we cannot do this; the expressions are not
2116  // equivalent when Y == 0.
2117  Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
2118  if (DCI.isBeforeLegalizeOps() ||
2119  isCondCodeLegal(Cond, N0.getSimpleValueType()))
2120  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
2121  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
2122  // If the target supports an 'and-not' or 'and-complement' logic operation,
2123  // try to use that to make a comparison operation more efficient.
2124  // But don't do this transform if the mask is a single bit because there are
2125  // more efficient ways to deal with that case (for example, 'bt' on x86 or
2126  // 'rlwinm' on PPC).
2127 
2128  // Bail out if the compare operand that we want to turn into a zero is
2129  // already a zero (otherwise, infinite loop).
2130  auto *YConst = dyn_cast<ConstantSDNode>(Y);
2131  if (YConst && YConst->isNullValue())
2132  return SDValue();
2133 
2134  // Transform this into: ~X & Y == 0.
2135  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
2136  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
2137  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
2138  }
2139 
2140  return SDValue();
2141 }
2142 
2143 /// There are multiple IR patterns that could be checking whether certain
2144 /// truncation of a signed number would be lossy or not. The pattern which is
2145 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
2146 /// We are looking for the following pattern: (KeptBits is a constant)
2147 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
2148 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
2149 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
2150 /// We will unfold it into the natural trunc+sext pattern:
2151 /// ((%x << C) a>> C) dstcond %x
2152 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
2153 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
2154  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
2155  const SDLoc &DL) const {
2156  // We must be comparing with a constant.
2157  ConstantSDNode *C1;
2158  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
2159  return SDValue();
2160 
2161  // N0 should be: add %x, (1 << (KeptBits-1))
2162  if (N0->getOpcode() != ISD::ADD)
2163  return SDValue();
2164 
2165  // And we must be 'add'ing a constant.
2166  ConstantSDNode *C01;
2167  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
2168  return SDValue();
2169 
2170  SDValue X = N0->getOperand(0);
2171  EVT XVT = X.getValueType();
2172 
2173  // Validate constants ...
2174 
2175  APInt I1 = C1->getAPIntValue();
2176 
2177  ISD::CondCode NewCond;
2178  if (Cond == ISD::CondCode::SETULT) {
2179  NewCond = ISD::CondCode::SETEQ;
2180  } else if (Cond == ISD::CondCode::SETULE) {
2181  NewCond = ISD::CondCode::SETEQ;
2182  // But need to 'canonicalize' the constant.
2183  I1 += 1;
2184  } else if (Cond == ISD::CondCode::SETUGT) {
2185  NewCond = ISD::CondCode::SETNE;
2186  // But need to 'canonicalize' the constant.
2187  I1 += 1;
2188  } else if (Cond == ISD::CondCode::SETUGE) {
2189  NewCond = ISD::CondCode::SETNE;
2190  } else
2191  return SDValue();
2192 
2193  APInt I01 = C01->getAPIntValue();
2194 
2195  auto checkConstants = [&I1, &I01]() -> bool {
2196  // Both of them must be power-of-two, and the constant from setcc is bigger.
2197  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
2198  };
2199 
2200  if (checkConstants()) {
2201  // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
2202  } else {
2203  // What if we invert constants? (and the target predicate)
2204  I1.negate();
2205  I01.negate();
2206  NewCond = getSetCCInverse(NewCond, /*isInteger=*/true);
2207  if (!checkConstants())
2208  return SDValue();
2209  // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
2210  }
2211 
2212  // They are power-of-two, so which bit is set?
2213  const unsigned KeptBits = I1.logBase2();
2214  const unsigned KeptBitsMinusOne = I01.logBase2();
2215 
2216  // Magic!
2217  if (KeptBits != (KeptBitsMinusOne + 1))
2218  return SDValue();
2219  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
2220 
2221  // We don't want to do this in every single case.
2222  SelectionDAG &DAG = DCI.DAG;
2223  if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
2224  XVT, KeptBits))
2225  return SDValue();
2226 
2227  const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
2228  assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
2229 
2230  // Unfold into: ((%x << C) a>> C) cond %x
2231  // Where 'cond' will be either 'eq' or 'ne'.
2232  SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
2233  SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
2234  SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
2235  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
2236 
2237  return T2;
2238 }
2239 
2240 /// Try to simplify a setcc built with the specified operands and cc. If it is
2241 /// unable to simplify it, return a null SDValue.
2243  ISD::CondCode Cond, bool foldBooleans,
2244  DAGCombinerInfo &DCI,
2245  const SDLoc &dl) const {
2246  SelectionDAG &DAG = DCI.DAG;
2247  EVT OpVT = N0.getValueType();
2248 
2249  // These setcc operations always fold.
2250  switch (Cond) {
2251  default: break;
2252  case ISD::SETFALSE:
2253  case ISD::SETFALSE2: return DAG.getBoolConstant(false, dl, VT, OpVT);
2254  case ISD::SETTRUE:
2255  case ISD::SETTRUE2: return DAG.getBoolConstant(true, dl, VT, OpVT);
2256  }
2257 
2258  // Ensure that the constant occurs on the RHS and fold constant comparisons.
2259  // TODO: Handle non-splat vector constants. All undef causes trouble.
2260  ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
2261  if (isConstOrConstSplat(N0) &&
2262  (DCI.isBeforeLegalizeOps() ||
2263  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
2264  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
2265 
2266  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
2267  const APInt &C1 = N1C->getAPIntValue();
2268 
2269  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
2270  // equality comparison, then we're just comparing whether X itself is
2271  // zero.
2272  if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
2273  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
2274  N0.getOperand(1).getOpcode() == ISD::Constant) {
2275  const APInt &ShAmt
2276  = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
2277  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2278  ShAmt == Log2_32(N0.getValueSizeInBits())) {
2279  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
2280  // (srl (ctlz x), 5) == 0 -> X != 0
2281  // (srl (ctlz x), 5) != 1 -> X != 0
2282  Cond = ISD::SETNE;
2283  } else {
2284  // (srl (ctlz x), 5) != 0 -> X == 0
2285  // (srl (ctlz x), 5) == 1 -> X == 0
2286  Cond = ISD::SETEQ;
2287  }
2288  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
2289  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
2290  Zero, Cond);
2291  }
2292  }
2293 
2294  SDValue CTPOP = N0;
2295  // Look through truncs that don't change the value of a ctpop.
2296  if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
2297  CTPOP = N0.getOperand(0);
2298 
2299  if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
2300  (N0 == CTPOP ||
2302  EVT CTVT = CTPOP.getValueType();
2303  SDValue CTOp = CTPOP.getOperand(0);
2304 
2305  // (ctpop x) u< 2 -> (x & x-1) == 0
2306  // (ctpop x) u> 1 -> (x & x-1) != 0
2307  if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
2308  SDValue Sub = DAG.getNode(ISD::SUB, dl, CTVT, CTOp,
2309  DAG.getConstant(1, dl, CTVT));
2310  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Sub);
2312  return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
2313  }
2314 
2315  // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal.
2316  }
2317 
2318  // (zext x) == C --> x == (trunc C)
2319  // (sext x) == C --> x == (trunc C)
2320  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2321  DCI.isBeforeLegalize() && N0->hasOneUse()) {
2322  unsigned MinBits = N0.getValueSizeInBits();
2323  SDValue PreExt;
2324  bool Signed = false;
2325  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
2326  // ZExt
2327  MinBits = N0->getOperand(0).getValueSizeInBits();
2328  PreExt = N0->getOperand(0);
2329  } else if (N0->getOpcode() == ISD::AND) {
2330  // DAGCombine turns costly ZExts into ANDs
2331  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
2332  if ((C->getAPIntValue()+1).isPowerOf2()) {
2333  MinBits = C->getAPIntValue().countTrailingOnes();
2334  PreExt = N0->getOperand(0);
2335  }
2336  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
2337  // SExt
2338  MinBits = N0->getOperand(0).getValueSizeInBits();
2339  PreExt = N0->getOperand(0);
2340  Signed = true;
2341  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
2342  // ZEXTLOAD / SEXTLOAD
2343  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
2344  MinBits = LN0->getMemoryVT().getSizeInBits();
2345  PreExt = N0;
2346  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
2347  Signed = true;
2348  MinBits = LN0->getMemoryVT().getSizeInBits();
2349  PreExt = N0;
2350  }
2351  }
2352 
2353  // Figure out how many bits we need to preserve this constant.
2354  unsigned ReqdBits = Signed ?
2355  C1.getBitWidth() - C1.getNumSignBits() + 1 :
2356  C1.getActiveBits();
2357 
2358  // Make sure we're not losing bits from the constant.
2359  if (MinBits > 0 &&
2360  MinBits < C1.getBitWidth() &&
2361  MinBits >= ReqdBits) {
2362  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
2363  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
2364  // Will get folded away.
2365  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
2366  if (MinBits == 1 && C1 == 1)
2367  // Invert the condition.
2368  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
2369  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2370  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
2371  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
2372  }
2373 
2374  // If truncating the setcc operands is not desirable, we can still
2375  // simplify the expression in some cases:
2376  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
2377  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
2378  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
2379  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
2380  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
2381  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
2382  SDValue TopSetCC = N0->getOperand(0);
2383  unsigned N0Opc = N0->getOpcode();
2384  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
2385  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
2386  TopSetCC.getOpcode() == ISD::SETCC &&
2387  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
2388  (isConstFalseVal(N1C) ||
2389  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
2390 
2391  bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
2392  (!N1C->isNullValue() && Cond == ISD::SETNE);
2393 
2394  if (!Inverse)
2395  return TopSetCC;
2396 
2398  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
2399  TopSetCC.getOperand(0).getValueType().isInteger());
2400  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
2401  TopSetCC.getOperand(1),
2402  InvCond);
2403  }
2404  }
2405  }
2406 
2407  // If the LHS is '(and load, const)', the RHS is 0, the test is for
2408  // equality or unsigned, and all 1 bits of the const are in the same
2409  // partial word, see if we can shorten the load.
2410  if (DCI.isBeforeLegalize() &&
2411  !ISD::isSignedIntSetCC(Cond) &&
2412  N0.getOpcode() == ISD::AND && C1 == 0 &&
2413  N0.getNode()->hasOneUse() &&
2414  isa<LoadSDNode>(N0.getOperand(0)) &&
2415  N0.getOperand(0).getNode()->hasOneUse() &&
2416  isa<ConstantSDNode>(N0.getOperand(1))) {
2417  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
2418  APInt bestMask;
2419  unsigned bestWidth = 0, bestOffset = 0;
2420  if (!Lod->isVolatile() && Lod->isUnindexed()) {
2421  unsigned origWidth = N0.getValueSizeInBits();
2422  unsigned maskWidth = origWidth;
2423  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
2424  // 8 bits, but have to be careful...
2425  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
2426  origWidth = Lod->getMemoryVT().getSizeInBits();
2427  const APInt &Mask =
2428  cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
2429  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
2430  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
2431  for (unsigned offset=0; offset<origWidth/width; offset++) {
2432  if (Mask.isSubsetOf(newMask)) {
2433  if (DAG.getDataLayout().isLittleEndian())
2434  bestOffset = (uint64_t)offset * (width/8);
2435  else
2436  bestOffset = (origWidth/width - offset - 1) * (width/8);
2437  bestMask = Mask.lshr(offset * (width/8) * 8);
2438  bestWidth = width;
2439  break;
2440  }
2441  newMask <<= width;
2442  }
2443  }
2444  }
2445  if (bestWidth) {
2446  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
2447  if (newVT.isRound() &&
2448  shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
2449  EVT PtrType = Lod->getOperand(1).getValueType();
2450  SDValue Ptr = Lod->getBasePtr();
2451  if (bestOffset != 0)
2452  Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
2453  DAG.getConstant(bestOffset, dl, PtrType));
2454  unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
2455  SDValue NewLoad = DAG.getLoad(
2456  newVT, dl, Lod->getChain(), Ptr,
2457  Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
2458  return DAG.getSetCC(dl, VT,
2459  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
2460  DAG.getConstant(bestMask.trunc(bestWidth),
2461  dl, newVT)),
2462  DAG.getConstant(0LL, dl, newVT), Cond);
2463  }
2464  }
2465  }
2466 
2467  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
2468  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
2469  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
2470 
2471  // If the comparison constant has bits in the upper part, the
2472  // zero-extended value could never match.
2473  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
2474  C1.getBitWidth() - InSize))) {
2475  switch (Cond) {
2476  case ISD::SETUGT:
2477  case ISD::SETUGE:
2478  case ISD::SETEQ:
2479  return DAG.getConstant(0, dl, VT);
2480  case ISD::SETULT:
2481  case ISD::SETULE:
2482  case ISD::SETNE:
2483  return DAG.getConstant(1, dl, VT);
2484  case ISD::SETGT:
2485  case ISD::SETGE:
2486  // True if the sign bit of C1 is set.
2487  return DAG.getConstant(C1.isNegative(), dl, VT);
2488  case ISD::SETLT:
2489  case ISD::SETLE:
2490  // True if the sign bit of C1 isn't set.
2491  return DAG.getConstant(C1.isNonNegative(), dl, VT);
2492  default:
2493  break;
2494  }
2495  }
2496 
2497  // Otherwise, we can perform the comparison with the low bits.
2498  switch (Cond) {
2499  case ISD::SETEQ:
2500  case ISD::SETNE:
2501  case ISD::SETUGT:
2502  case ISD::SETUGE:
2503  case ISD::SETULT:
2504  case ISD::SETULE: {
2505  EVT newVT = N0.getOperand(0).getValueType();
2506  if (DCI.isBeforeLegalizeOps() ||
2507  (isOperationLegal(ISD::SETCC, newVT) &&
2508  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
2509  EVT NewSetCCVT =
2510  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), newVT);
2511  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
2512 
2513  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
2514  NewConst, Cond);
2515  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
2516  }
2517  break;
2518  }
2519  default:
2520  break; // todo, be more careful with signed comparisons
2521  }
2522  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2523  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2524  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
2525  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
2526  EVT ExtDstTy = N0.getValueType();
2527  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
2528 
2529  // If the constant doesn't fit into the number of bits for the source of
2530  // the sign extension, it is impossible for both sides to be equal.
2531  if (C1.getMinSignedBits() > ExtSrcTyBits)
2532  return DAG.getConstant(Cond == ISD::SETNE, dl, VT);
2533 
2534  SDValue ZextOp;
2535  EVT Op0Ty = N0.getOperand(0).getValueType();
2536  if (Op0Ty == ExtSrcTy) {
2537  ZextOp = N0.getOperand(0);
2538  } else {
2539  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
2540  ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
2541  DAG.getConstant(Imm, dl, Op0Ty));
2542  }
2543  if (!DCI.isCalledByLegalizer())
2544  DCI.AddToWorklist(ZextOp.getNode());
2545  // Otherwise, make this a use of a zext.
2546  return DAG.getSetCC(dl, VT, ZextOp,
2548  ExtDstTyBits,
2549  ExtSrcTyBits),
2550  dl, ExtDstTy),
2551  Cond);
2552  } else if ((N1C->isNullValue() || N1C->isOne()) &&
2553  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
2554  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
2555  if (N0.getOpcode() == ISD::SETCC &&
2556  isTypeLegal(VT) && VT.bitsLE(N0.getValueType())) {
2557  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
2558  if (TrueWhenTrue)
2559  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
2560  // Invert the condition.
2561  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
2562  CC = ISD::getSetCCInverse(CC,
2563  N0.getOperand(0).getValueType().isInteger());
2564  if (DCI.isBeforeLegalizeOps() ||
2565  isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
2566  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
2567  }
2568 
2569  if ((N0.getOpcode() == ISD::XOR ||
2570  (N0.getOpcode() == ISD::AND &&
2571  N0.getOperand(0).getOpcode() == ISD::XOR &&
2572  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
2573  isa<ConstantSDNode>(N0.getOperand(1)) &&
2574  cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
2575  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
2576  // can only do this if the top bits are known zero.
2577  unsigned BitWidth = N0.getValueSizeInBits();
2578  if (DAG.MaskedValueIsZero(N0,
2579  APInt::getHighBitsSet(BitWidth,
2580  BitWidth-1))) {
2581  // Okay, get the un-inverted input value.
2582  SDValue Val;
2583  if (N0.getOpcode() == ISD::XOR) {
2584  Val = N0.getOperand(0);
2585  } else {
2586  assert(N0.getOpcode() == ISD::AND &&
2587  N0.getOperand(0).getOpcode() == ISD::XOR);
2588  // ((X^1)&1)^1 -> X & 1
2589  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
2590  N0.getOperand(0).getOperand(0),
2591  N0.getOperand(1));
2592  }
2593 
2594  return DAG.getSetCC(dl, VT, Val, N1,
2595  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2596  }
2597  } else if (N1C->isOne() &&
2598  (VT == MVT::i1 ||
2599  getBooleanContents(N0->getValueType(0)) ==
2601  SDValue Op0 = N0;
2602  if (Op0.getOpcode() == ISD::TRUNCATE)
2603  Op0 = Op0.getOperand(0);
2604 
2605  if ((Op0.getOpcode() == ISD::XOR) &&
2606  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
2607  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
2608  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
2609  Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
2610  return DAG.getSetCC(dl, VT, Op0.getOperand(0), Op0.getOperand(1),
2611  Cond);
2612  }
2613  if (Op0.getOpcode() == ISD::AND &&
2614  isa<ConstantSDNode>(Op0.getOperand(1)) &&
2615  cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
2616  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
2617  if (Op0.getValueType().bitsGT(VT))
2618  Op0 = DAG.getNode(ISD::AND, dl, VT,
2619  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
2620  DAG.getConstant(1, dl, VT));
2621  else if (Op0.getValueType().bitsLT(VT))
2622  Op0 = DAG.getNode(ISD::AND, dl, VT,
2623  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
2624  DAG.getConstant(1, dl, VT));
2625 
2626  return DAG.getSetCC(dl, VT, Op0,
2627  DAG.getConstant(0, dl, Op0.getValueType()),
2628  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2629  }
2630  if (Op0.getOpcode() == ISD::AssertZext &&
2631  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
2632  return DAG.getSetCC(dl, VT, Op0,
2633  DAG.getConstant(0, dl, Op0.getValueType()),
2634  Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
2635  }
2636  }
2637 
2638  if (SDValue V =
2639  optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
2640  return V;
2641  }
2642 
2643  // These simplifications apply to splat vectors as well.
2644  // TODO: Handle more splat vector cases.
2645  if (auto *N1C = isConstOrConstSplat(N1)) {
2646  const APInt &C1 = N1C->getAPIntValue();
2647 
2648  APInt MinVal, MaxVal;
2649  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
2650  if (ISD::isSignedIntSetCC(Cond)) {
2651  MinVal = APInt::getSignedMinValue(OperandBitSize);
2652  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
2653  } else {
2654  MinVal = APInt::getMinValue(OperandBitSize);
2655  MaxVal = APInt::getMaxValue(OperandBitSize);
2656  }
2657 
2658  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
2659  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
2660  // X >= MIN --> true
2661  if (C1 == MinVal)
2662  return DAG.getBoolConstant(true, dl, VT, OpVT);
2663 
2664  if (!VT.isVector()) { // TODO: Support this for vectors.
2665  // X >= C0 --> X > (C0 - 1)
2666  APInt C = C1 - 1;
2667  ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
2668  if ((DCI.isBeforeLegalizeOps() ||
2669  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2670  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2672  return DAG.getSetCC(dl, VT, N0,
2673  DAG.getConstant(C, dl, N1.getValueType()),
2674  NewCC);
2675  }
2676  }
2677  }
2678 
2679  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
2680  // X <= MAX --> true
2681  if (C1 == MaxVal)
2682  return DAG.getBoolConstant(true, dl, VT, OpVT);
2683 
2684  // X <= C0 --> X < (C0 + 1)
2685  if (!VT.isVector()) { // TODO: Support this for vectors.
2686  APInt C = C1 + 1;
2687  ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
2688  if ((DCI.isBeforeLegalizeOps() ||
2689  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
2690  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
2692  return DAG.getSetCC(dl, VT, N0,
2693  DAG.getConstant(C, dl, N1.getValueType()),
2694  NewCC);
2695  }
2696  }
2697  }
2698 
2699  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
2700  if (C1 == MinVal)
2701  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
2702 
2703  // TODO: Support this for vectors after legalize ops.
2704  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2705  // Canonicalize setlt X, Max --> setne X, Max
2706  if (C1 == MaxVal)
2707  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2708 
2709  // If we have setult X, 1, turn it into seteq X, 0
2710  if (C1 == MinVal+1)
2711  return DAG.getSetCC(dl, VT, N0,
2712  DAG.getConstant(MinVal, dl, N0.getValueType()),
2713  ISD::SETEQ);
2714  }
2715  }
2716 
2717  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
2718  if (C1 == MaxVal)
2719  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
2720 
2721  // TODO: Support this for vectors after legalize ops.
2722  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2723  // Canonicalize setgt X, Min --> setne X, Min
2724  if (C1 == MinVal)
2725  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
2726 
2727  // If we have setugt X, Max-1, turn it into seteq X, Max
2728  if (C1 == MaxVal-1)
2729  return DAG.getSetCC(dl, VT, N0,
2730  DAG.getConstant(MaxVal, dl, N0.getValueType()),
2731  ISD::SETEQ);
2732  }
2733  }
2734 
2735  // If we have "setcc X, C0", check to see if we can shrink the immediate
2736  // by changing cc.
2737  // TODO: Support this for vectors after legalize ops.
2738  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
2739  // SETUGT X, SINTMAX -> SETLT X, 0
2740  if (Cond == ISD::SETUGT &&
2741  C1 == APInt::getSignedMaxValue(OperandBitSize))
2742  return DAG.getSetCC(dl, VT, N0,
2743  DAG.getConstant(0, dl, N1.getValueType()),
2744  ISD::SETLT);
2745 
2746  // SETULT X, SINTMIN -> SETGT X, -1
2747  if (Cond == ISD::SETULT &&
2748  C1 == APInt::getSignedMinValue(OperandBitSize)) {
2749  SDValue ConstMinusOne =
2750  DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
2751  N1.getValueType());
2752  return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
2753  }
2754  }
2755  }
2756 
2757  // Back to non-vector simplifications.
2758  // TODO: Can we do these for vector splats?
2759  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
2760  const APInt &C1 = N1C->getAPIntValue();
2761 
2762  // Fold bit comparisons when we can.
2763  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2764  (VT == N0.getValueType() ||
2765  (isTypeLegal(VT) && VT.bitsLE(N0.getValueType()))) &&
2766  N0.getOpcode() == ISD::AND) {
2767  auto &DL = DAG.getDataLayout();
2768  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2769  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2770  !DCI.isBeforeLegalize());
2771  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
2772  // Perform the xform if the AND RHS is a single bit.
2773  if (AndRHS->getAPIntValue().isPowerOf2()) {
2774  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2775  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2776  DAG.getConstant(AndRHS->getAPIntValue().logBase2(), dl,
2777  ShiftTy)));
2778  }
2779  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
2780  // (X & 8) == 8 --> (X & 8) >> 3
2781  // Perform the xform if C1 is a single bit.
2782  if (C1.isPowerOf2()) {
2783  return DAG.getNode(ISD::TRUNCATE, dl, VT,
2784  DAG.getNode(ISD::SRL, dl, N0.getValueType(), N0,
2785  DAG.getConstant(C1.logBase2(), dl,
2786  ShiftTy)));
2787  }
2788  }
2789  }
2790  }
2791 
2792  if (C1.getMinSignedBits() <= 64 &&
2794  // (X & -256) == 256 -> (X >> 8) == 1
2795  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2796  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
2797  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2798  const APInt &AndRHSC = AndRHS->getAPIntValue();
2799  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
2800  unsigned ShiftBits = AndRHSC.countTrailingZeros();
2801  auto &DL = DAG.getDataLayout();
2802  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2803  !DCI.isBeforeLegalize());
2804  EVT CmpTy = N0.getValueType();
2805  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0.getOperand(0),
2806  DAG.getConstant(ShiftBits, dl,
2807  ShiftTy));
2808  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, CmpTy);
2809  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
2810  }
2811  }
2812  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
2813  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
2814  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
2815  // X < 0x100000000 -> (X >> 32) < 1
2816  // X >= 0x100000000 -> (X >> 32) >= 1
2817  // X <= 0x0ffffffff -> (X >> 32) < 1
2818  // X > 0x0ffffffff -> (X >> 32) >= 1
2819  unsigned ShiftBits;
2820  APInt NewC = C1;
2821  ISD::CondCode NewCond = Cond;
2822  if (AdjOne) {
2823  ShiftBits = C1.countTrailingOnes();
2824  NewC = NewC + 1;
2825  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2826  } else {
2827  ShiftBits = C1.countTrailingZeros();
2828  }
2829  NewC.lshrInPlace(ShiftBits);
2830  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
2832  auto &DL = DAG.getDataLayout();
2833  EVT ShiftTy = getShiftAmountTy(N0.getValueType(), DL,
2834  !DCI.isBeforeLegalize());
2835  EVT CmpTy = N0.getValueType();
2836  SDValue Shift = DAG.getNode(ISD::SRL, dl, CmpTy, N0,
2837  DAG.getConstant(ShiftBits, dl, ShiftTy));
2838  SDValue CmpRHS = DAG.getConstant(NewC, dl, CmpTy);
2839  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
2840  }
2841  }
2842  }
2843  }
2844 
2845  if (isa<ConstantFPSDNode>(N0.getNode())) {
2846  // Constant fold or commute setcc.
2847  SDValue O = DAG.FoldSetCC(VT, N0, N1, Cond, dl);
2848  if (O.getNode()) return O;
2849  } else if (auto *CFP = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
2850  // If the RHS of an FP comparison is a constant, simplify it away in
2851  // some cases.
2852  if (CFP->getValueAPF().isNaN()) {
2853  // If an operand is known to be a nan, we can fold it.
2854  switch (ISD::getUnorderedFlavor(Cond)) {
2855  default: llvm_unreachable("Unknown flavor!");
2856  case 0: // Known false.
2857  return DAG.getBoolConstant(false, dl, VT, OpVT);
2858  case 1: // Known true.
2859  return DAG.getBoolConstant(true, dl, VT, OpVT);
2860  case 2: // Undefined.
2861  return DAG.getUNDEF(VT);
2862  }
2863  }
2864 
2865  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
2866  // constant if knowing that the operand is non-nan is enough. We prefer to
2867  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
2868  // materialize 0.0.
2869  if (Cond == ISD::SETO || Cond == ISD::SETUO)
2870  return DAG.getSetCC(dl, VT, N0, N0, Cond);
2871 
2872  // setcc (fneg x), C -> setcc swap(pred) x, -C
2873  if (N0.getOpcode() == ISD::FNEG) {
2875  if (DCI.isBeforeLegalizeOps() ||
2876  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
2877  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
2878  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
2879  }
2880  }
2881 
2882  // If the condition is not legal, see if we can find an equivalent one
2883  // which is legal.
2884  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
2885  // If the comparison was an awkward floating-point == or != and one of
2886  // the comparison operands is infinity or negative infinity, convert the
2887  // condition to a less-awkward <= or >=.
2888  if (CFP->getValueAPF().isInfinity()) {
2889  if (CFP->getValueAPF().isNegative()) {
2890  if (Cond == ISD::SETOEQ &&
2892  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
2893  if (Cond == ISD::SETUEQ &&
2895  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
2896  if (Cond == ISD::SETUNE &&
2898  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
2899  if (Cond == ISD::SETONE &&
2901  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
2902  } else {
2903  if (Cond == ISD::SETOEQ &&
2905  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
2906  if (Cond == ISD::SETUEQ &&
2908  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
2909  if (Cond == ISD::SETUNE &&
2911  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
2912  if (Cond == ISD::SETONE &&
2914  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
2915  }
2916  }
2917  }
2918  }
2919 
2920  if (N0 == N1) {
2921  // The sext(setcc()) => setcc() optimization relies on the appropriate
2922  // constant being emitted.
2923 
2924  bool EqTrue = ISD::isTrueWhenEqual(Cond);
2925 
2926  // We can always fold X == X for integer setcc's.
2927  if (N0.getValueType().isInteger())
2928  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2929 
2930  unsigned UOF = ISD::getUnorderedFlavor(Cond);
2931  if (UOF == 2) // FP operators that are undefined on NaNs.
2932  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2933  if (UOF == unsigned(EqTrue))
2934  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
2935  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
2936  // if it is not already.
2937  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
2938  if (NewCond != Cond &&
2939  (DCI.isBeforeLegalizeOps() ||
2940  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
2941  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
2942  }
2943 
2944  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
2945  N0.getValueType().isInteger()) {
2946  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
2947  N0.getOpcode() == ISD::XOR) {
2948  // Simplify (X+Y) == (X+Z) --> Y == Z
2949  if (N0.getOpcode() == N1.getOpcode()) {
2950  if (N0.getOperand(0) == N1.getOperand(0))
2951  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
2952  if (N0.getOperand(1) == N1.getOperand(1))
2953  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
2954  if (isCommutativeBinOp(N0.getOpcode())) {
2955  // If X op Y == Y op X, try other combinations.
2956  if (N0.getOperand(0) == N1.getOperand(1))
2957  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
2958  Cond);
2959  if (N0.getOperand(1) == N1.getOperand(0))
2960  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
2961  Cond);
2962  }
2963  }
2964 
2965  // If RHS is a legal immediate value for a compare instruction, we need
2966  // to be careful about increasing register pressure needlessly.
2967  bool LegalRHSImm = false;
2968 
2969  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
2970  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2971  // Turn (X+C1) == C2 --> X == C2-C1
2972  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
2973  return DAG.getSetCC(dl, VT, N0.getOperand(0),
2974  DAG.getConstant(RHSC->getAPIntValue()-
2975  LHSR->getAPIntValue(),
2976  dl, N0.getValueType()), Cond);
2977  }
2978 
2979  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
2980  if (N0.getOpcode() == ISD::XOR)
2981  // If we know that all of the inverted bits are zero, don't bother
2982  // performing the inversion.
2983  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
2984  return
2985  DAG.getSetCC(dl, VT, N0.getOperand(0),
2986  DAG.getConstant(LHSR->getAPIntValue() ^
2987  RHSC->getAPIntValue(),
2988  dl, N0.getValueType()),
2989  Cond);
2990  }
2991 
2992  // Turn (C1-X) == C2 --> X == C1-C2
2993  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
2994  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
2995  return
2996  DAG.getSetCC(dl, VT, N0.getOperand(1),
2997  DAG.getConstant(SUBC->getAPIntValue() -
2998  RHSC->getAPIntValue(),
2999  dl, N0.getValueType()),
3000  Cond);
3001  }
3002  }
3003 
3004  // Could RHSC fold directly into a compare?
3005  if (RHSC->getValueType(0).getSizeInBits() <= 64)
3006  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
3007  }
3008 
3009  // Simplify (X+Z) == X --> Z == 0
3010  // Don't do this if X is an immediate that can fold into a cmp
3011  // instruction and X+Z has other uses. It could be an induction variable
3012  // chain, and the transform would increase register pressure.
3013  if (!LegalRHSImm || N0.getNode()->hasOneUse()) {
3014  if (N0.getOperand(0) == N1)
3015  return DAG.getSetCC(dl, VT, N0.getOperand(1),
3016  DAG.getConstant(0, dl, N0.getValueType()), Cond);
3017  if (N0.getOperand(1) == N1) {
3018  if (isCommutativeBinOp(N0.getOpcode()))
3019  return DAG.getSetCC(dl, VT, N0.getOperand(0),
3020  DAG.getConstant(0, dl, N0.getValueType()),
3021  Cond);
3022  if (N0.getNode()->hasOneUse()) {
3023  assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
3024  auto &DL = DAG.getDataLayout();
3025  // (Z-X) == X --> Z == X<<1
3026  SDValue SH = DAG.getNode(
3027  ISD::SHL, dl, N1.getValueType(), N1,
3028  DAG.getConstant(1, dl,
3029  getShiftAmountTy(N1.getValueType(), DL,
3030  !DCI.isBeforeLegalize())));
3031  if (!DCI.isCalledByLegalizer())
3032  DCI.AddToWorklist(SH.getNode());
3033  return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond);
3034  }
3035  }
3036  }
3037  }
3038 
3039  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
3040  N1.getOpcode() == ISD::XOR) {
3041  // Simplify X == (X+Z) --> Z == 0
3042  if (N1.getOperand(0) == N0)
3043  return DAG.getSetCC(dl, VT, N1.getOperand(1),
3044  DAG.getConstant(0, dl, N1.getValueType()), Cond);
3045  if (N1.getOperand(1) == N0) {
3046  if (isCommutativeBinOp(N1.getOpcode()))
3047  return DAG.getSetCC(dl, VT, N1.getOperand(0),
3048  DAG.getConstant(0, dl, N1.getValueType()), Cond);
3049  if (N1.getNode()->hasOneUse()) {
3050  assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
3051  auto &DL = DAG.getDataLayout();
3052  // X == (Z-X) --> X<<1 == Z
3053  SDValue SH = DAG.getNode(
3054  ISD::SHL, dl, N1.getValueType(), N0,
3055  DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL,
3056  !DCI.isBeforeLegalize())));
3057  if (!DCI.isCalledByLegalizer())
3058  DCI.AddToWorklist(SH.getNode());
3059  return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond);
3060  }
3061  }
3062  }
3063 
3064  if (SDValue V = simplifySetCCWithAnd(VT, N0, N1, Cond, DCI, dl))
3065  return V;
3066  }
3067 
3068  // Fold away ALL boolean setcc's.
3069  SDValue Temp;
3070  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
3071  EVT OpVT = N0.getValueType();
3072  switch (Cond) {
3073  default: llvm_unreachable("Unknown integer setcc!");
3074  case ISD::SETEQ: // X == Y -> ~(X^Y)
3075  Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
3076  N0 = DAG.getNOT(dl, Temp, OpVT);
3077  if (!DCI.isCalledByLegalizer())
3078  DCI.AddToWorklist(Temp.getNode());
3079  break;
3080  case ISD::SETNE: // X != Y --> (X^Y)
3081  N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
3082  break;
3083  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
3084  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
3085  Temp = DAG.getNOT(dl, N0, OpVT);
3086  N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
3087  if (!DCI.isCalledByLegalizer())
3088  DCI.AddToWorklist(Temp.getNode());
3089  break;
3090  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
3091  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
3092  Temp = DAG.getNOT(dl, N1, OpVT);
3093  N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
3094  if (!DCI.isCalledByLegalizer())
3095  DCI.AddToWorklist(Temp.getNode());
3096  break;
3097  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
3098  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
3099  Temp = DAG.getNOT(dl, N0, OpVT);
3100  N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
3101  if (!DCI.isCalledByLegalizer())
3102  DCI.AddToWorklist(Temp.getNode());
3103  break;
3104  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
3105  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
3106  Temp = DAG.getNOT(dl, N1, OpVT);
3107  N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
3108  break;
3109  }
3110  if (VT.getScalarType() != MVT::i1) {
3111  if (!DCI.isCalledByLegalizer())
3112  DCI.AddToWorklist(N0.getNode());
3113  // FIXME: If running after legalize, we probably can't do this.
3115  N0 = DAG.getNode(ExtendCode, dl, VT, N0);
3116  }
3117  return N0;
3118  }
3119 
3120  // Could not fold it.
3121  return SDValue();
3122 }
3123 
3124 /// Returns true (and the GlobalValue and the offset) if the node is a
3125 /// GlobalAddress + offset.
3127  int64_t &Offset) const {
3128 
3129  SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
3130 
3131  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
3132  GA = GASD->getGlobal();
3133  Offset += GASD->getOffset();
3134  return true;
3135  }
3136 
3137  if (N->getOpcode() == ISD::ADD) {
3138  SDValue N1 = N->getOperand(0);
3139  SDValue N2 = N->getOperand(1);
3140  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
3141  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
3142  Offset += V->getSExtValue();
3143  return true;
3144  }
3145  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
3146  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
3147  Offset += V->getSExtValue();
3148  return true;
3149  }
3150  }
3151  }
3152 
3153  return false;
3154 }
3155 
3157  DAGCombinerInfo &DCI) const {
3158  // Default implementation: no optimization.
3159  return SDValue();
3160 }
3161 
3162 //===----------------------------------------------------------------------===//
3163 // Inline Assembler Implementation Methods
3164 //===----------------------------------------------------------------------===//
3165 
3168  unsigned S = Constraint.size();
3169 
3170  if (S == 1) {
3171  switch (Constraint[0]) {
3172  default: break;
3173  case 'r': return C_RegisterClass;
3174  case 'm': // memory
3175  case 'o': // offsetable
3176  case 'V': // not offsetable
3177  return C_Memory;
3178  case 'i': // Simple Integer or Relocatable Constant
3179  case 'n': // Simple Integer
3180  case 'E': // Floating Point Constant
3181  case 'F': // Floating Point Constant
3182  case 's': // Relocatable Constant
3183  case 'p': // Address.
3184  case 'X': // Allow ANY value.
3185  case 'I': // Target registers.
3186  case 'J':
3187  case 'K':
3188  case 'L':
3189  case 'M':
3190  case 'N':
3191  case 'O':
3192  case 'P':
3193  case '<':
3194  case '>':
3195  return C_Other;
3196  }
3197  }
3198 
3199  if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') {
3200  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
3201  return C_Memory;
3202  return C_Register;
3203  }
3204  return C_Unknown;
3205 }
3206 
3207 /// Try to replace an X constraint, which matches anything, with another that
3208 /// has more specific requirements based on the type of the corresponding
3209 /// operand.
3210 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const{
3211  if (ConstraintVT.isInteger())
3212  return "r";
3213  if (ConstraintVT.isFloatingPoint())
3214  return "f"; // works for many targets
3215  return nullptr;
3216 }
3217 
3218 /// Lower the specified operand into the Ops vector.
3219 /// If it is invalid, don't add anything to Ops.
3221  std::string &Constraint,
3222  std::vector<SDValue> &Ops,
3223  SelectionDAG &DAG) const {
3224 
3225  if (Constraint.length() > 1) return;
3226 
3227  char ConstraintLetter = Constraint[0];
3228  switch (ConstraintLetter) {
3229  default: break;
3230  case 'X': // Allows any operand; labels (basic block) use this.
3231  if (Op.getOpcode() == ISD::BasicBlock) {
3232  Ops.push_back(Op);
3233  return;
3234  }
3236  case 'i': // Simple Integer or Relocatable Constant
3237  case 'n': // Simple Integer
3238  case 's': { // Relocatable Constant
3239  // These operands are interested in values of the form (GV+C), where C may
3240  // be folded in as an offset of GV, or it may be explicitly added. Also, it
3241  // is possible and fine if either GV or C are missing.
3244 
3245  // If we have "(add GV, C)", pull out GV/C
3246  if (Op.getOpcode() == ISD::ADD) {
3247  C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3249  if (!C || !GA) {
3250  C = dyn_cast<ConstantSDNode>(Op.getOperand(0));
3252  }
3253  if (!C || !GA) {
3254  C = nullptr;
3255  GA = nullptr;
3256  }
3257  }
3258 
3259  // If we find a valid operand, map to the TargetXXX version so that the
3260  // value itself doesn't get selected.
3261  if (GA) { // Either &GV or &GV+C
3262  if (ConstraintLetter != 'n') {
3263  int64_t Offs = GA->getOffset();
3264  if (C) Offs += C->getZExtValue();
3265  Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(),
3266  C ? SDLoc(C) : SDLoc(),
3267  Op.getValueType(), Offs));
3268  }
3269  return;
3270  }
3271  if (C) { // just C, no GV.
3272  // Simple constants are not allowed for 's'.
3273  if (ConstraintLetter != 's') {
3274  // gcc prints these as sign extended. Sign extend value to 64 bits
3275  // now; without this it would get ZExt'd later in
3276  // ScheduleDAGSDNodes::EmitNode, which is very generic.
3277  Ops.push_back(DAG.getTargetConstant(C->getSExtValue(),
3278  SDLoc(C), MVT::i64));
3279  }
3280  return;
3281  }
3282  break;
3283  }
3284  }
3285 }
3286 
3287 std::pair<unsigned, const TargetRegisterClass *>
3289  StringRef Constraint,
3290  MVT VT) const {
3291  if (Constraint.empty() || Constraint[0] != '{')
3292  return std::make_pair(0u, static_cast<TargetRegisterClass*>(nullptr));
3293  assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?");
3294 
3295  // Remove the braces from around the name.
3296  StringRef RegName(Constraint.data()+1, Constraint.size()-2);
3297 
3298  std::pair<unsigned, const TargetRegisterClass*> R =
3299  std::make_pair(0u, static_cast<const TargetRegisterClass*>(nullptr));
3300 
3301  // Figure out which register class contains this reg.
3302  for (const TargetRegisterClass *RC : RI->regclasses()) {
3303  // If none of the value types for this register class are valid, we
3304  // can't use it. For example, 64-bit reg classes on 32-bit targets.
3305  if (!isLegalRC(*RI, *RC))
3306  continue;
3307 
3308  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
3309  I != E; ++I) {
3310  if (RegName.equals_lower(RI->getRegAsmName(*I))) {
3311  std::pair<unsigned, const TargetRegisterClass*> S =
3312  std::make_pair(*I, RC);
3313 
3314  // If this register class has the requested value type, return it,
3315  // otherwise keep searching and return the first class found
3316  // if no other is found which explicitly has the requested type.
3317  if (RI->isTypeLegalForClass(*RC, VT))
3318  return S;
3319  if (!R.second)
3320  R = S;
3321  }
3322  }
3323  }
3324 
3325  return R;
3326 }
3327 
3328 //===----------------------------------------------------------------------===//
3329 // Constraint Selection.
3330 
3331 /// Return true of this is an input operand that is a matching constraint like
3332 /// "4".
3334  assert(!ConstraintCode.empty() && "No known constraint!");
3335  return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
3336 }
3337 
3338 /// If this is an input matching constraint, this method returns the output
3339 /// operand it matches.
3341  assert(!ConstraintCode.empty() && "No known constraint!");
3342  return atoi(ConstraintCode.c_str());
3343 }
3344 
3345 /// Split up the constraint string from the inline assembly value into the
3346 /// specific constraints and their prefixes, and also tie in the associated
3347 /// operand values.
3348 /// If this returns an empty vector, and if the constraint string itself
3349 /// isn't empty, there was an error parsing.
3352  const TargetRegisterInfo *TRI,
3353  ImmutableCallSite CS) const {
3354  /// Information about all of the constraints.
3355  AsmOperandInfoVector ConstraintOperands;
3356  const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
3357  unsigned maCount = 0; // Largest number of multiple alternative constraints.
3358 
3359  // Do a prepass over the constraints, canonicalizing them, and building up the
3360  // ConstraintOperands list.
3361  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
3362  unsigned ResNo = 0; // ResNo - The result number of the next output.
3363 
3364  for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
3365  ConstraintOperands.emplace_back(std::move(CI));
3366  AsmOperandInfo &OpInfo = ConstraintOperands.back();
3367 
3368  // Update multiple alternative constraint count.
3369  if (OpInfo.multipleAlternatives.size() > maCount)
3370  maCount = OpInfo.multipleAlternatives.size();
3371 
3372  OpInfo.ConstraintVT = MVT::Other;
3373 
3374  // Compute the value type for each operand.
3375  switch (OpInfo.Type) {
3376  case InlineAsm::isOutput:
3377  // Indirect outputs just consume an argument.
3378  if (OpInfo.isIndirect) {
3379  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
3380  break;
3381  }
3382 
3383  // The return value of the call is this value. As such, there is no
3384  // corresponding argument.
3385  assert(!CS.getType()->isVoidTy() &&
3386  "Bad inline asm!");
3387  if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
3388  OpInfo.ConstraintVT =
3389  getSimpleValueType(DL, STy->getElementType(ResNo));
3390  } else {
3391  assert(ResNo == 0 && "Asm only has one result!");
3392  OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
3393  }
3394  ++ResNo;
3395  break;
3396  case InlineAsm::isInput:
3397  OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
3398  break;
3399  case InlineAsm::isClobber:
3400  // Nothing to do.
3401  break;
3402  }
3403 
3404  if (OpInfo.CallOperandVal) {
3405  llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
3406  if (OpInfo.isIndirect) {
3407  llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
3408  if (!PtrTy)
3409  report_fatal_error("Indirect operand for inline asm not a pointer!");
3410  OpTy = PtrTy->getElementType();
3411  }
3412 
3413  // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
3414  if (StructType *STy = dyn_cast<StructType>(OpTy))
3415  if (STy->getNumElements() == 1)
3416  OpTy = STy->getElementType(0);
3417 
3418  // If OpTy is not a single value, it may be a struct/union that we
3419  // can tile with integers.
3420  if (!OpTy->isSingleValueType() && OpTy->isSized()) {
3421  unsigned BitSize = DL.getTypeSizeInBits(OpTy);
3422  switch (BitSize) {
3423  default: break;
3424  case 1:
3425  case 8:
3426  case 16:
3427  case 32:
3428  case 64:
3429  case 128:
3430  OpInfo.ConstraintVT =
3431  MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
3432  break;
3433  }
3434  } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
3435  unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
3436  OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
3437  } else {
3438  OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
3439  }
3440  }
3441  }
3442 
3443  // If we have multiple alternative constraints, select the best alternative.
3444  if (!ConstraintOperands.empty()) {
3445  if (maCount) {
3446  unsigned bestMAIndex = 0;
3447  int bestWeight = -1;
3448  // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
3449  int weight = -1;
3450  unsigned maIndex;
3451  // Compute the sums of the weights for each alternative, keeping track
3452  // of the best (highest weight) one so far.
3453  for (maIndex = 0; maIndex < maCount; ++maIndex) {
3454  int weightSum = 0;
3455  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3456  cIndex != eIndex; ++cIndex) {
3457  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
3458  if (OpInfo.Type == InlineAsm::isClobber)
3459  continue;
3460 
3461  // If this is an output operand with a matching input operand,
3462  // look up the matching input. If their types mismatch, e.g. one
3463  // is an integer, the other is floating point, or their sizes are
3464  // different, flag it as an maCantMatch.
3465  if (OpInfo.hasMatchingInput()) {
3466  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3467  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3468  if ((OpInfo.ConstraintVT.isInteger() !=
3469  Input.ConstraintVT.isInteger()) ||
3470  (OpInfo.ConstraintVT.getSizeInBits() !=
3471  Input.ConstraintVT.getSizeInBits())) {
3472  weightSum = -1; // Can't match.
3473  break;
3474  }
3475  }
3476  }
3477  weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
3478  if (weight == -1) {
3479  weightSum = -1;
3480  break;
3481  }
3482  weightSum += weight;
3483  }
3484  // Update best.
3485  if (weightSum > bestWeight) {
3486  bestWeight = weightSum;
3487  bestMAIndex = maIndex;
3488  }
3489  }
3490 
3491  // Now select chosen alternative in each constraint.
3492  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3493  cIndex != eIndex; ++cIndex) {
3494  AsmOperandInfo& cInfo = ConstraintOperands[cIndex];
3495  if (cInfo.Type == InlineAsm::isClobber)
3496  continue;
3497  cInfo.selectAlternative(bestMAIndex);
3498  }
3499  }
3500  }
3501 
3502  // Check and hook up tied operands, choose constraint code to use.
3503  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
3504  cIndex != eIndex; ++cIndex) {
3505  AsmOperandInfo& OpInfo = ConstraintOperands[cIndex];
3506 
3507  // If this is an output operand with a matching input operand, look up the
3508  // matching input. If their types mismatch, e.g. one is an integer, the
3509  // other is floating point, or their sizes are different, flag it as an
3510  // error.
3511  if (OpInfo.hasMatchingInput()) {
3512  AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
3513 
3514  if (OpInfo.ConstraintVT != Input.ConstraintVT) {
3515  std::pair<unsigned, const TargetRegisterClass *> MatchRC =
3517  OpInfo.ConstraintVT);
3518  std::pair<unsigned, const TargetRegisterClass *> InputRC =
3520  Input.ConstraintVT);
3521  if ((OpInfo.ConstraintVT.isInteger() !=
3522  Input.ConstraintVT.isInteger()) ||
3523  (MatchRC.second != InputRC.second)) {
3524  report_fatal_error("Unsupported asm: input constraint"
3525  " with a matching output constraint of"
3526  " incompatible type!");
3527  }
3528  }
3529  }
3530  }
3531 
3532  return ConstraintOperands;
3533 }
3534 
3535 /// Return an integer indicating how general CT is.
3537  switch (CT) {
3540  return 0;
3542  return 1;
3544  return 2;
3546  return 3;
3547  }
3548  llvm_unreachable("Invalid constraint type");
3549 }
3550 
3551 /// Examine constraint type and operand type and determine a weight value.
3552 /// This object must already have been set up with the operand type
3553 /// and the current alternative constraint selected.
3556  AsmOperandInfo &info, int maIndex) const {
3558  if (maIndex >= (int)info.multipleAlternatives.size())
3559  rCodes = &info.Codes;
3560  else
3561  rCodes = &info.multipleAlternatives[maIndex].Codes;
3562  ConstraintWeight BestWeight = CW_Invalid;
3563 
3564  // Loop over the options, keeping track of the most general one.
3565  for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
3566  ConstraintWeight weight =
3567  getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
3568  if (weight > BestWeight)
3569  BestWeight = weight;
3570  }
3571 
3572  return BestWeight;
3573 }
3574 
3575 /// Examine constraint type and operand type and determine a weight value.
3576 /// This object must already have been set up with the operand type
3577 /// and the current alternative constraint selected.
3580  AsmOperandInfo &info, const char *constraint) const {
3581  ConstraintWeight weight = CW_Invalid;
3582  Value *CallOperandVal = info.CallOperandVal;
3583  // If we don't have a value, we can't do a match,
3584  // but allow it at the lowest weight.
3585  if (!CallOperandVal)
3586  return CW_Default;
3587  // Look at the constraint type.
3588  switch (*constraint) {
3589  case 'i': // immediate integer.
3590  case 'n': // immediate integer with a known value.
3591  if (isa<ConstantInt>(CallOperandVal))
3592  weight = CW_Constant;
3593  break;
3594  case 's': // non-explicit intregal immediate.
3595  if (isa<GlobalValue>(CallOperandVal))
3596  weight = CW_Constant;
3597  break;
3598  case 'E': // immediate float if host format.
3599  case 'F': // immediate float.
3600  if (isa<ConstantFP>(CallOperandVal))
3601  weight = CW_Constant;
3602  break;
3603  case '<': // memory operand with autodecrement.
3604  case '>': // memory operand with autoincrement.
3605  case 'm': // memory operand.
3606  case 'o': // offsettable memory operand
3607  case 'V': // non-offsettable memory operand
3608  weight = CW_Memory;
3609  break;
3610  case 'r': // general register.
3611  case 'g': // general register, memory operand or immediate integer.
3612  // note: Clang converts "g" to "imr".
3613  if (CallOperandVal->getType()->isIntegerTy())
3614  weight = CW_Register;
3615  break;
3616  case 'X': // any operand.
3617  default:
3618  weight = CW_Default;
3619  break;
3620  }
3621  return weight;
3622 }
3623 
3624 /// If there are multiple different constraints that we could pick for this
3625 /// operand (e.g. "imr") try to pick the 'best' one.
3626 /// This is somewhat tricky: constraints fall into four classes:
3627 /// Other -> immediates and magic values
3628 /// Register -> one specific register
3629 /// RegisterClass -> a group of regs
3630 /// Memory -> memory
3631 /// Ideally, we would pick the most specific constraint possible: if we have
3632 /// something that fits into a register, we would pick it. The problem here
3633 /// is that if we have something that could either be in a register or in
3634 /// memory that use of the register could cause selection of *other*
3635 /// operands to fail: they might only succeed if we pick memory. Because of
3636 /// this the heuristic we use is:
3637 ///
3638 /// 1) If there is an 'other' constraint, and if the operand is valid for
3639 /// that constraint, use it. This makes us take advantage of 'i'
3640 /// constraints when available.
3641 /// 2) Otherwise, pick the most general constraint present. This prefers
3642 /// 'm' over 'r', for example.
3643 ///
3645  const TargetLowering &TLI,
3646  SDValue Op, SelectionDAG *DAG) {
3647  assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
3648  unsigned BestIdx = 0;
3650  int BestGenerality = -1;
3651 
3652  // Loop over the options, keeping track of the most general one.
3653  for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
3655  TLI.getConstraintType(OpInfo.Codes[i]);
3656 
3657  // If this is an 'other' constraint, see if the operand is valid for it.
3658  // For example, on X86 we might have an 'rI' constraint. If the operand
3659  // is an integer in the range [0..31] we want to use I (saving a load
3660  // of a register), otherwise we must use 'r'.
3661  if (CType == TargetLowering::C_Other && Op.getNode()) {
3662  assert(OpInfo.Codes[i].size() == 1 &&
3663  "Unhandled multi-letter 'other' constraint");
3664  std::vector<SDValue> ResultOps;
3665  TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
3666  ResultOps, *DAG);
3667  if (!ResultOps.empty()) {
3668  BestType = CType;
3669  BestIdx = i;
3670  break;
3671  }
3672  }
3673 
3674  // Things with matching constraints can only be registers, per gcc
3675  // documentation. This mainly affects "g" constraints.
3676  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
3677  continue;
3678 
3679  // This constraint letter is more general than the previous one, use it.
3680  int Generality = getConstraintGenerality(CType);
3681  if (Generality > BestGenerality) {
3682  BestType = CType;
3683  BestIdx = i;
3684  BestGenerality = Generality;
3685  }
3686  }
3687 
3688  OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
3689  OpInfo.ConstraintType = BestType;
3690 }
3691 
3692 /// Determines the constraint code and constraint type to use for the specific
3693 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
3695  SDValue Op,
3696  SelectionDAG *DAG) const {
3697  assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
3698 
3699  // Single-letter constraints ('r') are very common.
3700  if (OpInfo.Codes.size() == 1) {
3701  OpInfo.ConstraintCode = OpInfo.Codes[0];
3703  } else {
3704  ChooseConstraint(OpInfo, *this, Op, DAG);
3705  }
3706 
3707  // 'X' matches anything.
3708  if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
3709  // Labels and constants are handled elsewhere ('X' is the only thing
3710  // that matches labels). For Functions, the type here is the type of
3711  // the result, which is not what we want to look at; leave them alone.
3712  Value *v = OpInfo.CallOperandVal;
3713  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
3714  OpInfo.CallOperandVal = v;
3715  return;
3716  }
3717 
3718  // Otherwise, try to resolve it to something we know about by looking at
3719  // the actual operand type.
3720  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
3721  OpInfo.ConstraintCode = Repl;
3723  }
3724  }
3725 }
3726 
3727 /// Given an exact SDIV by a constant, create a multiplication
3728 /// with the multiplicative inverse of the constant.
3730  const SDLoc &dl, SelectionDAG &DAG,
3731  SmallVectorImpl<SDNode *> &Created) {
3732  SDValue Op0 = N->getOperand(0);
3733  SDValue Op1 = N->getOperand(1);
3734  EVT VT = N->getValueType(0);
3735  EVT SVT = VT.getScalarType();
3736  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3737  EVT ShSVT = ShVT.getScalarType();
3738 
3739  bool UseSRA = false;
3740  SmallVector<SDValue, 16> Shifts, Factors;
3741 
3742  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
3743  if (C->isNullValue())
3744  return false;
3745  APInt Divisor = C->getAPIntValue();
3746  unsigned Shift = Divisor.countTrailingZeros();
3747  if (Shift) {
3748  Divisor.ashrInPlace(Shift);
3749  UseSRA = true;
3750  }
3751  // Calculate the multiplicative inverse, using Newton's method.
3752  APInt t;
3753  APInt Factor = Divisor;
3754  while ((t = Divisor * Factor) != 1)
3755  Factor *= APInt(Divisor.getBitWidth(), 2) - t;
3756  Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
3757  Factors.push_back(DAG.getConstant(Factor, dl, SVT));
3758  return true;
3759  };
3760 
3761  // Collect all magic values from the build vector.
3762  if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
3763  return SDValue();
3764 
3765  SDValue Shift, Factor;
3766  if (VT.isVector()) {
3767  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
3768  Factor = DAG.getBuildVector(VT, dl, Factors);
3769  } else {
3770  Shift = Shifts[0];
3771  Factor = Factors[0];
3772  }
3773 
3774  SDValue Res = Op0;
3775 
3776  // Shift the value upfront if it is even, so the LSB is one.
3777  if (UseSRA) {
3778  // TODO: For UDIV use SRL instead of SRA.
3779  SDNodeFlags Flags;
3780  Flags.setExact(true);
3781  Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
3782  Created.push_back(Res.getNode());
3783  }
3784 
3785  return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
3786 }
3787 
3789  SelectionDAG &DAG,
3790  SmallVectorImpl<SDNode *> &Created) const {
3792  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3793  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
3794  return SDValue(N,0); // Lower SDIV as SDIV
3795  return SDValue();
3796 }
3797 
3798 /// Given an ISD::SDIV node expressing a divide by constant,
3799 /// return a DAG expression to select that will generate the same value by
3800 /// multiplying by a magic number.
3801 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3803  bool IsAfterLegalization,
3804  SmallVectorImpl<SDNode *> &Created) const {
3805  SDLoc dl(N);
3806  EVT VT = N->getValueType(0);
3807  EVT SVT = VT.getScalarType();
3808  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
3809  EVT ShSVT = ShVT.getScalarType();
3810  unsigned EltBits = VT.getScalarSizeInBits();
3811 
3812  // Check to see if we can do this.
3813  // FIXME: We should be more aggressive here.
3814  if (!isTypeLegal(VT))
3815  return SDValue();
3816 
3817  // If the sdiv has an 'exact' bit we can use a simpler lowering.
3818  if (N->getFlags().hasExact())
3819  return BuildExactSDIV(*this, N, dl, DAG, Created);
3820 
3821  SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
3822 
3823  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
3824  if (C->isNullValue())
3825  return false;
3826 
3827  const APInt &Divisor = C->getAPIntValue();
3828  APInt::ms magics = Divisor.magic();
3829  int NumeratorFactor = 0;
3830  int ShiftMask = -1;
3831 
3832  if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
3833  // If d is +1/-1, we just multiply the numerator by +1/-1.
3834  NumeratorFactor = Divisor.getSExtValue();
3835  magics.m = 0;
3836  magics.s = 0;
3837  ShiftMask = 0;
3838  } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
3839  // If d > 0 and m < 0, add the numerator.
3840  NumeratorFactor = 1;
3841  } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
3842  // If d < 0 and m > 0, subtract the numerator.
3843  NumeratorFactor = -1;
3844  }
3845 
3846  MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
3847  Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
3848  Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
3849  ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
3850  return true;
3851  };
3852 
3853  SDValue N0 = N->getOperand(0);
3854  SDValue N1 = N->getOperand(1);
3855 
3856  // Collect the shifts / magic values from each element.
3857  if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
3858  return SDValue();
3859 
3860  SDValue MagicFactor, Factor, Shift, ShiftMask;
3861  if (VT.isVector()) {
3862  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
3863  Factor = DAG.getBuildVector(VT, dl, Factors);
3864  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
3865  ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
3866  } else {
3867  MagicFactor = MagicFactors[0];
3868  Factor = Factors[0];
3869  Shift = Shifts[0];
3870  ShiftMask = ShiftMasks[0];
3871  }
3872 
3873  // Multiply the numerator (operand 0) by the magic value.
3874  // FIXME: We should support doing a MUL in a wider type.
3875  SDValue Q;
3876  if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
3878  Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
3879  else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
3881  SDValue LoHi =
3882  DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
3883  Q = SDValue(LoHi.getNode(), 1);
3884  } else
3885  return SDValue(); // No mulhs or equivalent.
3886  Created.push_back(Q.getNode());
3887 
3888  // (Optionally) Add/subtract the numerator using Factor.
3889  Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
3890  Created.push_back(Factor.getNode());
3891  Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
3892  Created.push_back(Q.getNode());
3893 
3894  // Shift right algebraic by shift value.
3895  Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
3896  Created.push_back(Q.getNode());
3897 
3898  // Extract the sign bit, mask it and add it to the quotient.
3899  SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
3900  SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
3901  Created.push_back(T.getNode());
3902  T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
3903  Created.push_back(T.getNode());
3904  return DAG.getNode(ISD::ADD, dl, VT, Q, T);
3905 }
3906 
3907 /// Given an ISD::UDIV node expressing a divide by constant,
3908 /// return a DAG expression to select that will generate the same value by
3909 /// multiplying by a magic number.
3910 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
3912  bool IsAfterLegalization,
3913  SmallVectorImpl<SDNode *> &Created) const {
3914  SDLoc dl(N);
3915  EVT VT = N->getValueType(0);
3916  EVT SVT = VT.getScalarType();
3917  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
3918  EVT ShSVT = ShVT.getScalarType();
3919  unsigned EltBits = VT.getScalarSizeInBits();
3920 
3921  // Check to see if we can do this.
3922  // FIXME: We should be more aggressive here.
3923  if (!isTypeLegal(VT))
3924  return SDValue();
3925 
3926  bool UseNPQ = false;
3927  SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
3928 
3929  auto BuildUDIVPattern = [&](ConstantSDNode *C) {
3930  if (C->isNullValue())
3931  return false;
3932  // FIXME: We should use a narrower constant when the upper
3933  // bits are known to be zero.
3934  APInt Divisor = C->getAPIntValue();
3935  APInt::mu magics = Divisor.magicu();
3936  unsigned PreShift = 0, PostShift = 0;
3937 
3938  // If the divisor is even, we can avoid using the expensive fixup by
3939  // shifting the divided value upfront.
3940  if (magics.a != 0 && !Divisor[0]) {
3941  PreShift = Divisor.countTrailingZeros();
3942  // Get magic number for the shifted divisor.
3943  magics = Divisor.lshr(PreShift).magicu(PreShift);
3944  assert(magics.a == 0 && "Should use cheap fixup now");
3945  }
3946 
3947  APInt Magic = magics.m;
3948 
3949  unsigned SelNPQ;
3950  if (magics.a == 0 || Divisor.isOneValue()) {
3951  assert(magics.s < Divisor.getBitWidth() &&
3952  "We shouldn't generate an undefined shift!");
3953  PostShift = magics.s;
3954  SelNPQ = false;
3955  } else {
3956  PostShift = magics.s - 1;
3957  SelNPQ = true;
3958  }
3959 
3960  PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
3961  MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
3962  NPQFactors.push_back(
3963  DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
3964  : APInt::getNullValue(EltBits),
3965  dl, SVT));
3966  PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
3967  UseNPQ |= SelNPQ;
3968  return true;
3969  };
3970 
3971  SDValue N0 = N->getOperand(0);
3972  SDValue N1 = N->getOperand(1);
3973 
3974  // Collect the shifts/magic values from each element.
3975  if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
3976  return SDValue();
3977 
3978  SDValue PreShift, PostShift, MagicFactor, NPQFactor;
3979  if (VT.isVector()) {
3980  PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
3981  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
3982  NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
3983  PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
3984  } else {
3985  PreShift = PreShifts[0];
3986  MagicFactor = MagicFactors[0];
3987  PostShift = PostShifts[0];
3988  }
3989 
3990  SDValue Q = N0;
3991  Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
3992  Created.push_back(Q.getNode());
3993 
3994  // FIXME: We should support doing a MUL in a wider type.
3995  auto GetMULHU = [&](SDValue X, SDValue Y) {
3996  if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
3998  return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
3999  if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
4001  SDValue LoHi =
4002  DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
4003  return SDValue(LoHi.getNode(), 1);
4004  }
4005  return SDValue(); // No mulhu or equivalent
4006  };
4007 
4008  // Multiply the numerator (operand 0) by the magic value.
4009  Q = GetMULHU(Q, MagicFactor);
4010  if (!Q)
4011  return SDValue();
4012 
4013  Created.push_back(Q.getNode());
4014 
4015  if (UseNPQ) {
4016  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
4017  Created.push_back(NPQ.getNode());
4018 
4019  // For vectors we might have a mix of non-NPQ/NPQ paths, so use
4020  // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
4021  if (VT.isVector())
4022  NPQ = GetMULHU(NPQ, NPQFactor);
4023  else
4024  NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
4025 
4026  Created.push_back(NPQ.getNode());
4027 
4028  Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
4029  Created.push_back(Q.getNode());
4030  }
4031 
4032  Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
4033  Created.push_back(Q.getNode());
4034 
4035  SDValue One = DAG.getConstant(1, dl, VT);
4036  SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
4037  return DAG.getSelect(dl, VT, IsOne, N0, Q);
4038 }
4039 
4040 bool TargetLowering::
4042  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
4043  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
4044  "be a constant integer");
4045  return true;
4046  }
4047 
4048  return false;
4049 }
4050 
4051 //===----------------------------------------------------------------------===//
4052 // Legalization Utilities
4053 //===----------------------------------------------------------------------===//
4054 
4055 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
4056  SDValue LHS, SDValue RHS,
4057  SmallVectorImpl<SDValue> &Result,
4058  EVT HiLoVT, SelectionDAG &DAG,
4060  SDValue LH, SDValue RL, SDValue RH) const {
4061  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
4062  Opcode == ISD::SMUL_LOHI);
4063 
4064  bool HasMULHS = (Kind == MulExpansionKind::Always) ||
4066  bool HasMULHU = (Kind == MulExpansionKind::Always) ||
4068  bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
4070  bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
4072 
4073  if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
4074  return false;
4075 
4076  unsigned OuterBitSize = VT.getScalarSizeInBits();
4077  unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
4078  unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
4079  unsigned RHSSB = DAG.ComputeNumSignBits(RHS);
4080 
4081  // LL, LH, RL, and RH must be either all NULL or all set to a value.
4082  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
4083  (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
4084 
4085  SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
4086  auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
4087  bool Signed) -> bool {
4088  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
4089  Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
4090  Hi = SDValue(Lo.getNode(), 1);
4091  return true;
4092  }
4093  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
4094  Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
4095  Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
4096  return true;
4097  }
4098  return false;
4099  };
4100 
4101  SDValue Lo, Hi;
4102 
4103  if (!LL.getNode() && !RL.getNode() &&
4105  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
4106  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
4107  }
4108 
4109  if (!LL.getNode())
4110  return false;
4111 
4112  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
4113  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
4114  DAG.MaskedValueIsZero(RHS, HighMask)) {
4115  // The inputs are both zero-extended.
4116  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
4117  Result.push_back(Lo);
4118  Result.push_back(Hi);
4119  if (Opcode != ISD::MUL) {
4120  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
4121  Result.push_back(Zero);
4122  Result.push_back(Zero);
4123  }
4124  return true;
4125  }
4126  }
4127 
4128  if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
4129  RHSSB > InnerBitSize) {
4130  // The input values are both sign-extended.
4131  // TODO non-MUL case?
4132  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
4133  Result.push_back(Lo);
4134  Result.push_back(Hi);
4135  return true;
4136  }
4137  }
4138 
4139  unsigned ShiftAmount = OuterBitSize - InnerBitSize;
4140  EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
4141  if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
4142  // FIXME getShiftAmountTy does not always return a sensible result when VT
4143  // is an illegal type, and so the type may be too small to fit the shift
4144  // amount. Override it with i32. The shift will have to be legalized.
4145  ShiftAmountTy = MVT::i32;
4146  }
4147  SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
4148 
4149  if (!LH.getNode() && !RH.getNode() &&
4152  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
4153  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
4154  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
4155  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
4156  }
4157 
4158  if (!LH.getNode())
4159  return false;
4160 
4161  if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
4162  return false;
4163 
4164  Result.push_back(Lo);
4165 
4166  if (Opcode == ISD::MUL) {
4167  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
4168  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
4169  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
4170  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
4171  Result.push_back(Hi);
4172  return true;
4173  }
4174 
4175  // Compute the full width result.
4176  auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
4177  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
4178  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
4179  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
4180  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
4181  };
4182 
4183  SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
4184  if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
4185  return false;
4186 
4187  // This is effectively the add part of a multiply-add of half-sized operands,
4188  // so it cannot overflow.
4189  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
4190 
4191  if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
4192  return false;
4193 
4194  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
4195  EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
4196 
4197  bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
4199  if (UseGlue)
4200  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
4201  Merge(Lo, Hi));
4202  else
4203  Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
4204  Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
4205 
4206  SDValue Carry = Next.getValue(1);
4207  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
4208  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
4209 
4210  if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
4211  return false;
4212 
4213  if (UseGlue)
4214  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
4215  Carry);
4216  else
4217  Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
4218  Zero, Carry);
4219 
4220  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
4221 
4222  if (Opcode == ISD::SMUL_LOHI) {
4223  SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
4224  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
4225  Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
4226 
4227  NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
4228  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
4229  Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
4230  }
4231 
4232  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
4233  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
4234  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
4235  return true;
4236 }
4237 
4240  SDValue LL, SDValue LH, SDValue RL,
4241  SDValue RH) const {
4242  SmallVector<SDValue, 2> Result;
4243  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), N,
4244  N->getOperand(0), N->getOperand(1), Result, HiLoVT,
4245  DAG, Kind, LL, LH, RL, RH);
4246  if (Ok) {
4247  assert(Result.size() == 2);
4248  Lo = Result[0];
4249  Hi = Result[1];
4250  }
4251  return Ok;
4252 }
4253 
4255  SelectionDAG &DAG) const {
4256  EVT VT = Node->getValueType(0);
4257 
4258  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
4262  return false;
4263 
4264  // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
4265  // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
4266  SDValue X = Node->getOperand(0);
4267  SDValue Y = Node->getOperand(1);
4268  SDValue Z = Node->getOperand(2);
4269 
4270  unsigned EltSizeInBits = VT.getScalarSizeInBits();
4271  bool IsFSHL = Node->getOpcode() == ISD::FSHL;
4272  SDLoc DL(SDValue(Node, 0));
4273 
4274  EVT ShVT = Z.getValueType();
4275  SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
4276  SDValue Zero = DAG.getConstant(0, DL, ShVT);
4277 
4278  SDValue ShAmt;
4279  if (isPowerOf2_32(EltSizeInBits)) {
4280  SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
4281  ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
4282  } else {
4283  ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
4284  }
4285 
4286  SDValue InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
4287  SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
4288  SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
4289  SDValue Or = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
4290 
4291  // If (Z % BW == 0), then the opposite direction shift is shift-by-bitwidth,
4292  // and that is undefined. We must compare and select to avoid UB.
4293  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShVT);
4294 
4295  // For fshl, 0-shift returns the 1st arg (X).
4296  // For fshr, 0-shift returns the 2nd arg (Y).
4297  SDValue IsZeroShift = DAG.getSetCC(DL, CCVT, ShAmt, Zero, ISD::SETEQ);
4298  Result = DAG.getSelect(DL, VT, IsZeroShift, IsFSHL ? X : Y, Or);
4299  return true;
4300 }
4301 
4302 // TODO: Merge with expandFunnelShift.
4304  SelectionDAG &DAG) const {
4305  EVT VT = Node->getValueType(0);
4306  unsigned EltSizeInBits = VT.getScalarSizeInBits();
4307  bool IsLeft = Node->getOpcode() == ISD::ROTL;
4308  SDValue Op0 = Node->getOperand(0);
4309  SDValue Op1 = Node->getOperand(1);
4310  SDLoc DL(SDValue(Node, 0));
4311 
4312  EVT ShVT = Op1.getValueType();
4313  SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
4314 
4315  // If a rotate in the other direction is legal, use it.
4316  unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
4317  if (isOperationLegal(RevRot, VT)) {
4318  SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
4319  Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
4320  return true;
4321  }
4322 
4323  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
4328  return false;
4329 
4330  // Otherwise,
4331  // (rotl x, c) -> (or (shl x, (and c, w-1)), (srl x, (and w-c, w-1)))
4332  // (rotr x, c) -> (or (srl x, (and c, w-1)), (shl x, (and w-c, w-1)))
4333  //
4334  assert(isPowerOf2_32(EltSizeInBits) && EltSizeInBits > 1 &&
4335  "Expecting the type bitwidth to be a power of 2");
4336  unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
4337  unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
4338  SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
4339  SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, Op1);
4340  SDValue And0 = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
4341  SDValue And1 = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
4342  Result = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ShOpc, DL, VT, Op0, And0),
4343  DAG.getNode(HsOpc, DL, VT, Op0, And1));
4344  return true;
4345 }
4346 
4348  SelectionDAG &DAG) const {
4349  SDValue Src = Node->getOperand(0);
4350  EVT SrcVT = Src.getValueType();
4351  EVT DstVT = Node->getValueType(0);
4352  SDLoc dl(SDValue(Node, 0));
4353 
4354  // FIXME: Only f32 to i64 conversions are supported.
4355  if (SrcVT != MVT::f32 || DstVT != MVT::i64)
4356  return false;
4357 
4358  // Expand f32 -> i64 conversion
4359  // This algorithm comes from compiler-rt's implementation of fixsfdi:
4360  // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c
4361  unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
4362  EVT IntVT = SrcVT.changeTypeToInteger();
4363  EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
4364 
4365  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
4366  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
4367  SDValue Bias = DAG.getConstant(127, dl, IntVT);
4368  SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
4369  SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
4370  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
4371 
4372  SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
4373 
4374  SDValue ExponentBits = DAG.getNode(
4375  ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
4376  DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
4377  SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
4378 
4379  SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
4380  DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
4381  DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
4382  Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
4383 
4384  SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
4385  DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
4386  DAG.getConstant(0x00800000, dl, IntVT));
4387 
4388  R = DAG.getZExtOrTrunc(R, dl, DstVT);
4389 
4390  R = DAG.getSelectCC(
4391  dl, Exponent, ExponentLoBit,
4392  DAG.getNode(ISD::SHL, dl, DstVT, R,
4393  DAG.getZExtOrTrunc(
4394  DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
4395  dl, IntShVT)),
4396  DAG.getNode(ISD::SRL, dl, DstVT, R,
4397  DAG.getZExtOrTrunc(
4398  DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
4399  dl, IntShVT)),
4400  ISD::SETGT);
4401 
4402  SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
4403  DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
4404 
4405  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
4406  DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
4407  return true;
4408 }
4409 
4411  SelectionDAG &DAG) const {
4412  SDLoc dl(SDValue(Node, 0));
4413  SDValue Src = Node->getOperand(0);
4414 
4415  EVT SrcVT = Src.getValueType();
4416  EVT DstVT = Node->getValueType(0);
4417  EVT SetCCVT =
4418  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
4419 
4420  // Only expand vector types if we have the appropriate vector bit operations.
4421  if (DstVT.isVector() && (!isOperationLegalOrCustom(ISD::FP_TO_SINT, DstVT) ||
4423  return false;
4424 
4425  // If the maximum float value is smaller then the signed integer range,
4426  // the destination signmask can't be represented by the float, so we can
4427  // just use FP_TO_SINT directly.
4428  const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
4429  APFloat APF(APFSem, APInt::getNullValue(SrcVT.getScalarSizeInBits()));
4430  APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
4431  if (APFloat::opOverflow &
4432  APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
4433  Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
4434  return true;
4435  }
4436 
4437  SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
4438  SDValue Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
4439 
4440  bool Strict = shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
4441  if (Strict) {
4442  // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
4443  // signmask then offset (the result of which should be fully representable).
4444  // Sel = Src < 0x8000000000000000
4445  // Val = select Sel, Src, Src - 0x8000000000000000
4446  // Ofs = select Sel, 0, 0x8000000000000000
4447  // Result = fp_to_sint(Val) ^ Ofs
4448 
4449  // TODO: Should any fast-math-flags be set for the FSUB?
4450  SDValue Val = DAG.getSelect(dl, SrcVT, Sel, Src,
4451  DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
4452  SDValue Ofs = DAG.getSelect(dl, DstVT, Sel, DAG.getConstant(0, dl, DstVT),
4453  DAG.getConstant(SignMask, dl, DstVT));
4454  Result = DAG.getNode(ISD::XOR, dl, DstVT,
4455  DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val), Ofs);
4456  } else {
4457  // Expand based on maximum range of FP_TO_SINT:
4458  // True = fp_to_sint(Src)
4459  // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
4460  // Result = select (Src < 0x8000000000000000), True, False
4461 
4462  SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
4463  // TODO: Should any fast-math-flags be set for the FSUB?
4464  SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
4465  DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
4466  False = DAG.getNode(ISD::XOR, dl, DstVT, False,
4467  DAG.getConstant(SignMask, dl, DstVT));
4468  Result = DAG.getSelect(dl, DstVT, Sel, True, False);
4469  }
4470  return true;
4471 }
4472 
4474  SelectionDAG &DAG) const {
4475  SDValue Src = Node->getOperand(0);
4476  EVT SrcVT = Src.getValueType();
4477  EVT DstVT = Node->getValueType(0);
4478 
4479  if (SrcVT.getScalarType() != MVT::i64)
4480  return false;
4481 
4482  SDLoc dl(SDValue(Node, 0));
4483  EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());
4484 
4485  if (DstVT.getScalarType() == MVT::f32) {
4486  // Only expand vector types if we have the appropriate vector bit
4487  // operations.
4488  if (SrcVT.isVector() &&
4489  (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
4494  return false;
4495 
4496  // For unsigned conversions, convert them to signed conversions using the
4497  // algorithm from the x86_64 __floatundidf in compiler_rt.
4498  SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
4499 
4500  SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
4501  SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Src, ShiftConst);
4502  SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
4503  SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Src, AndConst);
4504  SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
4505 
4506  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Or);
4507  SDValue Slow = DAG.getNode(ISD::FADD, dl, DstVT, SignCvt, SignCvt);
4508 
4509  // TODO: This really should be implemented using a branch rather than a
4510  // select. We happen to get lucky and machinesink does the right
4511  // thing most of the time. This would be a good candidate for a
4512  // pseudo-op, or, even better, for whole-function isel.
4513  EVT SetCCVT =
4514  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
4515 
4516  SDValue SignBitTest = DAG.getSetCC(
4517  dl, SetCCVT, Src, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
4518  Result = DAG.getSelect(dl, DstVT, SignBitTest, Slow, Fast);
4519  return true;
4520  }
4521 
4522  if (DstVT.getScalarType() == MVT::f64) {
4523  // Only expand vector types if we have the appropriate vector bit
4524  // operations.
4525  if (SrcVT.isVector() &&
4526  (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
4531  return false;
4532 
4533  // Implementation of unsigned i64 to f64 following the algorithm in
4534  // __floatundidf in compiler_rt. This implementation has the advantage
4535  // of performing rounding correctly, both in the default rounding mode
4536  // and in all alternate rounding modes.
4537  SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
4538  SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
4539  BitsToDouble(UINT64_C(0x4530000000100000)), dl, DstVT);
4540  SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
4541  SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
4542  SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
4543 
4544  SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
4545  SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
4546  SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
4547  SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
4548  SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
4549  SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
4550  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
4551  Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
4552  return true;
4553  }
4554 
4555  return false;
4556 }
4557 
4559  SelectionDAG &DAG) const {
4560  SDLoc dl(Node);
4561  unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
4563  EVT VT = Node->getValueType(0);
4564  if (isOperationLegalOrCustom(NewOp, VT)) {
4565  SDValue Quiet0 = Node->getOperand(0);
4566  SDValue Quiet1 = Node->getOperand(1);
4567 
4568  if (!Node->getFlags().hasNoNaNs()) {
4569  // Insert canonicalizes if it's possible we need to quiet to get correct
4570  // sNaN behavior.
4571  if (!DAG.isKnownNeverSNaN(Quiet0)) {
4572  Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
4573  Node->getFlags());
4574  }
4575  if (!DAG.isKnownNeverSNaN(Quiet1)) {
4576  Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
4577  Node->getFlags());
4578  }
4579  }
4580 
4581  return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
4582  }
4583 
4584  return SDValue();
4585 }
4586 
4588  SelectionDAG &DAG) const {
4589  SDLoc dl(Node);
4590  EVT VT = Node->getValueType(0);
4591  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4592  SDValue Op = Node->getOperand(0);
4593  unsigned Len = VT.getScalarSizeInBits();
4594  assert(VT.isInteger() && "CTPOP not implemented for this type.");
4595 
4596  // TODO: Add support for irregular type lengths.
4597  if (!(Len <= 128 && Len % 8 == 0))
4598  return false;
4599 
4600  // Only expand vector types if we have the appropriate vector bit operations.
4601  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||
4604  (Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||
4606  return false;
4607 
4608  // This is the "best" algorithm from
4609  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
4610  SDValue Mask55 =
4611  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
4612  SDValue Mask33 =
4613  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
4614  SDValue Mask0F =
4615  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
4616  SDValue Mask01 =
4617  DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
4618 
4619  // v = v - ((v >> 1) & 0x55555555...)
4620  Op = DAG.getNode(ISD::SUB, dl, VT, Op,
4621  DAG.getNode(ISD::AND, dl, VT,
4622  DAG.getNode(ISD::SRL, dl, VT, Op,
4623  DAG.getConstant(1, dl, ShVT)),
4624  Mask55));
4625  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
4626  Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
4627  DAG.getNode(ISD::AND, dl, VT,
4628  DAG.getNode(ISD::SRL, dl, VT, Op,
4629  DAG.getConstant(2, dl, ShVT)),
4630  Mask33));
4631  // v = (v + (v >> 4)) & 0x0F0F0F0F...
4632  Op = DAG.getNode(ISD::AND, dl, VT,
4633  DAG.getNode(ISD::ADD, dl, VT, Op,
4634  DAG.getNode(ISD::SRL, dl, VT, Op,
4635  DAG.getConstant(4, dl, ShVT))),
4636  Mask0F);
4637  // v = (v * 0x01010101...) >> (Len - 8)
4638  if (Len > 8)
4639  Op =
4640  DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
4641  DAG.getConstant(Len - 8, dl, ShVT));
4642 
4643  Result = Op;
4644  return true;
4645 }
4646 
4648  SelectionDAG &DAG) const {
4649  SDLoc dl(Node);
4650  EVT VT = Node->getValueType(0);
4651  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
4652  SDValue Op = Node->getOperand(0);
4653  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
4654 
4655  // If the non-ZERO_UNDEF version is supported we can use that instead.
4656  if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
4658  Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);
4659  return true;
4660  }
4661 
4662  // If the ZERO_UNDEF version is supported use that and handle the zero case.
4664  EVT SetCCVT =
4665  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
4666  SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
4667  SDValue Zero = DAG.getConstant(0, dl, VT);
4668  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
4669  Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
4670  DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
4671  return true;
4672  }
4673 
4674  // Only expand vector types if we have the appropriate vector bit operations.
4675  if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
4679  return false;
4680 
4681  // for now, we do this:
4682  // x = x | (x >> 1);
4683  // x = x | (x >> 2);
4684  // ...
4685  // x = x | (x >>16);
4686  // x = x | (x >>32); // for 64-bit input
4687  // return popcount(~x);
4688  //
4689  // Ref: "Hacker's Delight" by Henry Warren
4690  for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {
4691  SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
4692  Op = DAG.getNode(ISD::OR, dl, VT, Op,
4693  DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
4694  }
4695  Op = DAG.getNOT(dl, Op, VT);
4696  Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
4697  return true;
4698 }
4699 
4701  SelectionDAG &DAG) const {
4702  SDLoc dl(Node);
4703  EVT VT = Node->getValueType(0);
4704  SDValue Op = Node->getOperand(0);
4705  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
4706 
4707  // If the non-ZERO_UNDEF version is supported we can use that instead.
4708  if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
4710  Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
4711  return true;
4712  }
4713 
4714  // If the ZERO_UNDEF version is supported use that and handle the zero case.
4716  EVT SetCCVT =
4717  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
4718  SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
4719  SDValue Zero = DAG.getConstant(0, dl, VT);
4720  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
4721  Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
4722  DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
4723  return true;
4724  }
4725 
4726  // Only expand vector types if we have the appropriate vector bit operations.
4727  if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
4733  return false;
4734 
4735  // for now, we use: { return popcount(~x & (x - 1)); }
4736  // unless the target has ctlz but not ctpop, in which case we use:
4737  // { return 32 - nlz(~x & (x-1)); }
4738  // Ref: "Hacker's Delight" by Henry Warren
4739  SDValue Tmp = DAG.getNode(
4740  ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
4741  DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
4742 
4743  // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
4745  Result =
4746  DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
4747  DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
4748  return true;
4749  }
4750 
4751  Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
4752  return true;
4753 }
4754 
4756  SelectionDAG &DAG) const {
4757  SDLoc SL(LD);
4758  SDValue Chain = LD->getChain();
4759  SDValue BasePTR = LD->getBasePtr();
4760  EVT SrcVT = LD->getMemoryVT();
4762 
4763  unsigned NumElem = SrcVT.getVectorNumElements();
4764 
4765  EVT SrcEltVT = SrcVT.getScalarType();
4766  EVT DstEltVT = LD->getValueType(0).getScalarType();
4767 
4768  unsigned Stride = SrcEltVT.getSizeInBits() / 8;
4769  assert(SrcEltVT.isByteSized());
4770 
4772  SmallVector<SDValue, 8> LoadChains;
4773 
4774  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
4775  SDValue ScalarLoad =
4776  DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
4777  LD->getPointerInfo().getWithOffset(Idx * Stride),
4778  SrcEltVT, MinAlign(LD->getAlignment(), Idx * Stride),
4779  LD->getMemOperand()->getFlags(), LD->getAAInfo());
4780 
4781  BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, Stride);
4782 
4783  Vals.push_back(ScalarLoad.getValue(0));
4784  LoadChains.push_back(ScalarLoad.getValue(1));
4785  }
4786 
4787  SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
4788  SDValue Value = DAG.getBuildVector(LD->getValueType(0), SL, Vals);
4789 
4790  return DAG.getMergeValues({ Value, NewChain }, SL);
4791 }
4792 
4794  SelectionDAG &DAG) const {
4795  SDLoc SL(ST);
4796 
4797  SDValue Chain = ST->getChain();
4798  SDValue BasePtr = ST->getBasePtr();
4799  SDValue Value = ST->getValue();
4800  EVT StVT = ST->getMemoryVT();
4801 
4802  // The type of the data we want to save
4803  EVT RegVT = Value.getValueType();
4804  EVT RegSclVT = RegVT.getScalarType();
4805 
4806  // The type of data as saved in memory.
4807  EVT MemSclVT = StVT.getScalarType();
4808 
4809  EVT IdxVT = getVectorIdxTy(DAG.getDataLayout());
4810  unsigned NumElem = StVT.getVectorNumElements();
4811 
4812  // A vector must always be stored in memory as-is, i.e. without any padding
4813  // between the elements, since various code depend on it, e.g. in the
4814  // handling of a bitcast of a vector type to int, which may be done with a
4815  // vector store followed by an integer load. A vector that does not have
4816  // elements that are byte-sized must therefore be stored as an integer
4817  // built out of the extracted vector elements.
4818  if (!MemSclVT.isByteSized()) {
4819  unsigned NumBits = StVT.getSizeInBits();
4820  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
4821 
4822  SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
4823 
4824  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
4825  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
4826  DAG.getConstant(Idx, SL, IdxVT));
4827  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
4828  SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
4829  unsigned ShiftIntoIdx =
4830  (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
4831  SDValue ShiftAmount =
4832  DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
4833  SDValue ShiftedElt =
4834